Merge pull request #21353 from vespa-engine/bratseth/grammar-none

Add grammar:none

8 files changed, 73 insertions, 9 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 8cd32985057..9cc519ed9e2 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -1898,7 +1898,8 @@
       "public static final enum com.yahoo.search.Query$Type PROGRAMMATIC",
       "public static final enum com.yahoo.search.Query$Type YQL",
       "public static final enum com.yahoo.search.Query$Type SELECT",
-      "public static final enum com.yahoo.search.Query$Type WEAKAND"
+      "public static final enum com.yahoo.search.Query$Type WEAKAND",
+      "public static final enum com.yahoo.search.Query$Type TOKENIZE"
     ]
   },
   "com.yahoo.search.Query": {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
index f2a0fdba7ff..6a005bc0ec9 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
@@ -13,7 +13,6 @@ import java.util.Set;
 
 /**
  * @author Simon Thoresen Hult
- * @since 5.1.4
  */
 public final class ProgrammaticParser implements CustomParser {
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index 498b3185489..a35ad91acbd 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -7,7 +7,6 @@ import com.yahoo.search.query.parser.ParserEnvironment;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import static com.yahoo.prelude.query.parser.Token.Kind.*;
 
@@ -444,7 +443,7 @@ abstract class StructuredParser extends AbstractParser {
      *
      * @param quoted whether this token is inside quotes
      */
-    private Item word(String indexName, boolean quoted) {
+    protected Item word(String indexName, boolean quoted) {
         int position = tokens.getPosition();
         Item item = null;
 
@@ -470,7 +469,7 @@ abstract class StructuredParser extends AbstractParser {
                     if (buffer == null) {
                         buffer = getStringContents(item);
                     }
-                    buffer.append(token.toString());
+                    buffer.append(token);
                     tokens.skipNoIgnore();
                     token = tokens.currentNoIgnore();
                 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java
new file mode 100644
index 00000000000..dbbc321d057
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java
@@ -0,0 +1,46 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query.parser;
+
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.search.query.parser.ParserEnvironment;
+
+import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER;
+import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
+
+/**
+ * A parser which turns contiguous searchable character into tokens and filters out other characters.
+ * The resulting tokens are collected into a WeakAnd item.
+ *
+ * @author bratseth
+ */
+public final class TokenizeParser extends AbstractParser {
+
+    public TokenizeParser(ParserEnvironment environment) {
+        super(environment);
+    }
+
+    @Override
+    protected Item parseItems() {
+        WeakAndItem weakAnd = new WeakAndItem();
+        Token token;
+        while (null != (token = tokens.next())) {
+            Item termItem = toTerm(token);
+            if (termItem != null)
+                weakAnd.addItem(termItem);
+        }
+        return weakAnd;
+    }
+
+    /** Returns the item representing this token if it is searchable, and null otherwise */
+    private Item toTerm(Token token) {
+        if (token.kind == WORD)
+            return segment("", token, false);
+        else if (token.kind == NUMBER)
+            return new IntItem(token.image);
+        else
+            return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 1c38bb8f876..fb7281e1f24 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -1,8 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search;
 
-import ai.vespa.cloud.Environment;
-import ai.vespa.cloud.Zone;
 import ai.vespa.cloud.ZoneInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -109,7 +107,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         PROGRAMMATIC(5, "prog"),
         YQL(6, "yql"),
         SELECT(7, "select"),
-        WEAKAND(8, "weakAnd");
+        WEAKAND(8, "weakAnd"),
+        TOKENIZE(9, "tokenize");
 
         private final int intValue;
         private final String stringValue;
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
index f9b8f1785db..40d1c3e7877 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -46,6 +46,8 @@ public final class ParserFactory {
                 return new SelectParser(environment);
             case WEAKAND:
                 return new AllParser(environment, true);
+            case TOKENIZE:
+                return new TokenizeParser(environment);
             default:
                 throw new UnsupportedOperationException(type.toString());
         }
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 846f70fd26b..26508fec3c4 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -1461,7 +1461,6 @@ public class YqlParser implements Parser {
         return (Item) leafStyleSettings(ast, wordItem);
     }
 
-    @SuppressWarnings({"deprecation"})
     private boolean shouldSegment(String field, OperatorNode<ExpressionOperator> ast, boolean fromQuery) {
         return fromQuery && ! indexFactsSession.getIndex(indexNameExpander.expand(field)).isAttribute();
     }
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index 18a73625967..77d8b2b973e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -2563,4 +2563,23 @@ public class ParseTestCase {
         tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL);
     }
 
+    @Test
+    public void testNoGrammar1() {
+        tester.assertParsed("WEAKAND(100) foobar", "foobar", Query.Type.TOKENIZE);
+    }
+
+    @Test
+    public void testNoGrammar2() {
+        tester.assertParsed("WEAKAND(100) foobar", "-foobar", Query.Type.TOKENIZE);
+    }
+
+    @Test
+    public void testNoGrammar3() {
+        tester.assertParsed("WEAKAND(100) foo bar", "foo -bar", Query.Type.TOKENIZE);
+    }
+
+    @Test
+    public void testNoGrammar4() {
+        tester.assertParsed("WEAKAND(100) foo bar baz one two 37", "foo -(bar baz \"one two\" 37)", Query.Type.TOKENIZE);
+    }
 }