From 96916155c91b78bba6ce81b02b086e14872a0dcc Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 23 Feb 2022 19:46:26 +0100 Subject: 'tokenize', to make this work better as a query type --- .../prelude/query/parser/NoGrammarParser.java | 51 ---------------------- .../yahoo/prelude/query/parser/TokenizeParser.java | 46 +++++++++++++++++++ .../src/main/java/com/yahoo/search/Query.java | 4 +- .../yahoo/search/query/parser/ParserFactory.java | 4 +- .../prelude/query/parser/test/ParseTestCase.java | 8 ++-- 5 files changed, 53 insertions(+), 60 deletions(-) delete mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java (limited to 'container-search/src') diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java deleted file mode 100644 index 70e5ba66c6a..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.query.parser; - -import com.yahoo.prelude.query.BlockItem; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.Substring; -import com.yahoo.prelude.query.WeakAndItem; -import com.yahoo.prelude.query.WordItem; -import com.yahoo.search.query.parser.ParserEnvironment; - -import static com.yahoo.prelude.query.parser.Token.Kind.MINUS; -import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER; -import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE; -import static com.yahoo.prelude.query.parser.Token.Kind.WORD; - -/** - * A parser which turns contiguous searchable character into tokens and filters out other characters. - * The resulting tokens are collected into a WeakAnd item. - * - * @author bratseth - */ -public final class NoGrammarParser extends AbstractParser { - - public NoGrammarParser(ParserEnvironment environment) { - super(environment); - } - - @Override - protected Item parseItems() { - WeakAndItem weakAnd = new WeakAndItem(); - Token token; - while (null != (token = tokens.next())) { - Item termItem = toTerm(token); - if (termItem != null) - weakAnd.addItem(termItem); - } - return weakAnd; - } - - /** Returns the item representing this token if it is searchable, and null otherwise */ - private Item toTerm(Token token) { - if (token.kind == WORD) - return segment("", token, false); - else if (token.kind == NUMBER) - return new IntItem(token.image); - else - return null; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java new file mode 100644 index 00000000000..dbbc321d057 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.search.query.parser.ParserEnvironment; + +import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER; +import static com.yahoo.prelude.query.parser.Token.Kind.WORD; + +/** + * A parser which turns contiguous searchable character into tokens and filters out other characters. + * The resulting tokens are collected into a WeakAnd item. + * + * @author bratseth + */ +public final class TokenizeParser extends AbstractParser { + + public TokenizeParser(ParserEnvironment environment) { + super(environment); + } + + @Override + protected Item parseItems() { + WeakAndItem weakAnd = new WeakAndItem(); + Token token; + while (null != (token = tokens.next())) { + Item termItem = toTerm(token); + if (termItem != null) + weakAnd.addItem(termItem); + } + return weakAnd; + } + + /** Returns the item representing this token if it is searchable, and null otherwise */ + private Item toTerm(Token token) { + if (token.kind == WORD) + return segment("", token, false); + else if (token.kind == NUMBER) + return new IntItem(token.image); + else + return null; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 83be5e7f6cd..fb7281e1f24 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search; -import ai.vespa.cloud.Environment; -import ai.vespa.cloud.Zone; import ai.vespa.cloud.ZoneInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -110,7 +108,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { YQL(6, "yql"), SELECT(7, "select"), WEAKAND(8, "weakAnd"), - NONE(9, "none"); + TOKENIZE(9, "tokenize"); private final int intValue; private final String stringValue; diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 4d1cdc320cc..40d1c3e7877 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -46,8 +46,8 @@ public final class ParserFactory { return new SelectParser(environment); case WEAKAND: return new AllParser(environment, true); - case NONE: - return new NoGrammarParser(environment); + case TOKENIZE: + return new TokenizeParser(environment); default: throw new UnsupportedOperationException(type.toString()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 7f51cfa1667..77d8b2b973e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -2565,21 +2565,21 @@ public class ParseTestCase { @Test public void testNoGrammar1() { - tester.assertParsed("WEAKAND(100) foobar", "foobar", Query.Type.NONE); + tester.assertParsed("WEAKAND(100) foobar", "foobar", Query.Type.TOKENIZE); } @Test public void testNoGrammar2() { - tester.assertParsed("WEAKAND(100) foobar", "-foobar", Query.Type.NONE); + tester.assertParsed("WEAKAND(100) foobar", "-foobar", Query.Type.TOKENIZE); } @Test public void testNoGrammar3() { - tester.assertParsed("WEAKAND(100) foo bar", "foo -bar", Query.Type.NONE); + tester.assertParsed("WEAKAND(100) foo bar", "foo -bar", Query.Type.TOKENIZE); } @Test public void testNoGrammar4() { - tester.assertParsed("WEAKAND(100) foo bar baz one two 37", "foo -(bar baz \"one two\" 37)", Query.Type.NONE); + tester.assertParsed("WEAKAND(100) foo bar baz one two 37", "foo -(bar baz \"one two\" 37)", Query.Type.TOKENIZE); } } -- cgit v1.2.3