diff options
10 files changed, 121 insertions, 62 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 43ce1578e04..6201ec4bfd9 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1896,7 +1896,8 @@ "public static final enum com.yahoo.search.Query$Type WEB", "public static final enum com.yahoo.search.Query$Type PROGRAMMATIC", "public static final enum com.yahoo.search.Query$Type YQL", - "public static final enum com.yahoo.search.Query$Type SELECT" + "public static final enum com.yahoo.search.Query$Type SELECT", + "public static final enum com.yahoo.search.Query$Type WEAKAND" ] }, "com.yahoo.search.Query": { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 3759dbbcbee..80a2320b039 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query.parser; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NotItem; @@ -10,6 +11,7 @@ import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.ParserEnvironment; @@ -26,8 +28,16 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; */ public class AllParser extends SimpleParser { - public AllParser(ParserEnvironment environment) { + private final boolean weakAnd; + + /** + * Creates an And parser + * + * @param weakAnd false to parse into AndItem (by default), true to parse to WeakAnd + */ + public AllParser(ParserEnvironment environment, boolean weakAnd) { super(environment); + this.weakAnd = weakAnd; } @Override @@ -42,7 +52,7 @@ public class AllParser extends SimpleParser { protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and and'ed items, then combine. - AndItem and = null; + CompositeItem and = null; NotItem not = null; // Store negatives here as we go Item current; @@ -88,13 +98,17 @@ public class AllParser extends SimpleParser { return root.getRoot() instanceof NullItem ? null : root.getRoot(); } - protected AndItem addAnd(Item item, AndItem and) { + protected CompositeItem addAnd(Item item, CompositeItem and) { if (and == null) - and = new AndItem(); + and = createAnd(); and.addItem(item); return and; } + private CompositeItem createAnd() { + return weakAnd ? new WeakAndItem() : new AndItem(); + } + protected OrItem addOr(Item item, OrItem or) { if (or == null) or = new OrItem(); @@ -124,7 +138,7 @@ public class AllParser extends SimpleParser { if (item != null) { isComposited = true; if (item instanceof OrItem) { // Turn into And - AndItem and = new AndItem(); + CompositeItem and = createAnd(); for (Iterator<Item> i = ((OrItem) item).getItemIterator(); i.hasNext();) { and.addItem(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java index a7dbaa94fc0..d7c7dec4798 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java @@ -1,11 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.parser.ParserEnvironment; -import java.util.Set; - /** * Parser for web search queries. Language: * @@ -21,37 +24,38 @@ import java.util.Set; public class WebParser extends AllParser { public WebParser(ParserEnvironment environment) { - super(environment); + super(environment, false); } - protected @Override Item parseItemsBody() { + @Override + protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine. - AndItem and=null; - OrItem or=null; - NotItem not=null; // Store negatives here as we go + CompositeItem and = null; + OrItem or = null; + NotItem not = null; // Store negatives here as we go Item current; // Find all items do { - current=negativeItem(); - if (current!=null) { - not=addNot(current,not); + current = negativeItem(); + if (current != null) { + not = addNot(current, not); continue; } - current=positiveItem(); - if (current==null) + current = positiveItem(); + if (current == null) current = indexableItem(); - if (current!=null) { - if (and!=null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { - if (or==null) - or=addOr(and,or); - and=new AndItem(); + if (current != null) { + if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { + if (or == null) + or = addOr(and, or); + and = new AndItem(); or.addItem(and); } else { - and=addAnd(current,and); + and = addAnd(current, and); } } @@ -60,21 +64,17 @@ public class WebParser extends AllParser { } while (tokens.hasNext()); // Combine the items - Item topLevel=and; + Item topLevel = and; - if (or!=null) - topLevel=or; + if (or != null) + topLevel = or; - if (not!=null && topLevel!=null) { + if (not != null && topLevel != null) { not.setPositiveItem(topLevel); - topLevel=not; + topLevel = not; } return simplifyUnnecessaryComposites(topLevel); } - protected void setSubmodeFromIndex(String indexName, Set<String> searchDefinitions) { - // No submodes in this language - } - } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 786a0d0e04f..623c38fa9f0 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -108,7 +108,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { WEB(4,"web"), PROGRAMMATIC(5, "prog"), YQL(6, "yql"), - SELECT(7, "select"); + SELECT(7, "select"), + WEAKAND(8, "weakAnd"); private final int intValue; private final String stringValue; @@ -123,7 +124,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { for (Type type : Type.values()) if (type.stringValue.equals(typeString)) return type; - return ALL; + throw new IllegalArgumentException("No query type '" + typeString + "'"); } public int asInt() { return intValue; } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index a64af7658cc..f9b8f1785db 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -29,7 +29,7 @@ public final class ParserFactory { public static Parser newInstance(Query.Type type, ParserEnvironment environment) { switch (type) { case ALL: - return new AllParser(environment); + return new AllParser(environment, false); case ANY: return new AnyParser(environment); case PHRASE: @@ -44,6 +44,8 @@ public final class ParserFactory { return new YqlParser(environment); case SELECT: return new SelectParser(environment); + case WEAKAND: + return new AllParser(environment, true); default: throw new UnsupportedOperationException(type.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 649d678db55..93cac27059e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -18,6 +18,7 @@ import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.chain.After; import com.yahoo.yolean.chain.Before; import com.yahoo.yolean.chain.Provides; @@ -93,7 +94,9 @@ public class MinimalQueryInserter extends Searcher { Parsable parsable = Parsable.fromQueryModel(query.getModel()).setQuery(query.properties().getString(YQL)); newTree = parser.parse(parsable); } catch (RuntimeException e) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not instantiate query from YQL", e)); + return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not create query from YQL: " + + Exceptions.toMessageString(e), + e)); } if (parser.getOffset() != null) { int maxHits = query.properties().getInteger(MAX_HITS); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index 94332a52fed..69faed1da90 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -54,7 +54,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { } private static Item parseQuery(String query) { - AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE)); + AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE), false); return parser.parse(new Parsable().setQuery(query).setLanguage(Language.CHINESE_SIMPLIFIED)).getRoot(); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java index ac0d676caf7..efaaaa5fca7 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java @@ -23,7 +23,7 @@ public class TermListTestCase { @Test public void testTermListInWeightedSet() { URIBuilder builder = searchUri(); - builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}"); + builder.setParameter("myTerms", "{'1':1, '2':1, 3:1}"); builder.setParameter("yql", "select * from sources * where weightedSet(user_id, @myTerms)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where weightedSet(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", @@ -33,7 +33,7 @@ public class TermListTestCase { @Test public void testTermListInWand() { URIBuilder builder = searchUri(); - builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}"); + builder.setParameter("myTerms", "{'1':1, 2:1, '3':1}"); builder.setParameter("yql", "select * from sources * where wand(user_id, @myTerms)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where wand(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index cc1a275af5a..c41de3a73f1 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -72,7 +72,7 @@ public class UserInputTestCase { @Test public void testRawUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where [{grammar: \"raw\"}]userInput(\"nal le\")"); + builder.setParameter("yql", "select * from sources * where {grammar: \"raw\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); } @@ -81,7 +81,7 @@ public class UserInputTestCase { public void testSegmentedUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{grammar: \"segment\"}]userInput(\"nal le\")"); + "select * from sources * where {grammar: \"segment\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", offset: 0, length: 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); } @@ -90,12 +90,50 @@ public class UserInputTestCase { public void testSegmentedNoiseUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{grammar: \"segment\"}]userInput(\"^^^^^^^^\")"); + "select * from sources * where {grammar: \"segment\"}userInput(\"^^^^^^^^\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); } @Test + public void testAnyParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"any\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (default contains \"foo\" OR default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testAllParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"all\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (default contains \"foo\" AND default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testWeakAndParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"weakAnd\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where weakAnd(default contains \"foo\", default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testIllegalGrammar() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"nonesuch\"}userInput('foo bar')"); + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNotNull(r.hits().getError()); + assertEquals("Could not create query from YQL: No query type 'nonesuch'", + r.hits().getError().getDetailedMessage()); + } + + @Test public void testCustomDefaultIndexUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 636619bf1cc..15713dc1f97 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -399,7 +399,7 @@ public class YqlParserTestCase { assertFalse(root instanceof WordItem); assertTrue(root instanceof PhraseSegmentItem); - root = parse("select foo from bar where baz contains ([{grammar:\"raw\"}]\"yoni jo dima\")").getRoot(); + root = parse("select foo from bar where baz contains ({grammar:\"raw\"}\"yoni jo dima\")").getRoot(); assertEquals("baz:yoni jo dima", root.toString()); assertTrue(root instanceof WordItem); assertFalse(root instanceof ExactStringItem); @@ -410,7 +410,7 @@ public class YqlParserTestCase { AndItem andItem = (AndItem) root; assertEquals(3, andItem.getItemCount()); - root = parse("select foo from bar where [{grammar:\"raw\"}]userInput(\"yoni jo dima\")").getRoot(); + root = parse("select foo from bar where {grammar:\"raw\"}userInput(\"yoni jo dima\")").getRoot(); assertTrue(root instanceof WordItem); assertTrue(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem)root).getWord()); @@ -419,9 +419,9 @@ public class YqlParserTestCase { @Test public void testAccentDropping() { assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {accentDrop: false} ]\"colors\")").isNormalizable()); + "( {accentDrop: false} \"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {accentDrop: true} ]\"colors\")").isNormalizable()); + "( {accentDrop: true} \"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + "\"colors\"").isNormalizable()); } @@ -429,9 +429,9 @@ public class YqlParserTestCase { @Test public void testCaseNormalization() { assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {normalizeCase: false} ]\"colors\")").isLowercased()); + "( {normalizeCase: false} \"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {normalizeCase: true} ]\"colors\")").isLowercased()); + "( {normalizeCase: true} \"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + "\"colors\"").isLowercased()); } @@ -440,10 +440,10 @@ public class YqlParserTestCase { public void testSegmentingRule() { assertEquals(SegmentingRule.PHRASE, getRootWord("select foo from bar where baz contains " + - "([ {andSegmenting: false} ]\"colors\")").getSegmentingRule()); + "( {andSegmenting: false} \"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.BOOLEAN_AND, getRootWord("select foo from bar where baz contains " + - "([ {andSegmenting: true} ]\"colors\")").getSegmentingRule()); + "( {andSegmenting: true} \"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.LANGUAGE_DEFAULT, getRootWord("select foo from bar where baz contains " + "\"colors\"").getSegmentingRule()); @@ -453,10 +453,10 @@ public class YqlParserTestCase { public void testNfkc() { assertEquals("a\u030a", getRootWord("select foo from bar where baz contains " + - "([ {nfkc: false} ]\"a\\u030a\")").getWord()); + "( {nfkc: false} \"a\\u030a\")").getWord()); assertEquals("\u00e5", getRootWord("select foo from bar where baz contains " + - "([ {nfkc: true} ]\"a\\u030a\")").getWord()); + "( {nfkc: true} \"a\\u030a\")").getWord()); assertEquals("No NKFC by default", "a\u030a", getRootWord("select foo from bar where baz contains " + @@ -465,19 +465,19 @@ public class YqlParserTestCase { @Test public void testImplicitTransforms() { - assertFalse(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + - "false} ]\"cox\")").isFromQuery()); - assertTrue(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + - "true} ]\"cox\")").isFromQuery()); + assertFalse(getRootWord("select foo from bar where baz contains ({implicitTransforms: " + + "false} \"cox\")").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains ({implicitTransforms: " + + "true} \"cox\")").isFromQuery()); assertTrue(getRootWord("select foo from bar where baz contains \"cox\"").isFromQuery()); } @Test public void testConnectivity() { QueryTree parsed = parse("select foo from bar where " + - "title contains ([{id: 1, connectivity: {\"id\": 3, weight: 7.0}}]\"madonna\") " + - "and title contains ([{id: 2}]\"saint\") " + - "and title contains ([{id: 3}]\"angel\")"); + "title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " + + "and title contains ({id: 2}\"saint\") " + + "and title contains ({id: 3}\"angel\")"); assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); AndItem root = (AndItem)parsed.getRoot(); @@ -489,7 +489,7 @@ public class YqlParserTestCase { assertNull(second.getConnectedItem()); assertParseFail("select foo from bar where " + - "title contains ([{id: 1, connectivity: {id: 4, weight: 7.0}}]\"madonna\") " + + "title contains ({id: 1, connectivity: {id: 4, weight: 7.0}}\"madonna\") " + "and title contains ({id: 2}\"saint\") " + "and title contains ({id: 3}\"angel\")", new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " + |