diff options
| author | Arnstein Ressem <aressem@verizonmedia.com> | 2020-03-03 07:49:36 +0100 |
|---|---|---|
| committer | Arnstein Ressem <aressem@verizonmedia.com> | 2020-03-03 07:49:36 +0100 |
| commit | 560df96b5e4838261bc0729f449926f8e90a30f7 (patch) | |
| tree | 57a06a0c8a0f5c40a911dea183dd6b2d979e8369 | |
| parent | dc70c4bf6c9b064776f3c13e6769093611660621 (diff) | |
Revert parser changes that broke system tests.
16 files changed, 427 insertions, 452 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 5e7fddd7fe7..365ee299ca4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; + import com.yahoo.language.process.StemMode; import java.util.ArrayList; @@ -9,6 +10,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; + /** * Information about configured settings of a field or field collection (an actual index or not) in a search definition. * There are two types of settings: @@ -72,8 +74,8 @@ public class Index { private boolean isNGram = false; private int gramSize = 2; - /** Whether implicit phrases should lead to a phrase item or an and item. */ - private Boolean phraseSegmenting = false; + /** Whether implicit phrases should lead to a phrase item or an and item */ + private boolean phraseSegmenting = true; /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 00935392683..062a514056b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -109,6 +109,7 @@ public final class IndexModel { return searchDefinitions; } + @SuppressWarnings("deprecation") private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) { SearchDefinition union = new SearchDefinition(IndexFacts.unionName); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 49bdba2c90f..d9b969757c2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -30,7 +30,6 @@ public class AllParser extends SimpleParser { super(environment); } - @Override protected Item parseItems() { int position = tokens.getPosition(); try { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index b714a1d8b34..dd836e9c8e1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,12 +35,21 @@ public class AnyParser extends SimpleParser { return anyItems(true); } + Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) { + return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet())); + } + Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { + Item filterRoot; + setState(queryLanguage, indexFacts); tokenize(filter, null, indexFacts, queryLanguage); - Item filterRoot = anyItems(true); - if (filterRoot == null) return null; + filterRoot = anyItems(true); + + if (filterRoot == null) { + return null; + } markAllTermsAsFilters(filterRoot); return filterRoot; @@ -52,10 +61,18 @@ public class AnyParser extends SimpleParser { try { tokens.skipMultiple(PLUS); - if ( ! tokens.skipMultiple(MINUS)) return null; - if (tokens.currentIsNoIgnore(SPACE)) return null; - item = indexableItem(); + if (!tokens.skipMultiple(MINUS)) { + return null; + } + + if (tokens.currentIsNoIgnore(SPACE)) { + return null; + } + + if (item == null) { + item = indexableItem(); + } if (item == null) { item = compositeItem(); @@ -71,13 +88,13 @@ public class AnyParser extends SimpleParser { } } } - if (item != null) + if (item!=null) item.setProtected(true); - return item; } finally { - if (item == null) + if (item == null) { tokens.setPosition(position); + } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 3d244312b2f..9ddfea6dffb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -50,28 +50,32 @@ abstract class SimpleParser extends StructuredParser { private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item = null; + Item item; do { - item = positiveItem(); - if (item != null) { - if (not == null) { - not = new NotItem(); - not.addPositiveItem(item); - topLevelItem = combineItems(topLevelItem, not); - } else { - not.addPositiveItem(item); + item = null; + + if (item == null) { + item = positiveItem(); + if (item != null) { + if (not == null) { + not = new NotItem(); + not.addPositiveItem(item); + topLevelItem = combineItems(topLevelItem, not); + } else { + not.addPositiveItem(item); + } } } if (item == null) { item = negativeItem(); if (item != null) { - if (not == null) { + if (not == null && item != null) { not = new NotItem(); not.addNegativeItem(item); topLevelItem = combineItems(topLevelItem, not); - } else { + } else if (item != null) { not.addNegativeItem(item); } } @@ -93,8 +97,9 @@ abstract class SimpleParser extends StructuredParser { if (item != null) { if (topLevelItem == null) { topLevelItem = item; - } else if (needNewORTopLevel(topLevelItem, item)) { + } else if (needNewTopLevel(topLevelItem, item)) { CompositeItem newTop = new OrItem(); + newTop.addItem(topLevelItem); newTop.addItem(item); topLevelItem = newTop; @@ -126,7 +131,6 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives - System.out.println("Extracting positive item from " + topLevelItem); not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); return not; } else { // Only negatives - ignore them @@ -140,13 +144,21 @@ abstract class SimpleParser extends StructuredParser { } } - /** Says whether we need a new top level OR item given the new item */ - private boolean needNewORTopLevel(Item topLevelItem, Item item) { - if (item == null) return false; - if (topLevelItem instanceof TermItem) return true; - if (topLevelItem instanceof PhraseItem) return true; - if (topLevelItem instanceof BlockItem) return true; - if ( topLevelItem instanceof AndItem) return true; + + /** Says whether we need a new top level item given the new item */ + private boolean needNewTopLevel(Item topLevelItem, Item item) { + if (item == null) { + return false; + } + if (topLevelItem instanceof TermItem) { + return true; + } + if (topLevelItem instanceof PhraseItem) { + return true; + } + if (topLevelItem instanceof BlockItem) { + return true; + } return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 9ba6c1a8101..5e292a06b0f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -442,9 +442,9 @@ abstract class StructuredParser extends AbstractParser { Item item = null; try { - if ( ! tokens.currentIs(WORD) - && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) - && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { + if (!tokens.currentIs(WORD) + && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) + && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { return null; } Token word = tokens.next(); @@ -557,7 +557,6 @@ abstract class StructuredParser extends AbstractParser { if (composite != null) { composite.addItem(word); - connectLastTermsIn(composite); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); @@ -585,7 +584,6 @@ abstract class StructuredParser extends AbstractParser { } composite.addItem(firstWord); composite.addItem(word); - connectLastTermsIn(composite); } else if (word instanceof PhraseItem) { composite = (PhraseItem)word; } else { @@ -656,15 +654,6 @@ abstract class StructuredParser extends AbstractParser { } } - private void connectLastTermsIn(CompositeItem composite) { - int items = composite.items().size(); - if (items < 2) return; - Item nextToLast = composite.items().get(items - 2); - Item last = composite.items().get(items - 1); - if ( ! (nextToLast instanceof TermItem)) return; - ((TermItem)nextToLast).setConnectivity(last, 1); - } - private boolean addStartMarking() { if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) { tokens.skip(); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index e346a766738..d39a488626b 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -97,10 +97,11 @@ public class DocumentSourceSearcher extends Searcher { public Result search(Query query, Execution execution) { queryCount++; Result r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) + if (r == null) { r = defaultFilledResult.clone(); - else + } else { r = r.clone(); + } r.setQuery(query); r.hits().trim(query.getOffset(), query.getHits()); @@ -181,8 +182,11 @@ public class DocumentSourceSearcher extends Searcher { * reset. For testing - not reliable if multiple threads makes * queries simultaneously */ - public int getQueryCount() { return queryCount; } - - public void resetQueryCount() { queryCount = 0; } + public int getQueryCount() { + return queryCount; + } + public void resetQueryCount() { + queryCount=0; + } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java index df35d8dbdea..5cae40bd10d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java @@ -11,7 +11,6 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.junit.Assert.assertEquals; @@ -35,7 +34,7 @@ public class ExactMatchAndDefaultIndexTestCase { q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null))); assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString()); q = new Query("?query=" + enc("a/b foo.com")); - assertEquals("AND a b foo com", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -45,7 +44,11 @@ public class ExactMatchAndDefaultIndexTestCase { } private String enc(String s) { - return URLEncoder.encode(s, StandardCharsets.UTF_8); + try { + return URLEncoder.encode(s, "utf-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index c1db7d73561..0fdad1a1f9c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -7,7 +7,6 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; @@ -49,9 +48,7 @@ public class ParseTestCase { @Test public void testTermWithIndexPrefix() { - tester.assertParsed("url:foobar", - "url:foobar", - Query.Type.ANY); + tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY); } @Test @@ -62,98 +59,104 @@ public class ParseTestCase { @Test public void testMultipleTermsWithUTF8EncodingOred() { tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen", - "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", - Query.Type.ANY); + "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY); } @Test public void testMultipleTermsWithMultiplePrefixes() { tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); + "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); } @Test public void testSimpleQueryDefaultOr() { - tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", Query.Type.ANY); + tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", + Query.Type.ANY); } @Test public void testOrAndNot() { tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); + "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); } @Test public void testSimpleOrNestedAnd() { tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy", - "foobar +(foo bar baz) xyzzy", Query.Type.ANY); + "foobar +(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testSimpleOrNestedNot() { tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)", - "foobar -(foo bar baz) xyzzy", Query.Type.ANY); + "foobar -(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testOrNotNestedAnd() { - tester.assertParsed("RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrAndNotNestedNot() { - tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAnd() { - tester.assertParsed("RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", - "foobar +(fo ba foba) foo bar +(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed( + "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", + "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY); } @Test public void testOrMultipleNestedNot() { - tester.assertParsed("+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", - "foobar -(fo ba foba) foo bar -(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed( + "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", + "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAnd() { - tester.assertParsed("RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedNot() { - tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAndNot() { - tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", - "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", + "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAndNot() { - tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", - "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", + "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", + Query.Type.ANY); } @Test public void testExplicitPhrase() { - Item root = tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); + Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); assertTrue(root instanceof PhraseItem); assertTrue(((PhraseItem)root).isExplicit()); } @@ -161,20 +164,21 @@ public class ParseTestCase { @Test public void testPhraseWithIndex() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ANY); + "normal.title:\"foo bar foobar\"", Query.Type.ANY); } @Test public void testPhrasesAndTerms() { tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"", - "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); + "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); } @Test public void testPhrasesAndTermsWithOperators() { - tester.assertParsed("RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", - "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", - Query.Type.ANY); + tester.assertParsed( + "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", + "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", + Query.Type.ANY); } @Test @@ -184,40 +188,38 @@ public class ParseTestCase { @Test public void testTermWithCatalogAndIndexPrefixDefaultAnd() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ALL); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", + Query.Type.ALL); } @Test public void testMultipleTermsWithMultiplePrefixesDefaultAnd() { tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz", - "url:foobar +bar -normal.title:foo -baz", - Query.Type.ALL); + "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL); } @Test public void testSimpleQueryDefaultAnd() { - tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", Query.Type.ALL); + tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", + Query.Type.ALL); } @Test public void testNotDefaultAnd() { - tester.assertParsed("+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", - "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", - Query.Type.ALL); + tester.assertParsed( + "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", + "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL); } @Test public void testSimpleTermQueryDefaultPhrase() { - tester.assertParsed("foobar", - "foobar", - Query.Type.PHRASE); + tester.assertParsed("foobar", "foobar", Query.Type.PHRASE); } @Test public void testSimpleQueryDefaultPhrase() { - Item root = tester.assertParsed("\"foobar foo bar baz\"", - "foobar foo bar baz", - Query.Type.PHRASE); + Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz", + Query.Type.PHRASE); assertTrue(root instanceof PhraseItem); assertFalse(((PhraseItem)root).isExplicit()); } @@ -225,25 +227,23 @@ public class ParseTestCase { @Test public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() { tester.assertParsed("\"url foobar bar normal title foo baz\"", - "url:foobar +bar -normal.title:foo -baz", - Query.Type.PHRASE); + "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE); } @Test public void testOdd1() { - tester.assertParsed("AND window print error", "+window.print() +error", - Query.Type.ALL); + tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL); } @Test public void testOdd2() { - tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"", - Query.Type.ALL); + tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL); } @Test public void testOdd2Uppercase() { - tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", Query.Type.ALL); + tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", + Query.Type.ALL); } @Test @@ -280,19 +280,19 @@ public class ParseTestCase { @Test public void testNestedCompositesDefaultOr() { tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); } @Test public void testNestedCompositesDefaultAnd() { tester.assertParsed("AND foo (OR foobar bar baz) xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); } @Test public void testNestedCompositesPhraseDefault() { tester.assertParsed("\"foo foobar bar baz xyzzy\"", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); } @Test @@ -349,7 +349,8 @@ public class ParseTestCase { @Test public void testNumericWithIndex() { - tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ANY); + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", + Query.Type.ANY); } @Test @@ -360,14 +361,14 @@ public class ParseTestCase { @Test public void testMultipleIntegerWithIndex() { tester.assertParsed("OR document.size:[34;454] date:>1234567890", - "document.size:[34;454] date:>1234567890", Query.Type.ANY); + "document.size:[34;454] date:>1234567890", Query.Type.ANY); } @Test public void testMixedNumericAndOtherTerms() { tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890", - "foo +document.size:<1024 +xyzzy date:>123456890", - Query.Type.ANY); + "foo +document.size:<1024 +xyzzy date:>123456890", + Query.Type.ANY); } @Test @@ -377,18 +378,20 @@ public class ParseTestCase { @Test public void testItemPhraseEmptyPhrase() { - tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ANY); + tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", + Query.Type.ANY); } @Test public void testSimpleQuery() { - tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", Query.Type.ANY); + tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", + Query.Type.ANY); } @Test public void testExcessivePluses() { tester.assertParsed("+(AND other is nothing) -test", - "++other +++++is ++++++nothing -test", Query.Type.ANY); + "++other +++++is ++++++nothing -test", Query.Type.ANY); } @Test @@ -398,38 +401,39 @@ public class ParseTestCase { @Test public void testPlusesAndMinuses() { - tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ANY); + Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY); + assertTrue(root instanceof PhraseItem); + assertFalse(((PhraseItem)root).isExplicit()); } @Test public void testNumbers() { - tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ANY); + tester.assertParsed("\"123 2132odfd 934032 32423\"", + "123+2132odfd.934032,,32423", Query.Type.ANY); } @Test public void testOtherSignsInQuote() { - tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ANY); + tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", + Query.Type.ANY); } @Test public void testGribberish() { tester.assertParsed("1349832840234l3040roer\u00e6lf12", - ",1349832840234l3040roer\u00e6lf12", - Query.Type.ANY); + ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY); } @Test public void testUrl() { - tester.assertParsed("AND www:www www:hotelaiguablava www:com", - "+www:www.hotelaiguablava:com", - Query.Type.ANY); + tester.assertParsed("www:\"www hotelaiguablava com\"", + "+www:www.hotelaiguablava:com", Query.Type.ANY); } @Test public void testUrlGribberish() { - tester.assertParsed("OR (AND 3 16) fast.type:lycosoffensive", - "[ 3:16 fast.type:lycosoffensive", - Query.Type.ANY); + tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive", + "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY); } @Test @@ -471,7 +475,8 @@ public class ParseTestCase { @Test public void testPrefixWithDotAdvanced() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ADVANCED); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", + Query.Type.ADVANCED); } @Test @@ -481,21 +486,20 @@ public class ParseTestCase { @Test public void testSimplePhraseAdvanced() { - tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ADVANCED); + tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", + Query.Type.ADVANCED); } @Test public void testSimplePhraseWithIndexAdvanced() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", - Query.Type.ADVANCED); + "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED); } @Test public void testMultiplePhrasesAdvanced() { tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"", - "\"foo bar foobar\" and \"baz gaz faz\"", - Query.Type.ADVANCED); + "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED); } @Test @@ -657,23 +661,23 @@ public class ParseTestCase { @Test public void testImplicitPhrase1Advanced() { - tester.assertParsed("AND test if", "--test+-if", Query.Type.ADVANCED); + tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED); } @Test public void testImplicitPhrase2Advanced() { - tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ADVANCED); + tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED); } @Test public void testImplicitPhrase3Advanced() { - tester.assertParsed("AND 123 2132odfd 934032 32423", + tester.assertParsed("\"123 2132odfd 934032 32423\"", "123+2132odfd.934032,,32423", Query.Type.ADVANCED); } @Test public void testImplicitPhrase4Advanced() { - tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ADVANCED); + tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED); } @Test @@ -726,7 +730,7 @@ public class ParseTestCase { @Test public void testSingleHyphen() { - tester.assertParsed("AND a b", "a-b", Query.Type.ALL); + tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL); } @Test @@ -879,27 +883,27 @@ public class ParseTestCase { @Test public void testSimpleDotPhraseAny() { - tester.assertParsed("OR a (AND b c) d", "a b.c d", Query.Type.ANY); + tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY); } @Test public void testSimpleHyphenPhraseAny() { - tester.assertParsed("OR a (AND b c) d", "a b-c d", Query.Type.ANY); + tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY); } @Test public void testAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR (AND a b) c d", "a.b c d", Query.Type.ANY); + tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY); } @Test public void testYetAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR a b (AND c d)", "a b c.d", Query.Type.ANY); + tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY); } @Test public void testVariousSeparatorsPhraseAny() { - tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ANY); + tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY); } @Test @@ -914,44 +918,45 @@ public class ParseTestCase { @Test public void testIndexedDottedPhraseAny() { - tester.assertParsed("OR a (AND url:b url:c) d", "a url:b.c d", Query.Type.ANY); + tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY); } @Test public void testIndexedPlusedPhraseAny() { - tester.assertParsed("OR a (AND normal.title:b normal.title:c) d", "a normal.title:b+c d", Query.Type.ANY); + tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d", + Query.Type.ANY); } @Test public void testNestedNotAny() { tester.assertParsed( - "RANK (+(OR normal.title:foobar (AND url:www url:pvv url:org)) -foo) a", + "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a", "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY); } @Test public void testDottedPhraseAdvanced() { - tester.assertParsed("OR a (AND b c)", "a or b.c", Query.Type.ADVANCED); + tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED); } @Test public void testHyphenPhraseAdvanced() { - tester.assertParsed("OR (AND a (AND b c)) d", "a and b-c or d", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED); } @Test public void testAnotherDottedPhraseAdvanced() { - tester.assertParsed("OR (AND a b) c", "a.b or c", Query.Type.ADVANCED); + tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED); } @Test public void testNottedDottedPhraseAdvanced() { - tester.assertParsed("+a -(AND c d)", "a andnot c.d", Query.Type.ADVANCED); + tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED); } @Test public void testVariousSeparatorsPhraseAdvanced() { - tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ADVANCED); + tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED); } @Test @@ -971,14 +976,14 @@ public class ParseTestCase { @Test public void testNestedPlussedPhraseAdvanced() { - tester.assertParsed("AND (OR a (AND normal.title:b normal.title:c)) d", + tester.assertParsed("AND (OR a normal.title:\"b c\") d", "a or normal.title:b+c and d", Query.Type.ADVANCED); } @Test public void testNottedNestedDottedPhraseAdvanced() { tester.assertParsed( - "+(AND a (OR normal.title:foobar (AND url:www url:pvv url:org))) -foo", + "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo", "a and (normal.title:foobar or url:www.pvv.org) andnot foo", Query.Type.ADVANCED); } @@ -990,7 +995,7 @@ public class ParseTestCase { @Test public void testPlusedTwiceThenQuotedPhraseAny() { - tester.assertParsed("AND a b c d", "a+b+\"c d\"", Query.Type.ANY); + tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY); } @Test @@ -1000,7 +1005,7 @@ public class ParseTestCase { @Test public void testPhrasesInBraces() { - tester.assertParsed("AND url.domain:microsoft url.domain:com", + tester.assertParsed("url.domain:\"microsoft com\"", "+(url.domain:microsoft.com)", Query.Type.ALL); } @@ -1048,17 +1053,17 @@ public class ParseTestCase { @Test public void testPhraseNotPrefix() { - tester.assertParsed("OR foo (AND prefix bar)", "foo prefix*bar", Query.Type.ANY); + tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY); } @Test public void testPhraseNotSubstring() { - tester.assertParsed("OR foo (AND substring bar)", "foo *substring*bar", Query.Type.ANY); + tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY); } @Test public void testPhraseNotSuffix() { - tester.assertParsed("OR (AND foo suffix) bar", "foo*suffix bar", Query.Type.ANY); + tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY); } @Test @@ -1081,17 +1086,20 @@ public class ParseTestCase { @Test public void testIndexedPhraseNotPrefix() { - tester.assertParsed("AND foo.bar:prefix foo.bar:xyzzy", "foo.bar:prefix*xyzzy", Query.Type.ANY); + tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy", + Query.Type.ANY); } @Test public void testIndexedPhraseNotSubstring() { - tester.assertParsed("AND foo.bar:substring foo.bar:xyzzy", "foo.bar:*substring*xyzzy", Query.Type.ANY); + tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy", + Query.Type.ANY); } @Test public void testIndexedPhraseNotSuffix() { - tester.assertParsed("AND foo.bar:xyzzy foo.bar:suffix", "foo.bar:xyzzy*suffix", Query.Type.ANY); + tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix", + Query.Type.ANY); } @Test @@ -1112,20 +1120,20 @@ public class ParseTestCase { assertTrue(root instanceof SuffixItem); } - /** Non existing index → and **/ + /** Non existing index → phrase **/ @Test public void testNonIndexPhraseNotPrefix() { - tester.assertParsed("AND void prefix", "void:prefix*", Query.Type.ANY); + tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSubstring() { - tester.assertParsed("AND void substring", "void:*substring*", Query.Type.ANY); + tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSuffix() { - tester.assertParsed("AND void suffix", "void:*suffix", Query.Type.ANY); + tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY); } /** Explicit phrase → remove '*' **/ @@ -1190,7 +1198,7 @@ public class ParseTestCase { /** Extra spaces with index **/ @Test public void testIndexPrefixExtraSpace() { - tester.assertParsed("AND foo prefix", "foo:prefix *", Query.Type.ANY); + tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY); } @Test @@ -1411,7 +1419,7 @@ public class ParseTestCase { @Test public void testMultipleDifferentPhraseSeparators() { - tester.assertParsed("AND foo bar", "foo.-.bar", Query.Type.ANY); + tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY); } @Test @@ -1422,17 +1430,19 @@ public class ParseTestCase { @Test public void testReallyNoisyQuery1() { - tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", Query.Type.ALL); + tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", + Query.Type.ALL); } @Test public void testReallyNoisyQuery2() { - tester.assertParsed("AND \u03bc\u03bc hei", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); + tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); } @Test public void testReallyNoisyQuery3() { - tester.assertParsed("AND hei hallo du der", "hei-hallo;du;der", Query.Type.ALL); + tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der", + Query.Type.ALL); } @Test @@ -1468,7 +1478,7 @@ public class ParseTestCase { @Test public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() { - tester.assertParsed("AND yz a", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); + tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); } @Test @@ -1488,7 +1498,7 @@ public class ParseTestCase { @Test public void testImplicitPhrasingWithIndex() { - tester.assertParsed("AND a:b a:c", "a:/b/c", Query.Type.ANY); + tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY); } @Test @@ -1498,7 +1508,7 @@ public class ParseTestCase { @Test public void testSingleNoisyPhraseWithIndex() { - tester.assertParsed("AND mail:yahoo mail:com", "mail:@yahoo.com", Query.Type.ANY); + tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY); } @Test @@ -1589,7 +1599,7 @@ public class ParseTestCase { "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions", Query.Type.ALL); tester.assertParsed( - "AND ull:http ull:www ull:neue ull:oz ull:de ull:information ull:pub ull:Boulevard ull:index ull:html ull:file ull:a ull:3 ull:s ull:4 ull:file s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", + "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}", Query.Type.ALL); } @@ -1630,7 +1640,7 @@ public class ParseTestCase { @Test public void testTooLongQueryTerms() { - tester.assertParsed("AND 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof 2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", + tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"", "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", Query.Type.ALL); } @@ -1638,22 +1648,22 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); - customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); + customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY); } @Test public void testNonIndexWithColons1() { - tester.assertParsed("OR this is (AND notan iindex)", "this is notan:iindex", Query.Type.ANY); + tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY); } @Test public void testNonIndexWithColons2() { - tester.assertParsed("OR this is (AND notan iindex either)", "this is notan:iindex:either", Query.Type.ANY); + tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY); } @Test public void testIndexThenUnderscoreTermBecomesIndex() { - tester.assertParsed("AND name:batch name:article", "name:batch_article", Query.Type.ANY); + tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY); } @Test @@ -1661,17 +1671,17 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof AndSegmentItem); - AndSegmentItem segment = (AndSegmentItem) item; + assertTrue(item instanceof PhraseSegmentItem); + PhraseSegmentItem phrase = (PhraseSegmentItem) item; - assertEquals(3, segment.getItemCount()); - assertEquals("name:first", segment.getItem(0).toString()); - assertEquals("name:second", segment.getItem(1).toString()); - assertEquals("name:third", segment.getItem(2).toString()); + assertEquals(3, phrase.getItemCount()); + assertEquals("name:first", phrase.getItem(0).toString()); + assertEquals("name:second", phrase.getItem(1).toString()); + assertEquals("name:third", phrase.getItem(2).toString()); - assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem) segment.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName()); } @Test @@ -1679,22 +1689,22 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof AndSegmentItem); - AndSegmentItem segment = (AndSegmentItem) item; + assertTrue(item instanceof PhraseSegmentItem); + PhraseSegmentItem phrase = (PhraseSegmentItem) item; - assertEquals(3, segment.getItemCount()); - assertEquals("name:first", segment.getItem(0).toString()); - assertEquals("name:second", segment.getItem(1).toString()); - assertEquals("name:third", segment.getItem(2).toString()); + assertEquals(3, phrase.getItemCount()); + assertEquals("name:first", phrase.getItem(0).toString()); + assertEquals("name:second", phrase.getItem(1).toString()); + assertEquals("name:third", phrase.getItem(2).toString()); - assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem)segment.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName()); } @Test public void testAndItemAndImplicitPhrase() { - tester.assertParsed("AND \u00d8 \u00d8 \u00d8 \u00d9", + tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"", "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1726,7 +1736,7 @@ public class ParseTestCase { @Test public void testFakeCJKSegmentingOfMultiplePhrases() { Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertEquals("AND (SAND name:first name:second) name:s", item.toString()); + assertEquals("name:\"'first second' s\"", item.toString()); } @Test @@ -1791,7 +1801,7 @@ public class ParseTestCase { @Test public void testCommaOnlyLeadsToImplicitPhrasing() { - tester.assertParsed("AND A B C", "A,B,C", Query.Type.ALL); + tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL); } @Test @@ -1863,8 +1873,8 @@ public class ParseTestCase { @Test public void testJPMobileExceptionQuery() { - tester.assertParsed("OR concat and (AND make string) 1 47 or", - "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); + tester.assertParsed("OR concat and \"make string\" 1 47 or", + "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); } @Test @@ -1872,7 +1882,7 @@ public class ParseTestCase { tester.assertParsed("b", "a: b", Query.Type.ALL); tester.assertParsed("AND a b", "a : b", Query.Type.ALL); tester.assertParsed("AND a b", "a :b", Query.Type.ALL); - tester.assertParsed("AND a b", "a.:b", Query.Type.ALL); + tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL); tester.assertParsed("a:b", "a:b", Query.Type.ALL); } @@ -1907,7 +1917,8 @@ public class ParseTestCase { tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ALL); - // The last one here is a little weird, but it's not a problem, so let it pass for now... + // The last one here is a little weird, but it's not a problem, + // so I let it pass for now... tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ANY); @@ -1915,7 +1926,7 @@ public class ParseTestCase { @Test public void testMixedCaseIndexNames() { - tester.assertParsed("AND mixedCase:a mixedCase:b notAnIndex c mixedCase:d", + tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d", "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d", Query.Type.ALL); } @@ -1923,7 +1934,7 @@ public class ParseTestCase { /** CJK special tokens should be recognized also on non-boundaries */ @Test public void testChineseSpecialTokens() { - tester.assertParsed("AND cat tcp/ip zu foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz", + tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"", "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1934,7 +1945,7 @@ public class ParseTestCase { @Test public void testChineseSpecialTokensWithMultiSegmentReplace() { // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test - tester.assertParsed("AND tcp/ip firstsecond dotnet firstsecond (SAND first second)","tcp/ipspecial-token-fs.net special-token-fs firstsecond", + tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE); } @@ -2003,7 +2014,7 @@ public class ParseTestCase { @Test public void testVersionNumbers() { - tester.assertParsed("AND 1 0 9", "1.0.9", Query.Type.ALL); + tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL); } @Test @@ -2310,7 +2321,7 @@ public class ParseTestCase { @Test public void testOdd1Web() { - tester.assertParsed("AND window print error", "+window.print() +error",Query.Type.WEB); + tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB); } @Test @@ -2340,13 +2351,13 @@ public class ParseTestCase { @Test public void testDefaultWebIndices() { - tester.assertParsed("AND notanindex b","notanindex:b", Query.Type.WEB); - tester.assertParsed("site:\"b $\"","site:b", Query.Type.WEB); - tester.assertParsed("hostname:b","hostname:b", Query.Type.WEB); - tester.assertParsed("link:b","link:b", Query.Type.WEB); - tester.assertParsed("url:b","url:b", Query.Type.WEB); - tester.assertParsed("inurl:b","inurl:b", Query.Type.WEB); - tester.assertParsed("intitle:b","intitle:b", Query.Type.WEB); + tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB); + tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB); + tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB); + tester.assertParsed("link:b","link:b",Query.Type.WEB); + tester.assertParsed("url:b","url:b",Query.Type.WEB); + tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB); + tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB); } @Test @@ -2516,7 +2527,7 @@ public class ParseTestCase { @Test public void testSiteAndSegmentPhrasesFollowedByText() { - tester.assertParsed("AND host.all:\"www abc com x y-z $\" (SAND a b)", + tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'", "host.all:www.abc.com/x'y-z a'b", "", Query.Type.ALL, Language.ENGLISH); } @@ -2533,7 +2544,7 @@ public class ParseTestCase { @Test public void testNonAsciiNumber() { - tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); + tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java index 0ca4b8aa615..91cf5015cba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -45,7 +45,7 @@ public class CJKSearcherTestCase { @Test public void testCjkQueryWithOverlappingTokens() { // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd" - assertTransformed("bcd", "SAND bc cd", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, TestLinguistics.INSTANCE); // While "efg" will be segmented into one of the standard options, "e" "fg" diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java index 023cd3c2849..12e756a07ee 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java @@ -71,7 +71,7 @@ public class LiteralBoostSearcherTestCase { @Test public void testQueryWithoutBoost() { - assertEquals("RANK (AND nonexistant a nonexistant b) default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", + assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1")); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index 12619bf0a5e..4875121a501 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -40,8 +40,34 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public class FieldCollapsingSearcherTestCase { + private FastHit createHit(String uri,int relevancy,int mid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(mid)); + return hit; + } + + private void assertHit(String uri,int relevancy,int mid,Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + } + + private static class ZeroHitsControl extends com.yahoo.search.Searcher { + public int queryCount = 0; + public com.yahoo.search.Result search(com.yahoo.search.Query query, + com.yahoo.search.searchchain.Execution execution) { + ++queryCount; + if (query.getHits() == 0) { + return new Result(query); + } else { + return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); + } + } + } + @Test public void testFieldCollapsingWithoutHits() { // Set up @@ -90,14 +116,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -107,10 +133,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); } @Test @@ -126,14 +152,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -143,10 +169,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3, r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); } @Test @@ -159,14 +185,14 @@ public class FieldCollapsingSearcherTestCase { Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -194,16 +220,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 0)); - r.hits().add(createHit("http://acme.org/d.html", 8, 0)); - r.hits().add(createHit("http://acme.org/e.html", 8, 0)); - r.hits().add(createHit("http://acme.org/f.html", 7, 0)); - r.hits().add(createHit("http://acme.org/g.html", 7, 0)); - r.hits().add(createHit("http://acme.org/h.html", 6, 0)); - r.hits().add(createHit("http://acme.org/i.html", 5, 1)); - r.hits().add(createHit("http://acme.org/j.html", 4, 2)); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,0)); + r.hits().add(createHit("http://acme.org/d.html", 8,0)); + r.hits().add(createHit("http://acme.org/e.html", 8,0)); + r.hits().add(createHit("http://acme.org/f.html", 7,0)); + r.hits().add(createHit("http://acme.org/g.html", 7,0)); + r.hits().add(createHit("http://acme.org/h.html", 6,0)); + r.hits().add(createHit("http://acme.org/i.html", 5,1)); + r.hits().add(createHit("http://acme.org/j.html", 4,2)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -213,15 +239,15 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); - assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/i.html", 5,1,r.hits().get(1)); // Next results docsource.resetQueryCount(); r = doSearch(collapse, q, 2, 2, chained); assertEquals(1, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/j.html",4, 2, r.hits().get(0)); + assertHit("http://acme.org/j.html",4,2,r.hits().get(0)); } /** @@ -239,16 +265,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 1)); - r.hits().add(createHit("http://acme.org/b.html", 10, 1)); - r.hits().add(createHit("http://acme.org/c.html", 10, 0)); - r.hits().add(createHit("http://acme.org/d.html", 10, 0)); - r.hits().add(createHit("http://acme.org/e.html", 10, 0)); - r.hits().add(createHit("http://acme.org/f.html", 10, 0)); - r.hits().add(createHit("http://acme.org/g.html", 10, 0)); - r.hits().add(createHit("http://acme.org/h.html", 10, 0)); - r.hits().add(createHit("http://acme.org/i.html", 10, 0)); - r.hits().add(createHit("http://acme.org/j.html", 10, 1)); + r.hits().add(createHit("http://acme.org/a.html",10,1)); + r.hits().add(createHit("http://acme.org/b.html",10,1)); + r.hits().add(createHit("http://acme.org/c.html",10,0)); + r.hits().add(createHit("http://acme.org/d.html",10,0)); + r.hits().add(createHit("http://acme.org/e.html",10,0)); + r.hits().add(createHit("http://acme.org/f.html",10,0)); + r.hits().add(createHit("http://acme.org/g.html",10,0)); + r.hits().add(createHit("http://acme.org/h.html",10,0)); + r.hits().add(createHit("http://acme.org/i.html",10,0)); + r.hits().add(createHit("http://acme.org/j.html",10,1)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -261,6 +287,17 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/c.html",10,0,r.hits().get(1)); } + public static class QueryMessupSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + AndItem a = new AndItem(); + a.addItem(query.getModel().getQueryTree().getRoot()); + a.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(a); + + return execution.search(query); + } + } + @Test public void testQueryTransformAndCollapsing() { // Set up @@ -272,9 +309,9 @@ public class FieldCollapsingSearcherTestCase { chained.put(collapse, messUp); chained.put(messUp, docsource); - // Caveat: Collapse is set to false, because that's what the collapser asks for - Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid"); - System.out.println(q); + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse+b&collapsefield=amid"); // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); @@ -290,13 +327,13 @@ public class FieldCollapsingSearcherTestCase { docsource.addResult(q, r); // Test basic collapsing on mid - q = new Query("?query=%22test%20collapse%22&collapsefield=amid"); + q = new Query("?query=test_collapse&collapsefield=amid"); r = doSearch(collapse, q, 0, 2, chained); assertEquals(2, docsource.getQueryCount()); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); - assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); } @Test @@ -330,10 +367,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); assertTrue(r.isFilled("placeholder")); - assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); docsource.resetQueryCount(); // Test basic collapsing on mid @@ -344,10 +381,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, docsource.getQueryCount()); assertFalse(r.isFilled("placeholder")); assertTrue(r.isFilled("short")); - assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); } @Test @@ -363,14 +400,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -379,28 +416,29 @@ public class FieldCollapsingSearcherTestCase { Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); // Assert that the regular hits are collapsed - assertEquals(4 + 1, result.getHitCount()); + assertEquals(4+1, result.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10, 0, result.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3)); + assertHit("http://acme.org/a.html",10,0,result.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,result.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,result.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,result.hits().get(3)); // Assert that the aggregation group hierarchy is left intact - HitGroup root = getFirstGroupIn(result.hits()); + HitGroup root= getFirstGroupIn(result.hits()); assertNotNull(root); - assertEquals("group:root:",root.getId().stringValue().substring(0, 11)); // The id ends by a global counter currently - assertEquals(1, root.size()); - HitGroup groupList = (GroupList)root.get("grouplist:g1"); + assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently + assertEquals(1,root.size()); + HitGroup groupList= (GroupList)root.get("grouplist:g1"); assertNotNull(groupList); - assertEquals(1, groupList.size()); - HitGroup group = (HitGroup)groupList.get("group:long:37"); + assertEquals(1,groupList.size()); + HitGroup group= (HitGroup)groupList.get("group:long:37"); assertNotNull(group); } private Group getFirstGroupIn(HitGroup hits) { - for (Hit h : hits) + for (Hit h : hits) { if (h instanceof Group) return (Group)h; + } return null; } @@ -412,8 +450,9 @@ public class FieldCollapsingSearcherTestCase { private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) + for (Searcher current = topOfChain; current != null; current = chained.get(current)) { searchers.add(current); + } return new Chain<>(searchers); } @@ -431,7 +470,7 @@ public class FieldCollapsingSearcherTestCase { @Override public Result search(Query query, Execution execution) { - Result r = execution.search(query); + Result r=execution.search(query); r.hits().add(createAggregationGroup("g1")); return r; } @@ -440,51 +479,10 @@ public class FieldCollapsingSearcherTestCase { Group root = new Group(new RootId(0), new Relevance(1)); GroupList groupList = new GroupList(label); root.add(groupList); - Group value = new Group(new LongId(37l), new Relevance(2.11)); + Group value=new Group(new LongId(37l),new Relevance(2.11)); groupList.add(value); return root; } } - private FastHit createHit(String uri,int relevancy,int mid) { - FastHit hit = new FastHit(uri,relevancy); - hit.setField("amid", String.valueOf(mid)); - return hit; - } - - private void assertHit(String uri,int relevancy,int mid,Hit hit) { - assertEquals(uri,hit.getId().toString()); - assertEquals(relevancy, ((int) hit.getRelevance().getScore())); - assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); - } - - private static class ZeroHitsControl extends com.yahoo.search.Searcher { - - public int queryCount = 0; - - @Override - public Result search(Query query, Execution execution) { - ++queryCount; - if (query.getHits() == 0) { - return new Result(query); - } else { - return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); - } - } - } - - public static class QueryMessupSearcher extends Searcher { - - @Override - public Result search(com.yahoo.search.Query query, Execution execution) { - AndItem a = new AndItem(); - a.addItem(query.getModel().getQueryTree().getRoot()); - a.addItem(new WordItem("b")); - query.getModel().getQueryTree().setRoot(a); - - return execution.search(query); - } - - } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index a4cf7d8c380..b8db5e4d90f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -23,7 +23,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("AND first third", q); + assertSemantics("\"first third\"", q); } @Test @@ -32,7 +32,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("AND bc first third fg", q); + assertSemantics("\"bc first third fg\"", q); } @Test @@ -41,7 +41,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("+bc -(AND first third)", q); + assertSemantics("+bc -\"first third\"", q); } @Test @@ -50,7 +50,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("AND 9 2 7 0 bc third 2 3 8 9", q); + assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q); } private static Item parseQuery(String query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index e2ac44316e7..82a5a0c7a24 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -73,7 +73,7 @@ public class IndexFactsTestCase { Query q = newQuery("?query=a:b", indexFacts); assertEquals("a:b", q.getModel().getQueryTree().getRoot().toString()); q = newQuery("?query=notarealindex:b", indexFacts); - assertEquals("AND notarealindex b", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("\"notarealindex b\"", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -302,8 +302,8 @@ public class IndexFactsTestCase { IndexFacts.Session session2 = indexFacts.newSession(query2.getModel().getSources(), query2.getModel().getRestrict()); assertTrue(session1.getIndex("url").isUriIndex()); assertTrue(session2.getIndex("url").isUriIndex()); - assertEquals("AND url:https url:foo url:bar", query1.getModel().getQueryTree().toString()); - assertEquals("AND url:https url:foo url:bar", query2.getModel().getQueryTree().toString()); + assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString()); + assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java index aa507d38be5..dbeced57c52 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -5,7 +5,6 @@ import static org.junit.Assert.*; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import org.junit.After; import org.junit.Before; @@ -24,50 +23,50 @@ import com.yahoo.text.Utf8; /** * Functional test for InputCheckingSearcher. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class InputCheckingSearcherTestCase { Execution execution; @Before - public void setUp() { + public void setUp() throws Exception { execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)), - Execution.Context.createContextStub(new IndexFacts())); + Execution.Context.createContextStub(new IndexFacts())); } @After - public void tearDown() { + public void tearDown() throws Exception { execution = null; } @Test - public void testCommonCase() { + public final void testCommonCase() { Result r = execution.search(new Query("/search/?query=three+blind+mice")); assertNull(r.hits().getErrorHit()); } @Test - public void candidateButAsciiOnly() { + public final void candidateButAsciiOnly() { Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); assertNull(r.hits().getErrorHit()); } @Test - public void candidateButValid() throws UnsupportedEncodingException { + public final void candidateButValid() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public void testDoubleEncoded() throws UnsupportedEncodingException { + public final void testDoubleEncoded() throws UnsupportedEncodingException { String rawQuery = "å å å å å å"; byte[] encodedOnce = Utf8.toBytes(rawQuery); char[] secondEncodingBuffer = new char[encodedOnce.length]; @@ -75,42 +74,33 @@ public class InputCheckingSearcherTestCase { secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); } String query = new String(secondEncodingBuffer); - Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8))); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); assertEquals(1, r.hits().getErrorHit().errors().size()); } @Test - public void testRepeatedConsecutiveTermsInPhrase() { - Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); + public final void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); assertNotNull(r.hits().getErrorHit()); - assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", - r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); } - @Test - public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { - Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); + public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); assertNotNull(r.hits().getErrorHit()); - assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", - r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); - r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22")); + r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); } - @Test - public void testThatMaxRepeatedTermsInPhraseIs10() { - Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22")); + public final void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22")); + r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); assertNotNull(r.hits().getErrorHit()); - assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"", - r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); } - } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index c831ee29631..34c3da395b7 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -16,7 +16,6 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; @@ -942,12 +941,12 @@ public class QueryTestCase { @Test public void testImplicitPhraseIsDefault() { Query query = new Query(httpEncode("?query=it's fine")); - assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString()); + assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString()); } @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -962,7 +961,7 @@ public class QueryTestCase { @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -973,56 +972,6 @@ public class QueryTestCase { query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); - // 'it' and 's' should have connectivity 1 - AndItem root = (AndItem)query.getModel().getQueryTree().getRoot(); - AndSegmentItem sand = (AndSegmentItem)root.getItem(0); - WordItem it = (WordItem)sand.getItem(0); - assertEquals("it", it.getWord()); - WordItem s = (WordItem)sand.getItem(1); - assertEquals("s", s.getWord()); - assertEquals(s, it.getConnectedItem()); - assertEquals(1.0, it.getConnectivity(), 0.00000001); - } - - @Test - public void testImplicitAndConnectivity() { - SearchDefinition test = new SearchDefinition("test"); - Index myField = new Index("myfield"); - myField.addCommand("phrase-segmenting false"); - test.addIndex(myField); - IndexModel indexModel = new IndexModel(test); - - { - Query query = new Query(httpEncode("?query=myfield:b.c.d")); - query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString()); - AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); - WordItem b = (WordItem) root.getItem(0); - WordItem c = (WordItem) root.getItem(1); - WordItem d = (WordItem) root.getItem(2); - assertEquals(c, b.getConnectedItem()); - assertEquals(1.0, b.getConnectivity(), 0.00000001); - assertEquals(d, c.getConnectedItem()); - assertEquals(1.0, c.getConnectivity(), 0.00000001); - } - - { - Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e")); - query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString()); - AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); - WordItem a = (WordItem) root.getItem(0); - WordItem b = (WordItem) root.getItem(1); - WordItem c = (WordItem) root.getItem(2); - WordItem d = (WordItem) root.getItem(3); - WordItem e = (WordItem) root.getItem(4); - assertNull(a.getConnectedItem()); - assertEquals(c, b.getConnectedItem()); - assertEquals(1.0, b.getConnectivity(), 0.00000001); - assertEquals(d, c.getConnectedItem()); - assertEquals(1.0, c.getConnectivity(), 0.00000001); - assertNull(d.getConnectedItem()); - } } @Test |