diff options
| author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-03-02 11:57:12 +0100 |
|---|---|---|
| committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-03-02 11:57:12 +0100 |
| commit | cdeb661adda84714f83ce071286ac24be27ae1cb (patch) | |
| tree | a0a28d9bbe3e4e1a8956fde28e71082f6cf16364 /container-search/src | |
| parent | 2120128c0194fad3507527b2e1a48187ea71d5cc (diff) | |
Revert "Merge pull request #12385 from vespa-engine/revert-12365-bratseth/phrase-segmenting-off"
This reverts commit 6c52f21816ce79909feac4908d594204ad2fd4e7, reversing
changes made to 9ac4f3d18e1dbb11678b8d61fa50c9ad1cdf61d2.
Diffstat (limited to 'container-search/src')
16 files changed, 452 insertions, 427 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 365ee299ca4..5e7fddd7fe7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; - import com.yahoo.language.process.StemMode; import java.util.ArrayList; @@ -10,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; - /** * Information about configured settings of a field or field collection (an actual index or not) in a search definition. * There are two types of settings: @@ -74,8 +72,8 @@ public class Index { private boolean isNGram = false; private int gramSize = 2; - /** Whether implicit phrases should lead to a phrase item or an and item */ - private boolean phraseSegmenting = true; + /** Whether implicit phrases should lead to a phrase item or an and item. */ + private Boolean phraseSegmenting = false; /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 062a514056b..00935392683 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -109,7 +109,6 @@ public final class IndexModel { return searchDefinitions; } - @SuppressWarnings("deprecation") private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) { SearchDefinition union = new SearchDefinition(IndexFacts.unionName); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index d9b969757c2..49bdba2c90f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -30,6 +30,7 @@ public class AllParser extends SimpleParser { super(environment); } + @Override protected Item parseItems() { int position = tokens.getPosition(); try { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index dd836e9c8e1..b714a1d8b34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,21 +35,12 @@ public class AnyParser extends SimpleParser { return anyItems(true); } - Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) { - return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet())); - } - Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - Item filterRoot; - setState(queryLanguage, indexFacts); tokenize(filter, null, indexFacts, queryLanguage); - filterRoot = anyItems(true); - - if (filterRoot == null) { - return null; - } + Item filterRoot = anyItems(true); + if (filterRoot == null) return null; markAllTermsAsFilters(filterRoot); return filterRoot; @@ -61,18 +52,10 @@ public class AnyParser extends SimpleParser { try { tokens.skipMultiple(PLUS); + if ( ! tokens.skipMultiple(MINUS)) return null; + if (tokens.currentIsNoIgnore(SPACE)) return null; - if (!tokens.skipMultiple(MINUS)) { - return null; - } - - if (tokens.currentIsNoIgnore(SPACE)) { - return null; - } - - if (item == null) { - item = indexableItem(); - } + item = indexableItem(); if (item == null) { item = compositeItem(); @@ -88,13 +71,13 @@ public class AnyParser extends SimpleParser { } } } - if (item!=null) + if (item != null) item.setProtected(true); + return item; } finally { - if (item == null) { + if (item == null) tokens.setPosition(position); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 9ddfea6dffb..3d244312b2f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -50,32 +50,28 @@ abstract class SimpleParser extends StructuredParser { private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item; + Item item = null; do { - item = null; - - if (item == null) { - item = positiveItem(); - if (item != null) { - if (not == null) { - not = new NotItem(); - not.addPositiveItem(item); - topLevelItem = combineItems(topLevelItem, not); - } else { - not.addPositiveItem(item); - } + item = positiveItem(); + if (item != null) { + if (not == null) { + not = new NotItem(); + not.addPositiveItem(item); + topLevelItem = combineItems(topLevelItem, not); + } else { + not.addPositiveItem(item); } } if (item == null) { item = negativeItem(); if (item != null) { - if (not == null && item != null) { + if (not == null) { not = new NotItem(); not.addNegativeItem(item); topLevelItem = combineItems(topLevelItem, not); - } else if (item != null) { + } else { not.addNegativeItem(item); } } @@ -97,9 +93,8 @@ abstract class SimpleParser extends StructuredParser { if (item != null) { if (topLevelItem == null) { topLevelItem = item; - } else if (needNewTopLevel(topLevelItem, item)) { + } else if (needNewORTopLevel(topLevelItem, item)) { CompositeItem newTop = new OrItem(); - newTop.addItem(topLevelItem); newTop.addItem(item); topLevelItem = newTop; @@ -131,6 +126,7 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives + System.out.println("Extracting positive item from " + topLevelItem); not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); return not; } else { // Only negatives - ignore them @@ -144,21 +140,13 @@ abstract class SimpleParser extends StructuredParser { } } - - /** Says whether we need a new top level item given the new item */ - private boolean needNewTopLevel(Item topLevelItem, Item item) { - if (item == null) { - return false; - } - if (topLevelItem instanceof TermItem) { - return true; - } - if (topLevelItem instanceof PhraseItem) { - return true; - } - if (topLevelItem instanceof BlockItem) { - return true; - } + /** Says whether we need a new top level OR item given the new item */ + private boolean needNewORTopLevel(Item topLevelItem, Item item) { + if (item == null) return false; + if (topLevelItem instanceof TermItem) return true; + if (topLevelItem instanceof PhraseItem) return true; + if (topLevelItem instanceof BlockItem) return true; + if ( topLevelItem instanceof AndItem) return true; return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 5e292a06b0f..9ba6c1a8101 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -442,9 +442,9 @@ abstract class StructuredParser extends AbstractParser { Item item = null; try { - if (!tokens.currentIs(WORD) - && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) - && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { + if ( ! tokens.currentIs(WORD) + && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) + && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { return null; } Token word = tokens.next(); @@ -557,6 +557,7 @@ abstract class StructuredParser extends AbstractParser { if (composite != null) { composite.addItem(word); + connectLastTermsIn(composite); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); @@ -584,6 +585,7 @@ abstract class StructuredParser extends AbstractParser { } composite.addItem(firstWord); composite.addItem(word); + connectLastTermsIn(composite); } else if (word instanceof PhraseItem) { composite = (PhraseItem)word; } else { @@ -654,6 +656,15 @@ abstract class StructuredParser extends AbstractParser { } } + private void connectLastTermsIn(CompositeItem composite) { + int items = composite.items().size(); + if (items < 2) return; + Item nextToLast = composite.items().get(items - 2); + Item last = composite.items().get(items - 1); + if ( ! (nextToLast instanceof TermItem)) return; + ((TermItem)nextToLast).setConnectivity(last, 1); + } + private boolean addStartMarking() { if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) { tokens.skip(); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index d39a488626b..e346a766738 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -97,11 +97,10 @@ public class DocumentSourceSearcher extends Searcher { public Result search(Query query, Execution execution) { queryCount++; Result r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { + if (r == null) r = defaultFilledResult.clone(); - } else { + else r = r.clone(); - } r.setQuery(query); r.hits().trim(query.getOffset(), query.getHits()); @@ -182,11 +181,8 @@ public class DocumentSourceSearcher extends Searcher { * reset. For testing - not reliable if multiple threads makes * queries simultaneously */ - public int getQueryCount() { - return queryCount; - } + public int getQueryCount() { return queryCount; } + + public void resetQueryCount() { queryCount = 0; } - public void resetQueryCount() { - queryCount=0; - } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java index 5cae40bd10d..df35d8dbdea 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.junit.Assert.assertEquals; @@ -34,7 +35,7 @@ public class ExactMatchAndDefaultIndexTestCase { q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null))); assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString()); q = new Query("?query=" + enc("a/b foo.com")); - assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND a b foo com", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -44,11 +45,7 @@ public class ExactMatchAndDefaultIndexTestCase { } private String enc(String s) { - try { - return URLEncoder.encode(s, "utf-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + return URLEncoder.encode(s, StandardCharsets.UTF_8); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 0fdad1a1f9c..c1db7d73561 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; @@ -48,7 +49,9 @@ public class ParseTestCase { @Test public void testTermWithIndexPrefix() { - tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY); + tester.assertParsed("url:foobar", + "url:foobar", + Query.Type.ANY); } @Test @@ -59,104 +62,98 @@ public class ParseTestCase { @Test public void testMultipleTermsWithUTF8EncodingOred() { tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen", - "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY); + "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", + Query.Type.ANY); } @Test public void testMultipleTermsWithMultiplePrefixes() { tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); + "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); } @Test public void testSimpleQueryDefaultOr() { - tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", - Query.Type.ANY); + tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", Query.Type.ANY); } @Test public void testOrAndNot() { tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); + "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); } @Test public void testSimpleOrNestedAnd() { tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy", - "foobar +(foo bar baz) xyzzy", Query.Type.ANY); + "foobar +(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testSimpleOrNestedNot() { tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)", - "foobar -(foo bar baz) xyzzy", Query.Type.ANY); + "foobar -(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testOrNotNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrAndNotNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAnd() { - tester.assertParsed( - "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", - "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", + "foobar +(fo ba foba) foo bar +(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrMultipleNestedNot() { - tester.assertParsed( - "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", - "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", + "foobar -(fo ba foba) foo bar -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", - "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", + "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", - "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", + "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", + Query.Type.ANY); } @Test public void testExplicitPhrase() { - Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); + Item root = tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); assertTrue(root instanceof PhraseItem); assertTrue(((PhraseItem)root).isExplicit()); } @@ -164,21 +161,20 @@ public class ParseTestCase { @Test public void testPhraseWithIndex() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ANY); + "normal.title:\"foo bar foobar\"", Query.Type.ANY); } @Test public void testPhrasesAndTerms() { tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"", - "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); + "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); } @Test public void testPhrasesAndTermsWithOperators() { - tester.assertParsed( - "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", - "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", + "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", + Query.Type.ANY); } @Test @@ -188,38 +184,40 @@ public class ParseTestCase { @Test public void testTermWithCatalogAndIndexPrefixDefaultAnd() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ALL); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ALL); } @Test public void testMultipleTermsWithMultiplePrefixesDefaultAnd() { tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.ALL); } @Test public void testSimpleQueryDefaultAnd() { - tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", - Query.Type.ALL); + tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", Query.Type.ALL); } @Test public void testNotDefaultAnd() { - tester.assertParsed( - "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", - "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL); + tester.assertParsed("+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", + "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", + Query.Type.ALL); } @Test public void testSimpleTermQueryDefaultPhrase() { - tester.assertParsed("foobar", "foobar", Query.Type.PHRASE); + tester.assertParsed("foobar", + "foobar", + Query.Type.PHRASE); } @Test public void testSimpleQueryDefaultPhrase() { - Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz", - Query.Type.PHRASE); + Item root = tester.assertParsed("\"foobar foo bar baz\"", + "foobar foo bar baz", + Query.Type.PHRASE); assertTrue(root instanceof PhraseItem); assertFalse(((PhraseItem)root).isExplicit()); } @@ -227,23 +225,25 @@ public class ParseTestCase { @Test public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() { tester.assertParsed("\"url foobar bar normal title foo baz\"", - "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.PHRASE); } @Test public void testOdd1() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL); + tester.assertParsed("AND window print error", "+window.print() +error", + Query.Type.ALL); } @Test public void testOdd2() { - tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL); + tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"", + Query.Type.ALL); } @Test public void testOdd2Uppercase() { - tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", - Query.Type.ALL); + tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", Query.Type.ALL); } @Test @@ -280,19 +280,19 @@ public class ParseTestCase { @Test public void testNestedCompositesDefaultOr() { tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); } @Test public void testNestedCompositesDefaultAnd() { tester.assertParsed("AND foo (OR foobar bar baz) xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); } @Test public void testNestedCompositesPhraseDefault() { tester.assertParsed("\"foo foobar bar baz xyzzy\"", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); } @Test @@ -349,8 +349,7 @@ public class ParseTestCase { @Test public void testNumericWithIndex() { - tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", - Query.Type.ANY); + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ANY); } @Test @@ -361,14 +360,14 @@ public class ParseTestCase { @Test public void testMultipleIntegerWithIndex() { tester.assertParsed("OR document.size:[34;454] date:>1234567890", - "document.size:[34;454] date:>1234567890", Query.Type.ANY); + "document.size:[34;454] date:>1234567890", Query.Type.ANY); } @Test public void testMixedNumericAndOtherTerms() { tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890", - "foo +document.size:<1024 +xyzzy date:>123456890", - Query.Type.ANY); + "foo +document.size:<1024 +xyzzy date:>123456890", + Query.Type.ANY); } @Test @@ -378,20 +377,18 @@ public class ParseTestCase { @Test public void testItemPhraseEmptyPhrase() { - tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", - Query.Type.ANY); + tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ANY); } @Test public void testSimpleQuery() { - tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", - Query.Type.ANY); + tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", Query.Type.ANY); } @Test public void testExcessivePluses() { tester.assertParsed("+(AND other is nothing) -test", - "++other +++++is ++++++nothing -test", Query.Type.ANY); + "++other +++++is ++++++nothing -test", Query.Type.ANY); } @Test @@ -401,39 +398,38 @@ public class ParseTestCase { @Test public void testPlusesAndMinuses() { - Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY); - assertTrue(root instanceof PhraseItem); - assertFalse(((PhraseItem)root).isExplicit()); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ANY); } @Test public void testNumbers() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", - "123+2132odfd.934032,,32423", Query.Type.ANY); + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ANY); } @Test public void testOtherSignsInQuote() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", - Query.Type.ANY); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ANY); } @Test public void testGribberish() { tester.assertParsed("1349832840234l3040roer\u00e6lf12", - ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY); + ",1349832840234l3040roer\u00e6lf12", + Query.Type.ANY); } @Test public void testUrl() { - tester.assertParsed("www:\"www hotelaiguablava com\"", - "+www:www.hotelaiguablava:com", Query.Type.ANY); + tester.assertParsed("AND www:www www:hotelaiguablava www:com", + "+www:www.hotelaiguablava:com", + Query.Type.ANY); } @Test public void testUrlGribberish() { - tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive", - "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY); + tester.assertParsed("OR (AND 3 16) fast.type:lycosoffensive", + "[ 3:16 fast.type:lycosoffensive", + Query.Type.ANY); } @Test @@ -475,8 +471,7 @@ public class ParseTestCase { @Test public void testPrefixWithDotAdvanced() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ADVANCED); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ADVANCED); } @Test @@ -486,20 +481,21 @@ public class ParseTestCase { @Test public void testSimplePhraseAdvanced() { - tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", - Query.Type.ADVANCED); + tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ADVANCED); } @Test public void testSimplePhraseWithIndexAdvanced() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED); + "normal.title:\"foo bar foobar\"", + Query.Type.ADVANCED); } @Test public void testMultiplePhrasesAdvanced() { tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"", - "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED); + "\"foo bar foobar\" and \"baz gaz faz\"", + Query.Type.ADVANCED); } @Test @@ -661,23 +657,23 @@ public class ParseTestCase { @Test public void testImplicitPhrase1Advanced() { - tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED); + tester.assertParsed("AND test if", "--test+-if", Query.Type.ADVANCED); } @Test public void testImplicitPhrase2Advanced() { - tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ADVANCED); } @Test public void testImplicitPhrase3Advanced() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ADVANCED); } @Test public void testImplicitPhrase4Advanced() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ADVANCED); } @Test @@ -730,7 +726,7 @@ public class ParseTestCase { @Test public void testSingleHyphen() { - tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL); + tester.assertParsed("AND a b", "a-b", Query.Type.ALL); } @Test @@ -883,27 +879,27 @@ public class ParseTestCase { @Test public void testSimpleDotPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b.c d", Query.Type.ANY); } @Test public void testSimpleHyphenPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b-c d", Query.Type.ANY); } @Test public void testAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY); + tester.assertParsed("OR (AND a b) c d", "a.b c d", Query.Type.ANY); } @Test public void testYetAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY); + tester.assertParsed("OR a b (AND c d)", "a b c.d", Query.Type.ANY); } @Test public void testVariousSeparatorsPhraseAny() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ANY); } @Test @@ -918,45 +914,44 @@ public class ParseTestCase { @Test public void testIndexedDottedPhraseAny() { - tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND url:b url:c) d", "a url:b.c d", Query.Type.ANY); } @Test public void testIndexedPlusedPhraseAny() { - tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d", - Query.Type.ANY); + tester.assertParsed("OR a (AND normal.title:b normal.title:c) d", "a normal.title:b+c d", Query.Type.ANY); } @Test public void testNestedNotAny() { tester.assertParsed( - "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a", + "RANK (+(OR normal.title:foobar (AND url:www url:pvv url:org)) -foo) a", "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY); } @Test public void testDottedPhraseAdvanced() { - tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED); + tester.assertParsed("OR a (AND b c)", "a or b.c", Query.Type.ADVANCED); } @Test public void testHyphenPhraseAdvanced() { - tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a (AND b c)) d", "a and b-c or d", Query.Type.ADVANCED); } @Test public void testAnotherDottedPhraseAdvanced() { - tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a b) c", "a.b or c", Query.Type.ADVANCED); } @Test public void testNottedDottedPhraseAdvanced() { - tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED); + tester.assertParsed("+a -(AND c d)", "a andnot c.d", Query.Type.ADVANCED); } @Test public void testVariousSeparatorsPhraseAdvanced() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ADVANCED); } @Test @@ -976,14 +971,14 @@ public class ParseTestCase { @Test public void testNestedPlussedPhraseAdvanced() { - tester.assertParsed("AND (OR a normal.title:\"b c\") d", + tester.assertParsed("AND (OR a (AND normal.title:b normal.title:c)) d", "a or normal.title:b+c and d", Query.Type.ADVANCED); } @Test public void testNottedNestedDottedPhraseAdvanced() { tester.assertParsed( - "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo", + "+(AND a (OR normal.title:foobar (AND url:www url:pvv url:org))) -foo", "a and (normal.title:foobar or url:www.pvv.org) andnot foo", Query.Type.ADVANCED); } @@ -995,7 +990,7 @@ public class ParseTestCase { @Test public void testPlusedTwiceThenQuotedPhraseAny() { - tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a+b+\"c d\"", Query.Type.ANY); } @Test @@ -1005,7 +1000,7 @@ public class ParseTestCase { @Test public void testPhrasesInBraces() { - tester.assertParsed("url.domain:\"microsoft com\"", + tester.assertParsed("AND url.domain:microsoft url.domain:com", "+(url.domain:microsoft.com)", Query.Type.ALL); } @@ -1053,17 +1048,17 @@ public class ParseTestCase { @Test public void testPhraseNotPrefix() { - tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND prefix bar)", "foo prefix*bar", Query.Type.ANY); } @Test public void testPhraseNotSubstring() { - tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND substring bar)", "foo *substring*bar", Query.Type.ANY); } @Test public void testPhraseNotSuffix() { - tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY); + tester.assertParsed("OR (AND foo suffix) bar", "foo*suffix bar", Query.Type.ANY); } @Test @@ -1086,20 +1081,17 @@ public class ParseTestCase { @Test public void testIndexedPhraseNotPrefix() { - tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:prefix foo.bar:xyzzy", "foo.bar:prefix*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSubstring() { - tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:substring foo.bar:xyzzy", "foo.bar:*substring*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSuffix() { - tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:xyzzy foo.bar:suffix", "foo.bar:xyzzy*suffix", Query.Type.ANY); } @Test @@ -1120,20 +1112,20 @@ public class ParseTestCase { assertTrue(root instanceof SuffixItem); } - /** Non existing index → phrase **/ + /** Non existing index → and **/ @Test public void testNonIndexPhraseNotPrefix() { - tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY); + tester.assertParsed("AND void prefix", "void:prefix*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSubstring() { - tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY); + tester.assertParsed("AND void substring", "void:*substring*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSuffix() { - tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY); + tester.assertParsed("AND void suffix", "void:*suffix", Query.Type.ANY); } /** Explicit phrase → remove '*' **/ @@ -1198,7 +1190,7 @@ public class ParseTestCase { /** Extra spaces with index **/ @Test public void testIndexPrefixExtraSpace() { - tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY); + tester.assertParsed("AND foo prefix", "foo:prefix *", Query.Type.ANY); } @Test @@ -1419,7 +1411,7 @@ public class ParseTestCase { @Test public void testMultipleDifferentPhraseSeparators() { - tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY); + tester.assertParsed("AND foo bar", "foo.-.bar", Query.Type.ANY); } @Test @@ -1430,19 +1422,17 @@ public class ParseTestCase { @Test public void testReallyNoisyQuery1() { - tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", - Query.Type.ALL); + tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", Query.Type.ALL); } @Test public void testReallyNoisyQuery2() { - tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); + tester.assertParsed("AND \u03bc\u03bc hei", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); } @Test public void testReallyNoisyQuery3() { - tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der", - Query.Type.ALL); + tester.assertParsed("AND hei hallo du der", "hei-hallo;du;der", Query.Type.ALL); } @Test @@ -1478,7 +1468,7 @@ public class ParseTestCase { @Test public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() { - tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); + tester.assertParsed("AND yz a", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); } @Test @@ -1498,7 +1488,7 @@ public class ParseTestCase { @Test public void testImplicitPhrasingWithIndex() { - tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY); + tester.assertParsed("AND a:b a:c", "a:/b/c", Query.Type.ANY); } @Test @@ -1508,7 +1498,7 @@ public class ParseTestCase { @Test public void testSingleNoisyPhraseWithIndex() { - tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY); + tester.assertParsed("AND mail:yahoo mail:com", "mail:@yahoo.com", Query.Type.ANY); } @Test @@ -1599,7 +1589,7 @@ public class ParseTestCase { "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions", Query.Type.ALL); tester.assertParsed( - "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", + "AND ull:http ull:www ull:neue ull:oz ull:de ull:information ull:pub ull:Boulevard ull:index ull:html ull:file ull:a ull:3 ull:s ull:4 ull:file s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}", Query.Type.ALL); } @@ -1640,7 +1630,7 @@ public class ParseTestCase { @Test public void testTooLongQueryTerms() { - tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"", + tester.assertParsed("AND 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof 2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", Query.Type.ALL); } @@ -1648,22 +1638,22 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); - customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY); + customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); } @Test public void testNonIndexWithColons1() { - tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex)", "this is notan:iindex", Query.Type.ANY); } @Test public void testNonIndexWithColons2() { - tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex either)", "this is notan:iindex:either", Query.Type.ANY); } @Test public void testIndexThenUnderscoreTermBecomesIndex() { - tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY); + tester.assertParsed("AND name:batch name:article", "name:batch_article", Query.Type.ANY); } @Test @@ -1671,17 +1661,17 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + assertTrue(item instanceof AndSegmentItem); + AndSegmentItem segment = (AndSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(2)).getIndexName()); } @Test @@ -1689,22 +1679,22 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + assertTrue(item instanceof AndSegmentItem); + AndSegmentItem segment = (AndSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem)segment.getItem(2)).getIndexName()); } @Test public void testAndItemAndImplicitPhrase() { - tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"", + tester.assertParsed("AND \u00d8 \u00d8 \u00d8 \u00d9", "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1736,7 +1726,7 @@ public class ParseTestCase { @Test public void testFakeCJKSegmentingOfMultiplePhrases() { Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertEquals("name:\"'first second' s\"", item.toString()); + assertEquals("AND (SAND name:first name:second) name:s", item.toString()); } @Test @@ -1801,7 +1791,7 @@ public class ParseTestCase { @Test public void testCommaOnlyLeadsToImplicitPhrasing() { - tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL); + tester.assertParsed("AND A B C", "A,B,C", Query.Type.ALL); } @Test @@ -1873,8 +1863,8 @@ public class ParseTestCase { @Test public void testJPMobileExceptionQuery() { - tester.assertParsed("OR concat and \"make string\" 1 47 or", - "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); + tester.assertParsed("OR concat and (AND make string) 1 47 or", + "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); } @Test @@ -1882,7 +1872,7 @@ public class ParseTestCase { tester.assertParsed("b", "a: b", Query.Type.ALL); tester.assertParsed("AND a b", "a : b", Query.Type.ALL); tester.assertParsed("AND a b", "a :b", Query.Type.ALL); - tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL); + tester.assertParsed("AND a b", "a.:b", Query.Type.ALL); tester.assertParsed("a:b", "a:b", Query.Type.ALL); } @@ -1917,8 +1907,7 @@ public class ParseTestCase { tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ALL); - // The last one here is a little weird, but it's not a problem, - // so I let it pass for now... + // The last one here is a little weird, but it's not a problem, so let it pass for now... tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ANY); @@ -1926,7 +1915,7 @@ public class ParseTestCase { @Test public void testMixedCaseIndexNames() { - tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d", + tester.assertParsed("AND mixedCase:a mixedCase:b notAnIndex c mixedCase:d", "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d", Query.Type.ALL); } @@ -1934,7 +1923,7 @@ public class ParseTestCase { /** CJK special tokens should be recognized also on non-boundaries */ @Test public void testChineseSpecialTokens() { - tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"", + tester.assertParsed("AND cat tcp/ip zu foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz", "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1945,7 +1934,7 @@ public class ParseTestCase { @Test public void testChineseSpecialTokensWithMultiSegmentReplace() { // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test - tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond", + tester.assertParsed("AND tcp/ip firstsecond dotnet firstsecond (SAND first second)","tcp/ipspecial-token-fs.net special-token-fs firstsecond", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE); } @@ -2014,7 +2003,7 @@ public class ParseTestCase { @Test public void testVersionNumbers() { - tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL); + tester.assertParsed("AND 1 0 9", "1.0.9", Query.Type.ALL); } @Test @@ -2321,7 +2310,7 @@ public class ParseTestCase { @Test public void testOdd1Web() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB); + tester.assertParsed("AND window print error", "+window.print() +error",Query.Type.WEB); } @Test @@ -2351,13 +2340,13 @@ public class ParseTestCase { @Test public void testDefaultWebIndices() { - tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB); - tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB); - tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB); - tester.assertParsed("link:b","link:b",Query.Type.WEB); - tester.assertParsed("url:b","url:b",Query.Type.WEB); - tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB); - tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB); + tester.assertParsed("AND notanindex b","notanindex:b", Query.Type.WEB); + tester.assertParsed("site:\"b $\"","site:b", Query.Type.WEB); + tester.assertParsed("hostname:b","hostname:b", Query.Type.WEB); + tester.assertParsed("link:b","link:b", Query.Type.WEB); + tester.assertParsed("url:b","url:b", Query.Type.WEB); + tester.assertParsed("inurl:b","inurl:b", Query.Type.WEB); + tester.assertParsed("intitle:b","intitle:b", Query.Type.WEB); } @Test @@ -2527,7 +2516,7 @@ public class ParseTestCase { @Test public void testSiteAndSegmentPhrasesFollowedByText() { - tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'", + tester.assertParsed("AND host.all:\"www abc com x y-z $\" (SAND a b)", "host.all:www.abc.com/x'y-z a'b", "", Query.Type.ALL, Language.ENGLISH); } @@ -2544,7 +2533,7 @@ public class ParseTestCase { @Test public void testNonAsciiNumber() { - tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); + tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java index 91cf5015cba..0ca4b8aa615 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -45,7 +45,7 @@ public class CJKSearcherTestCase { @Test public void testCjkQueryWithOverlappingTokens() { // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd" - assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + assertTransformed("bcd", "SAND bc cd", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, TestLinguistics.INSTANCE); // While "efg" will be segmented into one of the standard options, "e" "fg" diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java index 12e756a07ee..023cd3c2849 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java @@ -71,7 +71,7 @@ public class LiteralBoostSearcherTestCase { @Test public void testQueryWithoutBoost() { - assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", + assertEquals("RANK (AND nonexistant a nonexistant b) default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1")); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index 4875121a501..12619bf0a5e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -40,34 +40,8 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public class FieldCollapsingSearcherTestCase { - private FastHit createHit(String uri,int relevancy,int mid) { - FastHit hit = new FastHit(uri,relevancy); - hit.setField("amid", String.valueOf(mid)); - return hit; - } - - private void assertHit(String uri,int relevancy,int mid,Hit hit) { - assertEquals(uri,hit.getId().toString()); - assertEquals(relevancy, ((int) hit.getRelevance().getScore())); - assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); - } - - private static class ZeroHitsControl extends com.yahoo.search.Searcher { - public int queryCount = 0; - public com.yahoo.search.Result search(com.yahoo.search.Query query, - com.yahoo.search.searchchain.Execution execution) { - ++queryCount; - if (query.getHits() == 0) { - return new Result(query); - } else { - return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); - } - } - } - @Test public void testFieldCollapsingWithoutHits() { // Set up @@ -116,14 +90,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -133,10 +107,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -152,14 +126,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -169,10 +143,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3, r.hits().get(3)); } @Test @@ -185,14 +159,14 @@ public class FieldCollapsingSearcherTestCase { Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -220,16 +194,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,0)); - r.hits().add(createHit("http://acme.org/d.html", 8,0)); - r.hits().add(createHit("http://acme.org/e.html", 8,0)); - r.hits().add(createHit("http://acme.org/f.html", 7,0)); - r.hits().add(createHit("http://acme.org/g.html", 7,0)); - r.hits().add(createHit("http://acme.org/h.html", 6,0)); - r.hits().add(createHit("http://acme.org/i.html", 5,1)); - r.hits().add(createHit("http://acme.org/j.html", 4,2)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0)); + r.hits().add(createHit("http://acme.org/d.html", 8, 0)); + r.hits().add(createHit("http://acme.org/e.html", 8, 0)); + r.hits().add(createHit("http://acme.org/f.html", 7, 0)); + r.hits().add(createHit("http://acme.org/g.html", 7, 0)); + r.hits().add(createHit("http://acme.org/h.html", 6, 0)); + r.hits().add(createHit("http://acme.org/i.html", 5, 1)); + r.hits().add(createHit("http://acme.org/j.html", 4, 2)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -239,15 +213,15 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/i.html", 5,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1)); // Next results docsource.resetQueryCount(); r = doSearch(collapse, q, 2, 2, chained); assertEquals(1, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/j.html",4,2,r.hits().get(0)); + assertHit("http://acme.org/j.html",4, 2, r.hits().get(0)); } /** @@ -265,16 +239,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,1)); - r.hits().add(createHit("http://acme.org/b.html",10,1)); - r.hits().add(createHit("http://acme.org/c.html",10,0)); - r.hits().add(createHit("http://acme.org/d.html",10,0)); - r.hits().add(createHit("http://acme.org/e.html",10,0)); - r.hits().add(createHit("http://acme.org/f.html",10,0)); - r.hits().add(createHit("http://acme.org/g.html",10,0)); - r.hits().add(createHit("http://acme.org/h.html",10,0)); - r.hits().add(createHit("http://acme.org/i.html",10,0)); - r.hits().add(createHit("http://acme.org/j.html",10,1)); + r.hits().add(createHit("http://acme.org/a.html", 10, 1)); + r.hits().add(createHit("http://acme.org/b.html", 10, 1)); + r.hits().add(createHit("http://acme.org/c.html", 10, 0)); + r.hits().add(createHit("http://acme.org/d.html", 10, 0)); + r.hits().add(createHit("http://acme.org/e.html", 10, 0)); + r.hits().add(createHit("http://acme.org/f.html", 10, 0)); + r.hits().add(createHit("http://acme.org/g.html", 10, 0)); + r.hits().add(createHit("http://acme.org/h.html", 10, 0)); + r.hits().add(createHit("http://acme.org/i.html", 10, 0)); + r.hits().add(createHit("http://acme.org/j.html", 10, 1)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -287,17 +261,6 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/c.html",10,0,r.hits().get(1)); } - public static class QueryMessupSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - AndItem a = new AndItem(); - a.addItem(query.getModel().getQueryTree().getRoot()); - a.addItem(new WordItem("b")); - query.getModel().getQueryTree().setRoot(a); - - return execution.search(query); - } - } - @Test public void testQueryTransformAndCollapsing() { // Set up @@ -309,9 +272,9 @@ public class FieldCollapsingSearcherTestCase { chained.put(collapse, messUp); chained.put(messUp, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse+b&collapsefield=amid"); + // Caveat: Collapse is set to false, because that's what the collapser asks for + Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid"); + System.out.println(q); // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); @@ -327,13 +290,13 @@ public class FieldCollapsingSearcherTestCase { docsource.addResult(q, r); // Test basic collapsing on mid - q = new Query("?query=test_collapse&collapsefield=amid"); + q = new Query("?query=%22test%20collapse%22&collapsefield=amid"); r = doSearch(collapse, q, 0, 2, chained); assertEquals(2, docsource.getQueryCount()); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1)); } @Test @@ -367,10 +330,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); assertTrue(r.isFilled("placeholder")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); docsource.resetQueryCount(); // Test basic collapsing on mid @@ -381,10 +344,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, docsource.getQueryCount()); assertFalse(r.isFilled("placeholder")); assertTrue(r.isFilled("short")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -400,14 +363,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -416,29 +379,28 @@ public class FieldCollapsingSearcherTestCase { Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); // Assert that the regular hits are collapsed - assertEquals(4+1, result.getHitCount()); + assertEquals(4 + 1, result.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,result.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,result.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,result.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,result.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, result.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3)); // Assert that the aggregation group hierarchy is left intact - HitGroup root= getFirstGroupIn(result.hits()); + HitGroup root = getFirstGroupIn(result.hits()); assertNotNull(root); - assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently - assertEquals(1,root.size()); - HitGroup groupList= (GroupList)root.get("grouplist:g1"); + assertEquals("group:root:",root.getId().stringValue().substring(0, 11)); // The id ends by a global counter currently + assertEquals(1, root.size()); + HitGroup groupList = (GroupList)root.get("grouplist:g1"); assertNotNull(groupList); - assertEquals(1,groupList.size()); - HitGroup group= (HitGroup)groupList.get("group:long:37"); + assertEquals(1, groupList.size()); + HitGroup group = (HitGroup)groupList.get("group:long:37"); assertNotNull(group); } private Group getFirstGroupIn(HitGroup hits) { - for (Hit h : hits) { + for (Hit h : hits) if (h instanceof Group) return (Group)h; - } return null; } @@ -450,9 +412,8 @@ public class FieldCollapsingSearcherTestCase { private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + for (Searcher current = topOfChain; current != null; current = chained.get(current)) searchers.add(current); - } return new Chain<>(searchers); } @@ -470,7 +431,7 @@ public class FieldCollapsingSearcherTestCase { @Override public Result search(Query query, Execution execution) { - Result r=execution.search(query); + Result r = execution.search(query); r.hits().add(createAggregationGroup("g1")); return r; } @@ -479,10 +440,51 @@ public class FieldCollapsingSearcherTestCase { Group root = new Group(new RootId(0), new Relevance(1)); GroupList groupList = new GroupList(label); root.add(groupList); - Group value=new Group(new LongId(37l),new Relevance(2.11)); + Group value = new Group(new LongId(37l), new Relevance(2.11)); groupList.add(value); return root; } } + private FastHit createHit(String uri,int relevancy,int mid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(mid)); + return hit; + } + + private void assertHit(String uri,int relevancy,int mid,Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + } + + private static class ZeroHitsControl extends com.yahoo.search.Searcher { + + public int queryCount = 0; + + @Override + public Result search(Query query, Execution execution) { + ++queryCount; + if (query.getHits() == 0) { + return new Result(query); + } else { + return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); + } + } + } + + public static class QueryMessupSearcher extends Searcher { + + @Override + public Result search(com.yahoo.search.Query query, Execution execution) { + AndItem a = new AndItem(); + a.addItem(query.getModel().getQueryTree().getRoot()); + a.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(a); + + return execution.search(query); + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index b8db5e4d90f..a4cf7d8c380 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -23,7 +23,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"first third\"", q); + assertSemantics("AND first third", q); } @Test @@ -32,7 +32,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"bc first third fg\"", q); + assertSemantics("AND bc first third fg", q); } @Test @@ -41,7 +41,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("+bc -\"first third\"", q); + assertSemantics("+bc -(AND first third)", q); } @Test @@ -50,7 +50,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q); + assertSemantics("AND 9 2 7 0 bc third 2 3 8 9", q); } private static Item parseQuery(String query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 82a5a0c7a24..e2ac44316e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -73,7 +73,7 @@ public class IndexFactsTestCase { Query q = newQuery("?query=a:b", indexFacts); assertEquals("a:b", q.getModel().getQueryTree().getRoot().toString()); q = newQuery("?query=notarealindex:b", indexFacts); - assertEquals("\"notarealindex b\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND notarealindex b", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -302,8 +302,8 @@ public class IndexFactsTestCase { IndexFacts.Session session2 = indexFacts.newSession(query2.getModel().getSources(), query2.getModel().getRestrict()); assertTrue(session1.getIndex("url").isUriIndex()); assertTrue(session2.getIndex("url").isUriIndex()); - assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString()); - assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query1.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query2.getModel().getQueryTree().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java index dbeced57c52..aa507d38be5 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -5,6 +5,7 @@ import static org.junit.Assert.*; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import org.junit.After; import org.junit.Before; @@ -23,50 +24,50 @@ import com.yahoo.text.Utf8; /** * Functional test for InputCheckingSearcher. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class InputCheckingSearcherTestCase { Execution execution; @Before - public void setUp() throws Exception { + public void setUp() { execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)), - Execution.Context.createContextStub(new IndexFacts())); + Execution.Context.createContextStub(new IndexFacts())); } @After - public void tearDown() throws Exception { + public void tearDown() { execution = null; } @Test - public final void testCommonCase() { + public void testCommonCase() { Result r = execution.search(new Query("/search/?query=three+blind+mice")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButAsciiOnly() { + public void candidateButAsciiOnly() { Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValid() throws UnsupportedEncodingException { + public void candidateButValid() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void testDoubleEncoded() throws UnsupportedEncodingException { + public void testDoubleEncoded() throws UnsupportedEncodingException { String rawQuery = "å å å å å å"; byte[] encodedOnce = Utf8.toBytes(rawQuery); char[] secondEncodingBuffer = new char[encodedOnce.length]; @@ -74,33 +75,42 @@ public class InputCheckingSearcherTestCase { secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); } String query = new String(secondEncodingBuffer); - Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8))); assertEquals(1, r.hits().getErrorHit().errors().size()); } @Test - public final void testRepeatedConsecutiveTermsInPhrase() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedTermsInPhraseIs10() { - Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); + public void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); + r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); } + } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 34c3da395b7..c831ee29631 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -16,6 +16,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; @@ -941,12 +942,12 @@ public class QueryTestCase { @Test public void testImplicitPhraseIsDefault() { Query query = new Query(httpEncode("?query=it's fine")); - assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString()); } @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -961,7 +962,7 @@ public class QueryTestCase { @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -972,6 +973,56 @@ public class QueryTestCase { query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); + // 'it' and 's' should have connectivity 1 + AndItem root = (AndItem)query.getModel().getQueryTree().getRoot(); + AndSegmentItem sand = (AndSegmentItem)root.getItem(0); + WordItem it = (WordItem)sand.getItem(0); + assertEquals("it", it.getWord()); + WordItem s = (WordItem)sand.getItem(1); + assertEquals("s", s.getWord()); + assertEquals(s, it.getConnectedItem()); + assertEquals(1.0, it.getConnectivity(), 0.00000001); + } + + @Test + public void testImplicitAndConnectivity() { + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting false"); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + + { + Query query = new Query(httpEncode("?query=myfield:b.c.d")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem b = (WordItem) root.getItem(0); + WordItem c = (WordItem) root.getItem(1); + WordItem d = (WordItem) root.getItem(2); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + } + + { + Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem a = (WordItem) root.getItem(0); + WordItem b = (WordItem) root.getItem(1); + WordItem c = (WordItem) root.getItem(2); + WordItem d = (WordItem) root.getItem(3); + WordItem e = (WordItem) root.getItem(4); + assertNull(a.getConnectedItem()); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + assertNull(d.getConnectedItem()); + } } @Test |