diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-10-26 11:03:28 +0000 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-10-26 11:03:28 +0000 |
commit | 365972496256236dfd3599d547a167af0d2d4de2 (patch) | |
tree | a9851548e37b765fac222f35e30008e6cc0c82af /container-search/src | |
parent | 57ba28e794416ae7cc6e14660249459dac7df937 (diff) |
Revert "Merge pull request #7409 from vespa-engine/revert-7380-bratseth/segment-yql-by-default"
This reverts commit 1cb09174cb579936eae49f7db632113795b0666a, reversing
changes made to eedcfc980db65da04eaca866693ef215f337f325.
Diffstat (limited to 'container-search/src')
4 files changed, 82 insertions, 305 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java index 4e9d3d11cc5..03e85fa3260 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java @@ -6,13 +6,13 @@ package com.yahoo.prelude.query; * An interface used for anything which may be addressed using an external, * unique ID in the query tree in the backend. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface TaggableItem { - public int getUniqueID(); - public void setUniqueID(int id); - public boolean hasUniqueID(); + int getUniqueID(); + void setUniqueID(int id); + boolean hasUniqueID(); /** * Set the connectivity to another term in the same query tree. @@ -30,9 +30,9 @@ public interface TaggableItem { * @param connectivity a value between 0 (none) and 1 (maximal), defining the connectivity between this and the * argument item. The default connectivity is 0.1. */ - public void setConnectivity(Item item, double connectivity); - public Item getConnectedItem(); - public double getConnectivity(); + void setConnectivity(Item item, double connectivity); + Item getConnectedItem(); + double getConnectivity(); /** @@ -41,8 +41,9 @@ public interface TaggableItem { * This influences ranking features which take term significance into account and overrides the default * partial corpus based term significance computation happening in the backend. */ - public void setSignificance(double significance); - public boolean hasExplicitSignificance(); - public void setExplicitSignificance(boolean significance); - public double getSignificance(); + void setSignificance(double significance); + boolean hasExplicitSignificance(); + void setExplicitSignificance(boolean significance); + double getSignificance(); + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index e0e9042e1a3..e4ae759eec7 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -18,10 +18,8 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; -import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.language.Language; -import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; @@ -117,9 +115,6 @@ public class YqlParser implements Parser { private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it"; private static final String ORIGIN_DESCRIPTION = "string origin for a term"; private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking"; - private static final String SEGMENTER_BACKEND = "backend"; - private static final String SEGMENTER = "segmenter"; - private static final String SEGMENTER_VERSION = "version"; private static final String STEM_DESCRIPTION = "setting for whether to use stem if field implies it"; private static final String USE_POSITION_DATA_DESCRIPTION = "setting for whether to use position data for ranking this item"; private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty"; @@ -190,8 +185,6 @@ public class YqlParser implements Parser { private final Detector detector; private final Set<String> yqlSources = LazySet.newHashSet(); private final Set<String> yqlSummaryFields = LazySet.newHashSet(); - private final String localSegmenterBackend; - private final Version localSegmenterVersion; private Integer hits; private Integer offset; private Integer timeout; @@ -201,10 +194,7 @@ public class YqlParser implements Parser { private IndexNameExpander indexNameExpander = new IndexNameExpander(); private Set<String> docTypes; private Sorting sorting; - private String segmenterBackend; - private Version segmenterVersion; private boolean queryParser = true; - private boolean resegment = false; private final Deque<OperatorNode<?>> annotationStack = new ArrayDeque<>(); private final ParserEnvironment environment; @@ -238,10 +228,6 @@ public class YqlParser implements Parser { segmenter = environment.getLinguistics().getSegmenter(); detector = environment.getLinguistics().getDetector(); this.environment = environment; - - Tuple2<String, Version> version = environment.getLinguistics().getVersion(Linguistics.Component.SEGMENTER); - localSegmenterBackend = version.first; - localSegmenterVersion = version.second; } @NonNull @@ -261,10 +247,7 @@ public class YqlParser implements Parser { currentlyParsing = query; docTypes = null; sorting = null; - segmenterBackend = null; - segmenterVersion = null; // queryParser set prior to calling this - resegment = false; return buildTree(parseYqlProgram()); } @@ -287,32 +270,12 @@ public class YqlParser implements Parser { filterPart.getArguments().length); populateYqlSources(filterPart.<OperatorNode<?>> getArgument(0)); OperatorNode<ExpressionOperator> filterExpression = filterPart.getArgument(1); - populateLinguisticsAnnotations(filterExpression); Item root = convertExpression(filterExpression); connectItems(); userQuery = null; return new QueryTree(root); } - private void populateLinguisticsAnnotations(OperatorNode<ExpressionOperator> filterExpression) { - Map<?, ?> segmenter = getAnnotation(filterExpression, SEGMENTER, - Map.class, null, "segmenter engine and version"); - if (segmenter == null) { - segmenterVersion = null; - segmenterBackend = null; - resegment = false; - } else { - segmenterBackend = getMapValue(SEGMENTER, segmenter, SEGMENTER_BACKEND, String.class); - try { - segmenterVersion = new Version(getMapValue(SEGMENTER, segmenter, SEGMENTER_VERSION, String.class)); - } catch (RuntimeException e) { - segmenterVersion = null; - } - resegment = ! localSegmenterBackend.equals(segmenterBackend) || - ! localSegmenterVersion.equals(segmenterVersion); - } - } - private void populateYqlSources(OperatorNode<?> filterArgs) { yqlSources.clear(); if (filterArgs.getOperator() == SequenceOperator.SCAN) { @@ -614,8 +577,7 @@ public class YqlParser implements Parser { } phrase.setIndexName(field); - if (resegment - && getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION)) { + if (getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION)) { words = segmenter.segment(origin.getValue(), currentlyParsing.getLanguage()); } @@ -719,16 +681,16 @@ public class YqlParser implements Parser { return Language.ENGLISH; } - private String getStringContents(OperatorNode<ExpressionOperator> propertySniffer) { - switch (propertySniffer.getOperator()) { + private String getStringContents(OperatorNode<ExpressionOperator> operator) { + switch (operator.getOperator()) { case LITERAL: - return propertySniffer.getArgument(0, String.class); + return operator.getArgument(0, String.class); case VARREF: Preconditions.checkState(userQuery != null, "properties must be available when trying to fetch user input"); - return userQuery.properties().getString(propertySniffer.getArgument(0, String.class)); + return userQuery.properties().getString(operator.getArgument(0, String.class)); default: - throw newUnexpectedArgumentException(propertySniffer.getOperator(), + throw newUnexpectedArgumentException(operator.getOperator(), ExpressionOperator.LITERAL, ExpressionOperator.VARREF); } } @@ -1310,22 +1272,20 @@ public class YqlParser implements Parser { wordData = normalizer.normalize(wordData); } boolean fromQuery = getAnnotation(ast, IMPLICIT_TRANSFORMS, - Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION); - boolean prefixMatch = getAnnotation(ast, PREFIX, Boolean.class, - Boolean.FALSE, - "setting for whether to use prefix match of input data"); - boolean suffixMatch = getAnnotation(ast, SUFFIX, Boolean.class, - Boolean.FALSE, - "setting for whether to use suffix match of input data"); - boolean substrMatch = getAnnotation(ast, SUBSTRING, Boolean.class, - Boolean.FALSE, - "setting for whether to use substring match of input data"); - Preconditions.checkArgument((prefixMatch ? 1 : 0) - + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, - "Only one of prefix, substring and suffix can be set."); - @NonNull - final TaggableItem wordItem; + Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION); + boolean prefixMatch = getAnnotation(ast, PREFIX, Boolean.class, Boolean.FALSE, + "setting for whether to use prefix match of input data"); + boolean suffixMatch = getAnnotation(ast, SUFFIX, Boolean.class, Boolean.FALSE, + "setting for whether to use suffix match of input data"); + boolean substrMatch = getAnnotation(ast, SUBSTRING, Boolean.class, Boolean.FALSE, + "setting for whether to use substring match of input data"); + String grammar = getAnnotation(ast, USER_INPUT_GRAMMAR, String.class, + Query.Type.ALL.toString(), "grammar for handling word input"); + Preconditions.checkArgument((prefixMatch ? 1 : 0) + + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, + "Only one of prefix, substring and suffix can be set."); + TaggableItem wordItem; if (exactMatch) { wordItem = new ExactStringItem(wordData, fromQuery); } else if (prefixMatch) { @@ -1340,21 +1300,21 @@ public class YqlParser implements Parser { wordItem = new WordItem(wordData, fromQuery); break; case POSSIBLY: - if (shouldResegmentWord(field, fromQuery)) { - wordItem = resegment(field, ast, wordData, fromQuery, parent, language); + if (shouldSegment(field, fromQuery) && ! grammar.equals(USER_INPUT_RAW)) { + wordItem = segment(field, ast, wordData, fromQuery, parent, language); } else { wordItem = new WordItem(wordData, fromQuery); } break; case ALWAYS: - wordItem = resegment(field, ast, wordData, fromQuery, parent, language); + wordItem = segment(field, ast, wordData, fromQuery, parent, language); break; default: throw new IllegalArgumentException("Unexpected segmenting rule: " + segmentPolicy); } } if (wordItem instanceof WordItem) { - prepareWord(field, ast, fromQuery, (WordItem) wordItem); + prepareWord(field, ast, (WordItem) wordItem); } if (language != Language.ENGLISH) // mark the language used, unless it's the default ((Item)wordItem).setLanguage(language); @@ -1362,13 +1322,13 @@ public class YqlParser implements Parser { } @SuppressWarnings({"deprecation"}) - private boolean shouldResegmentWord(String field, boolean fromQuery) { - return resegment && fromQuery && ! indexFactsSession.getIndex(field).isAttribute(); + private boolean shouldSegment(String field, boolean fromQuery) { + return fromQuery && ! indexFactsSession.getIndex(field).isAttribute(); } @NonNull - private TaggableItem resegment(String field, OperatorNode<ExpressionOperator> ast, String wordData, - boolean fromQuery, Class<?> parent, Language language) { + private TaggableItem segment(String field, OperatorNode<ExpressionOperator> ast, String wordData, + boolean fromQuery, Class<?> parent, Language language) { String toSegment = wordData; Substring s = getOrigin(ast); Language usedLanguage = language == null ? currentlyParsing.getLanguage() : language; @@ -1387,7 +1347,7 @@ public class YqlParser implements Parser { ((PhraseSegmentItem) wordItem).setIndexName(field); for (String w : words) { WordItem segment = new WordItem(w, fromQuery); - prepareWord(field, ast, fromQuery, segment); + prepareWord(field, ast, segment); ((PhraseSegmentItem) wordItem).addItem(segment); } ((PhraseSegmentItem) wordItem).lock(); @@ -1404,16 +1364,9 @@ public class YqlParser implements Parser { return parent == EquivItem.class; } - private void prepareWord(String field, OperatorNode<ExpressionOperator> ast, boolean fromQuery, - WordItem wordItem) { + private void prepareWord(String field, OperatorNode<ExpressionOperator> ast, WordItem wordItem) { wordItem.setIndexName(field); wordStyleSettings(ast, wordItem); - if (shouldResegmentWord(field, fromQuery)) { - // force re-stemming, new case normalization, etc - wordItem.setStemmed(false); - wordItem.setLowercased(false); - wordItem.setNormalizable(true); - } } @NonNull @@ -1421,10 +1374,12 @@ public class YqlParser implements Parser { { Map<?, ?> connectivity = getAnnotation(ast, CONNECTIVITY, Map.class, null, "connectivity settings"); if (connectivity != null) { - connectedItems.add(new ConnectedItem(out, getMapValue( - CONNECTIVITY, connectivity, CONNECTION_ID, - Integer.class), getMapValue(CONNECTIVITY, connectivity, - CONNECTION_WEIGHT, Number.class).doubleValue())); + connectedItems.add(new ConnectedItem(out, + getMapValue(CONNECTIVITY, connectivity, CONNECTION_ID, + Integer.class), getMapValue(CONNECTIVITY, + connectivity, + CONNECTION_WEIGHT, + Number.class).doubleValue())); } Number significance = getAnnotation(ast, SIGNIFICANCE, Number.class, null, "term significance"); if (significance != null) { @@ -1713,14 +1668,6 @@ public class YqlParser implements Parser { return new IllegalArgumentException(out.toString()); } - String getSegmenterBackend() { - return segmenterBackend; - } - - Version getSegmenterVersion() { - return segmenterVersion; - } - private static final class ConnectedItem { final double weight; diff --git a/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java deleted file mode 100644 index 1bed8ff2233..00000000000 --- a/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.yql; - -import static org.junit.Assert.assertEquals; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.search.query.parser.Parsable; -import com.yahoo.search.query.parser.ParserEnvironment; - -/** - * Check rules for resegmenting words in YQL+ when segmenter is deemed - * incompatible. The class under testing is {@link YqlParser}. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class ResegmentingTestCase { - private YqlParser parser; - - @Before - public void setUp() throws Exception { - ParserEnvironment env = new ParserEnvironment(); - parser = new YqlParser(env); - } - - @After - public void tearDown() throws Exception { - parser = null; - } - - @Test - public final void testWord() { - assertEquals( - "title:'a b'", - parser.parse( - new Parsable() - .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains \"a b\");")) - .toString()); - } - - @Test - public final void testPhraseSegment() { - assertEquals( - "title:'c d'", - parser.parse( - new Parsable() - .setQuery("select * from sources * where" - + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" - + " phrase(\"a\", \"b\")));")) - .toString()); - } - - @Test - public final void testPhraseInEquiv() { - assertEquals( - "EQUIV title:a title:'c d'", - parser.parse( - new Parsable() - .setQuery("select * from sources * where" - + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains" - + " equiv(\"a\"," - + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]\"b\")" - + ")" - + ");")) - .toString()); - } - - @Test - public final void testPhraseSegmentToAndSegment() { - assertEquals( - "SAND title:c title:d", - parser.parse( - new Parsable() - .setQuery("select * from sources * where" - + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true}]" - + " phrase(\"a\", \"b\")));")) - .toString()); - } - - @Test - public final void testPhraseSegmentInPhrase() { - assertEquals( - "title:\"a 'c d'\"", - parser.parse( - new Parsable() - .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains phrase(\"a\"," - + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" - + " phrase(\"e\", \"f\"))));")) - .toString()); - } - - @Test - public final void testWordNoImplicitTransforms() { - assertEquals( - "title:a b", - parser.parse( - new Parsable() - .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains ([{\"implicitTransforms\": false}]\"a b\"));")) - .toString()); - } - - @Test - public final void testPhraseSegmentNoImplicitTransforms() { - assertEquals( - "title:'a b'", - parser.parse( - new Parsable() - .setQuery("select * from sources * where" - + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" - + " phrase(\"a\", \"b\")));")) - .toString()); - } - - @Test - public final void testPhraseSegmentToAndSegmentNoImplicitTransforms() { - assertEquals( - "SAND title:a title:b", - parser.parse( - new Parsable() - .setQuery("select * from sources * where" - + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true, \"implicitTransforms\": false}]" - + " phrase(\"a\", \"b\")));")) - .toString()); - } - - @Test - public final void testPhraseSegmentInPhraseNoImplicitTransforms() { - assertEquals( - "title:\"a 'e f'\"", - parser.parse( - new Parsable() - .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" - + " (title contains phrase(\"a\"," - + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" - + " phrase(\"e\", \"f\"))));")) - .toString()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index e2325e52f63..127820bb7ae 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -11,6 +11,7 @@ import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.RegExpItem; @@ -55,15 +56,15 @@ import static org.junit.Assert.fail; /** * Specification for the conversion of YQL+ expressions to Vespa search queries. * - * @author steinar - * @author stiankri + * @author Steinar Knutsen + * @author Stian Kristoffersen */ public class YqlParserTestCase { private final YqlParser parser = new YqlParser(new ParserEnvironment()); @Test - public void requireThatDefaultsAreSane() { + public void testParserDefaults() { assertTrue(parser.isQueryParser()); assertNull(parser.getDocTypes()); } @@ -76,7 +77,7 @@ public class YqlParserTestCase { } @Test - public void requireThatGroupingStepCanBeParsed() { + public void testGroupingStep() { assertParse("select foo from bar where baz contains 'cox';", "baz:cox"); assertEquals("[]", @@ -98,7 +99,7 @@ public class YqlParserTestCase { } @Test - public void requireThatGroupingContinuationCanBeParsed() { + public void testGroupingContinuation() { assertParse("select foo from bar where baz contains 'cox' " + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())));", "baz:cox"); @@ -320,12 +321,14 @@ public class YqlParserTestCase { @Test public void testRaw() { + // Default: Not raw, for comparison Item root = parse("select foo from bar where baz contains (\"yoni jo dima\");").getRoot(); - assertTrue(root instanceof WordItem); - assertFalse(root instanceof ExactStringItem); - assertEquals("yoni jo dima", ((WordItem)root).getWord()); + assertEquals("baz:'yoni jo dima'", root.toString()); + assertFalse(root instanceof WordItem); + assertTrue(root instanceof PhraseSegmentItem); root = parse("select foo from bar where baz contains ([{\"grammar\":\"raw\"}]\"yoni jo dima\");").getRoot(); + assertEquals("baz:yoni jo dima", root.toString()); assertTrue(root instanceof WordItem); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem)root).getWord()); @@ -735,44 +738,17 @@ public class YqlParserTestCase { @Test public void testSegmenting() { - assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + - "\"yell\"}}] title contains \"madonna\");", - "title:madonna"); - assertEquals("yell", parser.getSegmenterBackend()); - assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); - - assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"8.7.3\", \"backend\": " + - "\"yell\"}}]([{\"targetNumHits\": 9999438}] weakAnd(format contains \"online\", title contains " + - "\"madonna\")));", - "WAND(9999438) format:online title:madonna"); - assertEquals("yell", parser.getSegmenterBackend()); - assertEquals(new Version("8.7.3"), parser.getSegmenterVersion()); - - assertParse("select * from bar where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": " + - "\"yell\"}}] ([{\"targetNumHits\": 99909438}] weakAnd(format contains \"online\", title contains " + - "\"madonna\"));", - "WAND(99909438) format:online title:madonna"); - assertEquals("yell", parser.getSegmenterBackend()); - assertEquals(new Version("18.47.39"), parser.getSegmenterVersion()); - - assertParse("select * from bar where [{\"targetNumHits\": 99909438}] weakAnd(format contains " + - "\"online\", title contains \"madonna\");", - "WAND(99909438) format:online title:madonna"); - assertNull(parser.getSegmenterBackend()); - assertNull(parser.getSegmenterVersion()); - - assertParse("select * from bar where [{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + - "\"yell\"}}](title contains \"madonna\") order by shoesize;", - "title:madonna"); - assertEquals("yell", parser.getSegmenterBackend()); - assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); + assertParse("select * from bar where title contains 'foo.bar';", + "title:'foo bar'"); + + assertParse("select * from bar where title contains 'foo&123';", + "title:'foo 123'"); } @Test public void testNegativeHitLimit() { - assertParse( - "select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", - "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", + "foo:[0;1;-38]"); } @Test @@ -830,26 +806,26 @@ public class YqlParserTestCase { @Test public void testMoreInheritedAnnotations() { - final String yqlQuery = "select * from sources * where " - + "([{\"ranked\": false}](foo contains \"a\" " - + "and ([{\"ranked\": true}](bar contains \"b\" " - + "or ([{\"ranked\": false}](foo contains \"c\" " - + "and foo contains ([{\"ranked\": true}]\"d\")))))));"; + String yqlQuery = "select * from sources * where " + + "([{\"ranked\": false}](foo contains \"a\" " + + "and ([{\"ranked\": true}](bar contains \"b\" " + + "or ([{\"ranked\": false}](foo contains \"c\" " + + "and foo contains ([{\"ranked\": true}]\"d\")))))));"; QueryTree x = parse(yqlQuery); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(4, terms.size()); for (IndexedItem term : terms) { switch (term.getIndexedString()) { - case "a": - case "c": - assertFalse(((Item) term).isRanked()); - break; - case "b": - case "d": - assertTrue(((Item) term).isRanked()); - break; - default: - fail(); + case "a": + case "c": + assertFalse(((Item) term).isRanked()); + break; + case "b": + case "d": + assertTrue(((Item) term).isRanked()); + break; + default: + fail(); } } } @@ -921,8 +897,8 @@ public class YqlParserTestCase { private void checkWordAlternativesContent(WordAlternativesItem alternatives) { boolean seenTree = false; boolean seenForest = false; - final String forest = "trees"; - final String tree = "tree"; + String forest = "trees"; + String tree = "tree"; assertEquals(2, alternatives.getAlternatives().size()); for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) { if (tree.equals(alternative.word)) { |