diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/prelude/query')
7 files changed, 30 insertions, 37 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java index 0dd8e1c36cc..38e2b82ea35 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.query.textualrepresentation.Discloser; * extend AndItem to avoid code using instanceof handling it as an * AndItem. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public abstract class SegmentItem extends CompositeItem implements BlockItem { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 5051108ea9b..da73aab3396 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -134,14 +134,15 @@ public abstract class AbstractParser implements CustomParser { @Override public final Item parse(String queryToParse, String filterToParse, Language parsingLanguage, IndexFacts.Session indexFacts, String defaultIndexName) { - if (queryToParse == null) { - return null; - } + if (queryToParse == null) return null; + + tokenize(queryToParse, defaultIndexName, indexFacts, parsingLanguage); + if (parsingLanguage == null) { parsingLanguage = environment.getLinguistics().getDetector().detect(queryToParse, null).getLanguage(); } setState(parsingLanguage, indexFacts); - tokenize(queryToParse, defaultIndexName, indexFacts); + Item root = parseItems(); if (filterToParse != null) { AnyParser filterParser = new AnyParser(environment); @@ -167,25 +168,19 @@ public abstract class AbstractParser implements CustomParser { * @param defaultIndex The default index to assign. * @param item The item to check. */ - private static void assignDefaultIndex(final String defaultIndex, - final Item item) { - if (defaultIndex == null || item == null) { - return; - } + private static void assignDefaultIndex(final String defaultIndex, Item item) { + if (defaultIndex == null || item == null) return; if (item instanceof IndexedItem) { - final IndexedItem indexName = (IndexedItem) item; + IndexedItem indexName = (IndexedItem) item; - if ("".equals(indexName.getIndexName())) { + if ("".equals(indexName.getIndexName())) indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - final Iterator<Item> items = ((CompositeItem) item) - .getItemIterator(); - while (items.hasNext()) { - final Item i = items.next(); - assignDefaultIndex(defaultIndex, i); - } + } + else if (item instanceof CompositeItem) { + Iterator<Item> items = ((CompositeItem)item).getItemIterator(); + while (items.hasNext()) + assignDefaultIndex(defaultIndex, items.next()); } } @@ -215,10 +210,11 @@ public abstract class AbstractParser implements CustomParser { * @param query the string to tokenize. * @param defaultIndexName the name of the index to use as default. * @param indexFacts resolved information about the index we are searching + * @param language the language set for this query, or null if none */ - protected void tokenize(String query, String defaultIndexName, IndexFacts.Session indexFacts) { + protected void tokenize(String query, String defaultIndexName, IndexFacts.Session indexFacts, Language language) { Tokenizer tokenizer = new Tokenizer(environment.getLinguistics()); - tokenizer.setSubstringSpecialTokens(language.isCjk()); + tokenizer.setSubstringSpecialTokens(language != null && language.isCjk()); tokenizer.setSpecialTokens(environment.getSpecialTokens()); tokens.initialize(tokenizer.tokenize(query, defaultIndexName, indexFacts)); } @@ -265,9 +261,8 @@ public abstract class AbstractParser implements CustomParser { // TODO: The segmenting stuff is a mess now, this will fix it: // - Make Segmenter a class which is instantiated per parsing - // - Make the instance know the language, etc and do all dispatching - // internally - // -JSB + // - Make the instance know the language, etc and do all dispatching internally + // -bratseth // TODO: Use segmenting for forced phrase searches? protected Item segment(Token token) { String normalizedToken = normalize(token.toString()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index e0089fb89ea..95cce001469 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,7 +35,7 @@ public class AnyParser extends SimpleParser { Item filterRoot; setState(queryLanguage, indexFacts); - tokenize(filter, null, indexFacts); + tokenize(filter, null, indexFacts, queryLanguage); filterRoot = anyItems(true); @@ -134,7 +134,7 @@ public class AnyParser extends SimpleParser { Item applyFilter(Item root, String filter, Language queryLanguage, IndexFacts.Session indexFacts) { setState(queryLanguage, indexFacts); - tokenize(filter, null, indexFacts); + tokenize(filter, null, indexFacts, queryLanguage); return filterItems(root); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index ba10b7b6ee1..dfd05ca0da5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -8,7 +8,7 @@ import com.yahoo.search.query.parser.ParserEnvironment; /** * Parser for queries of type phrase. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class PhraseParser extends AbstractParser { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 6117e8e29ed..d1df74fcfa5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -13,7 +13,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; * Base class for parsers of the "simple" query languages (query types * ANY and ALL). * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ abstract class SimpleParser extends StructuredParser { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index eb35655e4ca..fd1617b5350 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -15,7 +15,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.*; * Base class for parsers of the query languages which can be used * for structured queries (types ANY, ALL and ADVANCED). * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ abstract class StructuredParser extends AbstractParser { @@ -430,9 +430,7 @@ abstract class StructuredParser extends AbstractParser { Item item = null; try { - if (item == null) { - item = word(); - } + item = word(); if (item == null && tokens.currentIs(NUMBER)) { Token t = tokens.next(); @@ -542,7 +540,7 @@ abstract class StructuredParser extends AbstractParser { PhraseItem phrase = null; Item firstWord = null; boolean starAfterFirst = false; - boolean starBeforeFirst = false; + boolean starBeforeFirst; if (tokens.skipMultiple(QUOTE)) { quoted = !quoted; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 582395bc738..ed0af8d5060 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -16,7 +16,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.*; /** * Query tokenizer. Singlethreaded. * - * @author bratseth + * @author bratseth */ public final class Tokenizer { @@ -28,7 +28,7 @@ public final class Tokenizer { private SpecialTokens specialTokens = null; /** Whether to recognize tokens also as substrings of other tokens, needed for cjk */ - private boolean substringSpecialTokens=false; + private boolean substringSpecialTokens = false; private final CharacterClasses characterClasses; @@ -53,7 +53,7 @@ public final class Tokenizer { /** Sets whether to recognize tokens also as substrings of other tokens, needed for cjk. Default false. */ public void setSubstringSpecialTokens(boolean substringSpecialTokens) { - this.substringSpecialTokens=substringSpecialTokens; + this.substringSpecialTokens = substringSpecialTokens; } /** |