From 360894c9120b1cb6f89809fedc90f6fc0047b662 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 5 May 2021 18:22:15 +0200 Subject: Revert "Reapply "Bratseth/special tokens"" --- .../yahoo/prelude/query/parser/AbstractParser.java | 3 +- .../prelude/query/parser/SpecialTokenRegistry.java | 137 +++++++++++++++++ .../yahoo/prelude/query/parser/SpecialTokens.java | 167 +++++++++++++++++++++ .../com/yahoo/prelude/query/parser/Tokenizer.java | 34 ++--- .../search/query/parser/ParserEnvironment.java | 4 +- .../com/yahoo/search/searchchain/Execution.java | 6 +- .../yahoo/search/searchchain/ExecutionFactory.java | 2 +- 7 files changed, 326 insertions(+), 27 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java (limited to 'container-search/src/main/java/com') diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 732466748eb..902be7e15dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -19,6 +19,7 @@ import java.util.*; * @author bratseth * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public abstract class AbstractParser implements CustomParser { /** The current submodes of this parser */ @@ -47,7 +48,7 @@ public abstract class AbstractParser implements CustomParser { * of these may be active at the same time. SubModes are activated or * deactivated by specifying special indexes in the query. */ - static final class Submodes { + final class Submodes { /** * Url mode allows "_" and "-" as word characters. Default is false diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java new file mode 100644 index 00000000000..be2d9f9f68b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java @@ -0,0 +1,137 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; + +import java.util.*; +import java.util.logging.Logger; + + +/** + * A registry which is responsible for knowing the current + * set of special tokens. The default registry returns empty token lists + * for all names. Usage of this registry is multithread safe. + * + * @author bratseth + */ +public class SpecialTokenRegistry { + + /** The log of this */ + private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName()); + + private static final SpecialTokens nullSpecialTokens = new SpecialTokens(); + + /** + * The current authorative special token lists, indexed on name. + * These lists are unmodifiable and used directly by clients of this + */ + private Map specialTokenMap = new HashMap<>(); + + private boolean frozen = false; + + /** + * Creates an empty special token registry which + * does not subscribe to any configuration + */ + public SpecialTokenRegistry() {} + + /** + * Create a special token registry which subscribes to the specialtokens + * configuration. Only used for testing. + */ + public SpecialTokenRegistry(String configId) { + try { + build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId)); + } catch (Exception e) { + log.config( + "No special tokens are configured (" + e.getMessage() + ")"); + } + } + + /** + * Create a special token registry from a configuration object. This is the production code path. + */ + public SpecialTokenRegistry(SpecialtokensConfig config) { + if (config != null) { + build(config); + } + freeze(); + } + + private void freeze() { + frozen = true; + } + + private void build(SpecialtokensConfig config) { + List list = new ArrayList<>(); + for (Iterator i = config.tokenlist().iterator(); i.hasNext();) { + Tokenlist tokenList = i.next(); + SpecialTokens tokens = new SpecialTokens(tokenList.name()); + + for (Iterator j = tokenList.tokens().iterator(); j.hasNext();) { + Tokens token = j.next(); + tokens.addSpecialToken(token.token(), token.replace()); + } + tokens.freeze(); + list.add(tokens); + } + addSpecialTokens(list); + } + + /** + * Adds a SpecialTokens instance to the registry. That is, add the + * tokens contained for the name of the SpecialTokens instance + * given. + * + * @param specialTokens the SpecialTokens object to add + */ + public void addSpecialTokens(SpecialTokens specialTokens) { + ensureNotFrozen(); + List list = new ArrayList<>(); + list.add(specialTokens); + addSpecialTokens(list); + + } + + private void ensureNotFrozen() { + if (frozen) { + throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance."); + } + } + + private void addSpecialTokens(List list) { + HashMap tokens = new HashMap<>(specialTokenMap); + for(SpecialTokens t: list) { + tokens.put(t.getName(),t); + } + specialTokenMap = tokens; + } + + + /** + * Returns the currently authorative list of special tokens for + * a given name. + * + * @param name the name of the special tokens to return + * null, the empth string or the string "default" returns + * the default ones + * @return a read-only list of SpecialToken instances, an empty list if this name + * has no special tokens + */ + public SpecialTokens getSpecialTokens(String name) { + if (name == null || name.trim().equals("")) { + name = "default"; + } + SpecialTokens specialTokens = specialTokenMap.get(name); + + if (specialTokens == null) { + return nullSpecialTokens; + } + return specialTokens; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java new file mode 100644 index 00000000000..f45ecefefa6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -0,0 +1,167 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import java.util.logging.Level; +import com.yahoo.prelude.query.Substring; + +import java.util.*; +import java.util.logging.Logger; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * A list of special tokens - string that should be treated as word + * no matter what they contain. Special tokens are case insensitive. + * + * @author bratseth + */ +public class SpecialTokens { + + private static final Logger log = Logger.getLogger(SpecialTokens.class.getName()); + + private final String name; + + private final List specialTokens = new ArrayList<>(); + + private boolean frozen = false; + + private int currentMaximumLength = 0; + + /** Creates a null list of special tokens */ + public SpecialTokens() { + this.name = "(null)"; + } + + public SpecialTokens(String name) { + this.name = name; + } + + /** Returns the name of this special tokens list */ + public String getName() { + return name; + } + + /** + * Adds a special token to this + * + * @param token the special token string to add + * @param replace the token to replace instances of the special token with, or null to keep the token + */ + public void addSpecialToken(String token, String replace) { + ensureNotFrozen(); + if (!caseIndependentLength(token)) { + return; + } + // TODO are special tokens correctly unicode normalized in reagards to query parsing? + final SpecialToken specialTokenToAdd = new SpecialToken(token, replace); + currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length()); + specialTokens.add(specialTokenToAdd); + Collections.sort(specialTokens); + } + + private boolean caseIndependentLength(String token) { + // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... + String asLow = toLowerCase(token); + // TODO put along with the global toLowerCase + String asHigh = token.toUpperCase(Locale.ENGLISH); + if (asLow.length() != token.length() || asHigh.length() != token.length()) { + log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + + " Please report this message in a bug to the Vespa team."); + return false; + } else { + return true; + } + } + + /** + * Returns the special token starting at the start of the given string, or null if no + * special token starts at this string + * + * @param string the string to search for a special token at the start position + * @param substring true to allow the special token to be followed by a character which does not + * mark the end of a token + */ + public SpecialToken tokenize(String string, boolean substring) { + // XXX detonator pattern token.length may be != the length of the + // matching data in string, ref caseIndependentLength(String) + final String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength))); + for (Iterator i = specialTokens.iterator(); i.hasNext();) { + SpecialTokens.SpecialToken special = i.next(); + + if (input.startsWith(special.token())) { + if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) + return special; + } + } + return null; + } + + private boolean tokenEndsAt(int position,String string) { + return !Character.isLetterOrDigit(string.charAt(position)); + } + + /** Returns the number of special tokens in this */ + public int size() { + return specialTokens.size(); + } + + private void ensureNotFrozen() { + if (frozen) { + throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance."); + } + } + + public void freeze() { + frozen = true; + } + + /** An immutable special token */ + public final static class SpecialToken implements Comparable { + + private String token; + + private String replace; + + public SpecialToken(String token, String replace) { + this.token = toLowerCase(token); + if (replace == null || replace.trim().equals("")) { + this.replace = this.token; + } else { + this.replace = toLowerCase(replace); + } + } + + /** Returns the special token */ + public String token() { + return token; + } + + /** Returns the right replace value, never null or an empty string */ + public String replace() { + return replace; + } + + @Override + public int compareTo(SpecialToken other) { + if (this.token().length() < other.token().length()) return 1; + if (this.token().length() == other.token().length()) return 0; + return -1; + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof SpecialToken)) return false; + return Objects.equals(this.token, ((SpecialToken)other).token); + } + + @Override + public int hashCode() { return token.hashCode(); } + + public Token toToken(int start, String rawSource) { + return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe? + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index b71bd57539f..2dc2254df68 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.process.CharacterClasses; -import com.yahoo.language.process.SpecialTokens; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; @@ -201,7 +200,7 @@ public final class Tokenizer { } StringBuilder tmp = new StringBuilder(); for (int i = 0; i < tokencnt; i++) { - Token useToken = tokens.get(backtrack + i); + Token useToken = tokens.get(backtrack+i); tmp.append(useToken.image); } String indexName = tmp.toString(); @@ -217,20 +216,20 @@ public final class Tokenizer { } private int consumeSpecialToken(int start) { - SpecialTokens.Token token = getSpecialToken(start); - if (token == null) return start; - tokens.add(toToken(token, start, source)); - return start + token.token().length(); + SpecialTokens.SpecialToken specialToken=getSpecialToken(start); + if (specialToken==null) return start; + tokens.add(specialToken.toToken(start,source)); + return start + specialToken.token().length(); } - private SpecialTokens.Token getSpecialToken(int start) { + private SpecialTokens.SpecialToken getSpecialToken(int start) { if (specialTokens == null) return null; return specialTokens.tokenize(source.substring(start), substringSpecialTokens); } private int consumeExact(int start,Index index) { if (index.getExactTerminator() == null) return consumeHeuristicExact(start); - return consumeToTerminator(start, index.getExactTerminator()); + return consumeToTerminator(start,index.getExactTerminator()); } private boolean looksLikeExactEnd(int end) { @@ -468,7 +467,7 @@ public final class Tokenizer { /** Consumes a word or number and/or possibly a special token starting within this word or number */ private int consumeWordOrNumber(int start, Index currentIndex) { int tokenEnd = start; - SpecialTokens.Token substringToken = null; + SpecialTokens.SpecialToken substringSpecialToken = null; boolean digitsOnly = true; // int underscores = 0; // boolean underscoresOnly = true; @@ -476,8 +475,8 @@ public final class Tokenizer { while (tokenEnd < source.length()) { if (substringSpecialTokens) { - substringToken = getSpecialToken(tokenEnd); - if (substringToken != null) break; + substringSpecialToken = getSpecialToken(tokenEnd); + if (substringSpecialToken != null) break; } int c = source.codePointAt(tokenEnd); @@ -525,11 +524,11 @@ public final class Tokenizer { } } - if (substringToken == null) + if (substringSpecialToken == null) return --tokenEnd; // TODO: test the logic around tokenEnd with friends - addToken(toToken(substringToken, tokenEnd, source)); - return --tokenEnd + substringToken.token().length(); + addToken(substringSpecialToken.toToken(tokenEnd, source)); + return --tokenEnd + substringSpecialToken.token().length(); } private void addToken(Token.Kind kind, String word, int start, int end) { @@ -540,11 +539,4 @@ public final class Tokenizer { tokens.add(token); } - public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) { - return new Token(Token.Kind.WORD, - specialToken.replacement(), - true, - new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe? - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index df96d314455..94b9bf6ce65 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -4,7 +4,7 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; -import com.yahoo.language.process.SpecialTokens; +import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; @@ -18,7 +18,7 @@ public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); private Linguistics linguistics = new SimpleLinguistics(); - private SpecialTokens specialTokens = SpecialTokens.empty(); + private SpecialTokens specialTokens = new SpecialTokens(); public IndexFacts getIndexFacts() { return indexFacts; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0574fc660c3..84fe88d0292 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; -import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -17,6 +17,8 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; +import java.util.logging.Logger; + /** *

An execution of a search chain. This keeps track of the call state for an execution (in the calling thread) * of the searchers of a search chain.

@@ -109,7 +111,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics) { - owner = null; + owner=null; // The next time something is added here, compose into wrapper objects. Many arguments... // Four methods need to be updated when adding something: diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index a813229c984..31b6d06f78e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; -import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; -- cgit v1.2.3