From 6a378ba13339c2de59b82b9364354658b088b09c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 5 May 2021 11:46:21 +0200 Subject: Revert "Revert "Revert "Bratseth/special tokens""" --- .../java/com/yahoo/vespa/model/HostResource.java | 1 - .../model/container/docproc/DocprocChain.java | 5 +- .../vespa/model/search/IndexingDocprocChain.java | 2 + container-search/abi-spec.json | 10 +- .../yahoo/prelude/query/parser/AbstractParser.java | 3 +- .../prelude/query/parser/SpecialTokenRegistry.java | 137 +++++++++++++++++ .../yahoo/prelude/query/parser/SpecialTokens.java | 167 +++++++++++++++++++++ .../com/yahoo/prelude/query/parser/Tokenizer.java | 34 ++--- .../search/query/parser/ParserEnvironment.java | 4 +- .../com/yahoo/search/searchchain/Execution.java | 6 +- .../yahoo/search/searchchain/ExecutionFactory.java | 2 +- .../prelude/query/parser/test/ParseTestCase.java | 6 +- .../prelude/query/parser/test/ParsingTester.java | 26 ++-- .../query/parser/test/TokenizerTestCase.java | 106 +++++++------ .../prelude/query/parser/test/replacingtokens.cfg | 12 ++ .../query/rewrite/RewriterFeaturesTestCase.java | 2 +- linguistics/abi-spec.json | 51 ------- .../yahoo/language/opennlp/OpenNlpTokenizer.java | 9 +- .../language/process/SpecialTokenRegistry.java | 72 --------- .../com/yahoo/language/process/SpecialTokens.java | 141 ----------------- .../java/com/yahoo/language/process/TokenType.java | 2 +- .../yahoo/language/simple/SimpleLinguistics.java | 7 +- .../com/yahoo/language/simple/SimpleTokenizer.java | 27 ++-- .../language/process/SpecialTokensTestCase.java | 40 ----- 24 files changed, 438 insertions(+), 434 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg delete mode 100644 linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java delete mode 100644 linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java delete mode 100644 linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 3bc07db9507..78d9dd473b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java index 2b2b17c76c3..ee246b5e485 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java @@ -23,9 +23,8 @@ public class DocprocChain extends Chain { } /** - * The field name schema map that applies to this whole chain. - * - * @return doctype, from → to + * The field name schema map that applies to this whole chain + * @return doctype,from → to */ public Map,String> fieldNameSchemaMap() { return fieldNameSchemaMap; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java index 8fe6b51f2b4..2c1d979e2c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.model.search; import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.Phase; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import com.yahoo.vespa.model.container.docproc.DocprocChain; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 74ed9d33f04..b5933936adf 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5574,8 +5574,8 @@ "public com.yahoo.search.query.parser.ParserEnvironment setIndexFacts(com.yahoo.prelude.IndexFacts)", "public com.yahoo.language.Linguistics getLinguistics()", "public com.yahoo.search.query.parser.ParserEnvironment setLinguistics(com.yahoo.language.Linguistics)", - "public com.yahoo.language.process.SpecialTokens getSpecialTokens()", - "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.language.process.SpecialTokens)", + "public com.yahoo.prelude.query.parser.SpecialTokens getSpecialTokens()", + "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.prelude.query.parser.SpecialTokens)", "public static com.yahoo.search.query.parser.ParserEnvironment fromExecutionContext(com.yahoo.search.searchchain.Execution$Context)", "public static com.yahoo.search.query.parser.ParserEnvironment fromParserEnvironment(com.yahoo.search.query.parser.ParserEnvironment)" ], @@ -7765,7 +7765,7 @@ "final" ], "methods": [ - "public void (com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)", + "public void (com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.prelude.query.parser.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub()", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)", @@ -7779,8 +7779,8 @@ "public void setIndexFacts(com.yahoo.prelude.IndexFacts)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()", - "public com.yahoo.language.process.SpecialTokenRegistry getTokenRegistry()", - "public void setTokenRegistry(com.yahoo.language.process.SpecialTokenRegistry)", + "public com.yahoo.prelude.query.parser.SpecialTokenRegistry getTokenRegistry()", + "public void setTokenRegistry(com.yahoo.prelude.query.parser.SpecialTokenRegistry)", "public void setDetailedDiagnostics(boolean)", "public boolean getDetailedDiagnostics()", "public boolean getBreakdown()", diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 732466748eb..902be7e15dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -19,6 +19,7 @@ import java.util.*; * @author bratseth * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public abstract class AbstractParser implements CustomParser { /** The current submodes of this parser */ @@ -47,7 +48,7 @@ public abstract class AbstractParser implements CustomParser { * of these may be active at the same time. SubModes are activated or * deactivated by specifying special indexes in the query. */ - static final class Submodes { + final class Submodes { /** * Url mode allows "_" and "-" as word characters. Default is false diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java new file mode 100644 index 00000000000..be2d9f9f68b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java @@ -0,0 +1,137 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; + +import java.util.*; +import java.util.logging.Logger; + + +/** + * A registry which is responsible for knowing the current + * set of special tokens. The default registry returns empty token lists + * for all names. Usage of this registry is multithread safe. + * + * @author bratseth + */ +public class SpecialTokenRegistry { + + /** The log of this */ + private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName()); + + private static final SpecialTokens nullSpecialTokens = new SpecialTokens(); + + /** + * The current authorative special token lists, indexed on name. + * These lists are unmodifiable and used directly by clients of this + */ + private Map specialTokenMap = new HashMap<>(); + + private boolean frozen = false; + + /** + * Creates an empty special token registry which + * does not subscribe to any configuration + */ + public SpecialTokenRegistry() {} + + /** + * Create a special token registry which subscribes to the specialtokens + * configuration. Only used for testing. + */ + public SpecialTokenRegistry(String configId) { + try { + build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId)); + } catch (Exception e) { + log.config( + "No special tokens are configured (" + e.getMessage() + ")"); + } + } + + /** + * Create a special token registry from a configuration object. This is the production code path. + */ + public SpecialTokenRegistry(SpecialtokensConfig config) { + if (config != null) { + build(config); + } + freeze(); + } + + private void freeze() { + frozen = true; + } + + private void build(SpecialtokensConfig config) { + List list = new ArrayList<>(); + for (Iterator i = config.tokenlist().iterator(); i.hasNext();) { + Tokenlist tokenList = i.next(); + SpecialTokens tokens = new SpecialTokens(tokenList.name()); + + for (Iterator j = tokenList.tokens().iterator(); j.hasNext();) { + Tokens token = j.next(); + tokens.addSpecialToken(token.token(), token.replace()); + } + tokens.freeze(); + list.add(tokens); + } + addSpecialTokens(list); + } + + /** + * Adds a SpecialTokens instance to the registry. That is, add the + * tokens contained for the name of the SpecialTokens instance + * given. + * + * @param specialTokens the SpecialTokens object to add + */ + public void addSpecialTokens(SpecialTokens specialTokens) { + ensureNotFrozen(); + List list = new ArrayList<>(); + list.add(specialTokens); + addSpecialTokens(list); + + } + + private void ensureNotFrozen() { + if (frozen) { + throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance."); + } + } + + private void addSpecialTokens(List list) { + HashMap tokens = new HashMap<>(specialTokenMap); + for(SpecialTokens t: list) { + tokens.put(t.getName(),t); + } + specialTokenMap = tokens; + } + + + /** + * Returns the currently authorative list of special tokens for + * a given name. + * + * @param name the name of the special tokens to return + * null, the empth string or the string "default" returns + * the default ones + * @return a read-only list of SpecialToken instances, an empty list if this name + * has no special tokens + */ + public SpecialTokens getSpecialTokens(String name) { + if (name == null || name.trim().equals("")) { + name = "default"; + } + SpecialTokens specialTokens = specialTokenMap.get(name); + + if (specialTokens == null) { + return nullSpecialTokens; + } + return specialTokens; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java new file mode 100644 index 00000000000..f45ecefefa6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -0,0 +1,167 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import java.util.logging.Level; +import com.yahoo.prelude.query.Substring; + +import java.util.*; +import java.util.logging.Logger; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * A list of special tokens - string that should be treated as word + * no matter what they contain. Special tokens are case insensitive. + * + * @author bratseth + */ +public class SpecialTokens { + + private static final Logger log = Logger.getLogger(SpecialTokens.class.getName()); + + private final String name; + + private final List specialTokens = new ArrayList<>(); + + private boolean frozen = false; + + private int currentMaximumLength = 0; + + /** Creates a null list of special tokens */ + public SpecialTokens() { + this.name = "(null)"; + } + + public SpecialTokens(String name) { + this.name = name; + } + + /** Returns the name of this special tokens list */ + public String getName() { + return name; + } + + /** + * Adds a special token to this + * + * @param token the special token string to add + * @param replace the token to replace instances of the special token with, or null to keep the token + */ + public void addSpecialToken(String token, String replace) { + ensureNotFrozen(); + if (!caseIndependentLength(token)) { + return; + } + // TODO are special tokens correctly unicode normalized in reagards to query parsing? + final SpecialToken specialTokenToAdd = new SpecialToken(token, replace); + currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length()); + specialTokens.add(specialTokenToAdd); + Collections.sort(specialTokens); + } + + private boolean caseIndependentLength(String token) { + // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... + String asLow = toLowerCase(token); + // TODO put along with the global toLowerCase + String asHigh = token.toUpperCase(Locale.ENGLISH); + if (asLow.length() != token.length() || asHigh.length() != token.length()) { + log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + + " Please report this message in a bug to the Vespa team."); + return false; + } else { + return true; + } + } + + /** + * Returns the special token starting at the start of the given string, or null if no + * special token starts at this string + * + * @param string the string to search for a special token at the start position + * @param substring true to allow the special token to be followed by a character which does not + * mark the end of a token + */ + public SpecialToken tokenize(String string, boolean substring) { + // XXX detonator pattern token.length may be != the length of the + // matching data in string, ref caseIndependentLength(String) + final String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength))); + for (Iterator i = specialTokens.iterator(); i.hasNext();) { + SpecialTokens.SpecialToken special = i.next(); + + if (input.startsWith(special.token())) { + if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) + return special; + } + } + return null; + } + + private boolean tokenEndsAt(int position,String string) { + return !Character.isLetterOrDigit(string.charAt(position)); + } + + /** Returns the number of special tokens in this */ + public int size() { + return specialTokens.size(); + } + + private void ensureNotFrozen() { + if (frozen) { + throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance."); + } + } + + public void freeze() { + frozen = true; + } + + /** An immutable special token */ + public final static class SpecialToken implements Comparable { + + private String token; + + private String replace; + + public SpecialToken(String token, String replace) { + this.token = toLowerCase(token); + if (replace == null || replace.trim().equals("")) { + this.replace = this.token; + } else { + this.replace = toLowerCase(replace); + } + } + + /** Returns the special token */ + public String token() { + return token; + } + + /** Returns the right replace value, never null or an empty string */ + public String replace() { + return replace; + } + + @Override + public int compareTo(SpecialToken other) { + if (this.token().length() < other.token().length()) return 1; + if (this.token().length() == other.token().length()) return 0; + return -1; + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof SpecialToken)) return false; + return Objects.equals(this.token, ((SpecialToken)other).token); + } + + @Override + public int hashCode() { return token.hashCode(); } + + public Token toToken(int start, String rawSource) { + return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe? + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index b71bd57539f..2dc2254df68 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.process.CharacterClasses; -import com.yahoo.language.process.SpecialTokens; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; @@ -201,7 +200,7 @@ public final class Tokenizer { } StringBuilder tmp = new StringBuilder(); for (int i = 0; i < tokencnt; i++) { - Token useToken = tokens.get(backtrack + i); + Token useToken = tokens.get(backtrack+i); tmp.append(useToken.image); } String indexName = tmp.toString(); @@ -217,20 +216,20 @@ public final class Tokenizer { } private int consumeSpecialToken(int start) { - SpecialTokens.Token token = getSpecialToken(start); - if (token == null) return start; - tokens.add(toToken(token, start, source)); - return start + token.token().length(); + SpecialTokens.SpecialToken specialToken=getSpecialToken(start); + if (specialToken==null) return start; + tokens.add(specialToken.toToken(start,source)); + return start + specialToken.token().length(); } - private SpecialTokens.Token getSpecialToken(int start) { + private SpecialTokens.SpecialToken getSpecialToken(int start) { if (specialTokens == null) return null; return specialTokens.tokenize(source.substring(start), substringSpecialTokens); } private int consumeExact(int start,Index index) { if (index.getExactTerminator() == null) return consumeHeuristicExact(start); - return consumeToTerminator(start, index.getExactTerminator()); + return consumeToTerminator(start,index.getExactTerminator()); } private boolean looksLikeExactEnd(int end) { @@ -468,7 +467,7 @@ public final class Tokenizer { /** Consumes a word or number and/or possibly a special token starting within this word or number */ private int consumeWordOrNumber(int start, Index currentIndex) { int tokenEnd = start; - SpecialTokens.Token substringToken = null; + SpecialTokens.SpecialToken substringSpecialToken = null; boolean digitsOnly = true; // int underscores = 0; // boolean underscoresOnly = true; @@ -476,8 +475,8 @@ public final class Tokenizer { while (tokenEnd < source.length()) { if (substringSpecialTokens) { - substringToken = getSpecialToken(tokenEnd); - if (substringToken != null) break; + substringSpecialToken = getSpecialToken(tokenEnd); + if (substringSpecialToken != null) break; } int c = source.codePointAt(tokenEnd); @@ -525,11 +524,11 @@ public final class Tokenizer { } } - if (substringToken == null) + if (substringSpecialToken == null) return --tokenEnd; // TODO: test the logic around tokenEnd with friends - addToken(toToken(substringToken, tokenEnd, source)); - return --tokenEnd + substringToken.token().length(); + addToken(substringSpecialToken.toToken(tokenEnd, source)); + return --tokenEnd + substringSpecialToken.token().length(); } private void addToken(Token.Kind kind, String word, int start, int end) { @@ -540,11 +539,4 @@ public final class Tokenizer { tokens.add(token); } - public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) { - return new Token(Token.Kind.WORD, - specialToken.replacement(), - true, - new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe? - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index df96d314455..94b9bf6ce65 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -4,7 +4,7 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; -import com.yahoo.language.process.SpecialTokens; +import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; @@ -18,7 +18,7 @@ public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); private Linguistics linguistics = new SimpleLinguistics(); - private SpecialTokens specialTokens = SpecialTokens.empty(); + private SpecialTokens specialTokens = new SpecialTokens(); public IndexFacts getIndexFacts() { return indexFacts; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0574fc660c3..84fe88d0292 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; -import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -17,6 +17,8 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; +import java.util.logging.Logger; + /** *

An execution of a search chain. This keeps track of the call state for an execution (in the calling thread) * of the searchers of a search chain.

@@ -109,7 +111,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics) { - owner = null; + owner=null; // The next time something is added here, compose into wrapper objects. Many arguments... // Four methods need to be updated when adding something: diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index a813229c984..31b6d06f78e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; -import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index cef8ae1751c..6afea895f3a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -18,14 +18,16 @@ import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.TaggableItem; import com.yahoo.prelude.query.WordItem; -import com.yahoo.language.process.SpecialTokens; +import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.prelude.query.parser.TestLinguistics; import com.yahoo.search.Query; import org.junit.Test; +import java.util.Collections; import java.util.Iterator; import static org.junit.Assert.assertEquals; @@ -1637,7 +1639,7 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { - ParsingTester customTester = new ParsingTester(SpecialTokens.empty()); + ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java index fd7e4cbe0e6..17155fff5de 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -11,8 +11,8 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NullItem; -import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.language.process.SpecialTokens; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.search.Query; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.query.parser.Parsable; @@ -20,9 +20,6 @@ import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.query.parser.ParserFactory; -import java.util.ArrayList; -import java.util.List; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -35,7 +32,7 @@ import static org.junit.Assert.assertTrue; public class ParsingTester { private static final Linguistics linguistics = new SimpleLinguistics(); - private final IndexFacts indexFacts; + private IndexFacts indexFacts; private SpecialTokenRegistry tokenRegistry; public ParsingTester() { @@ -52,10 +49,11 @@ public class ParsingTester { public ParsingTester(IndexFacts indexFacts, SpecialTokens specialTokens) { indexFacts.freeze(); + specialTokens.freeze(); this.indexFacts = indexFacts; tokenRegistry = new SpecialTokenRegistry(); - tokenRegistry = new SpecialTokenRegistry(List.of(specialTokens)); + tokenRegistry.addSpecialTokens(specialTokens); } /** @@ -74,13 +72,13 @@ public class ParsingTester { * This can be used to add new tokens and passing the resulting special tokens to the constructor of this. */ public static SpecialTokens createSpecialTokens() { - List tokens = new ArrayList<>(); - tokens.add(new SpecialTokens.Token("c++")); - tokens.add(new SpecialTokens.Token(".net", "dotnet")); - tokens.add(new SpecialTokens.Token("tcp/ip")); - tokens.add(new SpecialTokens.Token("c#")); - tokens.add(new SpecialTokens.Token("special-token-fs","firstsecond")); - return new SpecialTokens("default", tokens); + SpecialTokens tokens = new SpecialTokens("default"); + tokens.addSpecialToken("c++", null); + tokens.addSpecialToken(".net", "dotnet"); + tokens.addSpecialToken("tcp/ip", null); + tokens.addSpecialToken("c#", null); + tokens.addSpecialToken("special-token-fs","firstsecond"); + return tokens; } /** diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index e10fbd71c72..aa2e9dbcf75 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -6,13 +6,12 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.language.process.SpecialTokens; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.prelude.query.parser.Token; import com.yahoo.prelude.query.parser.Tokenizer; import org.junit.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -40,11 +39,13 @@ import static org.junit.Assert.assertTrue; */ public class TokenizerTestCase { + private SpecialTokenRegistry defaultRegistry = new SpecialTokenRegistry("file:src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg"); + @Test public void testPlainTokenization() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a"); assertEquals(new Token(WORD, "drive"), tokens.get(0)); @@ -86,7 +87,7 @@ public class TokenizerTestCase { public void testOneSpecialToken() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("c++ lovers, please apply"); assertEquals(new Token(WORD, "c++"), tokens.get(0)); @@ -96,7 +97,7 @@ public class TokenizerTestCase { public void testSpecialTokenCombination() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("c#, c++ or .net know, not tcp/ip"); assertEquals(new Token(WORD, "c#"), tokens.get(0)); @@ -122,9 +123,10 @@ public class TokenizerTestCase { */ @Test public void testSpecialTokenCJK() { + assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); tokenizer.setSubstringSpecialTokens(true); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing")); + tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); List tokens = tokenizer.tokenize("fooc#bar,c++with spacebarknowknowknow,knowknownot know"); assertEquals(new Token(WORD, "foo"), tokens.get(0)); @@ -149,7 +151,7 @@ public class TokenizerTestCase { public void testSpecialTokenCaseInsensitive() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("The AS/400 is great"); assertEquals(new Token(WORD, "The"), tokens.get(0)); @@ -165,7 +167,7 @@ public class TokenizerTestCase { public void testSpecialTokenNonMatch() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("c++ c+ aS/400 i/o .net i/ooo ap.net"); assertEquals(new Token(WORD, "c++"), tokens.get(0)); @@ -188,9 +190,18 @@ public class TokenizerTestCase { @Test public void testSpecialTokenConfigurationDefault() { + String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + assertEquals("Special tokens configured", 6, + r.getSpecialTokens("default").size()); + assertEquals("Special tokens configured", 4, + r.getSpecialTokens("other").size()); + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens( + r.getSpecialTokens("default")); List tokens = tokenizer.tokenize( "with space, c++ or .... know, not b.s.d."); @@ -213,9 +224,18 @@ public class TokenizerTestCase { @Test public void testSpecialTokenConfigurationOther() { + String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + assertEquals("Special tokens configured", 6, + r.getSpecialTokens("default").size()); + assertEquals("Special tokens configured", 4, + r.getSpecialTokens("other").size()); + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("other")); + tokenizer.setSpecialTokens( + r.getSpecialTokens("other")); List tokens = tokenizer.tokenize( "with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d."); @@ -246,10 +266,27 @@ public class TokenizerTestCase { assertTrue(((Token) tokens.get(10)).isSpecial()); } + @Test + public void testSpecialTokenConfigurationMissing() { + String tokenFile = "file:source/bogus/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(r.getSpecialTokens("other")); + List tokens = tokenizer.tokenize("c++"); + + assertEquals(new Token(WORD, "c"), tokens.get(0)); + assertEquals(new Token(PLUS, "+"), tokens.get(1)); + assertEquals(new Token(PLUS, "+"), tokens.get(2)); + } + @Test public void testTokenReplacing() { + assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing")); + tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); List tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d."); assertEquals(new Token(WORD, "with-space"), tokens.get(0)); @@ -708,7 +745,7 @@ public class TokenizerTestCase { public void testSingleQuoteAsWordCharacter() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens()); List tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'"); assertEquals(new Token(WORD, "drive"), tokens.get(0)); @@ -744,38 +781,17 @@ public class TokenizerTestCase { assertEquals(new Token(WORD, "a'"), tokens.get(30)); } - private SpecialTokenRegistry createSpecialTokens() { - List tokens = new ArrayList<>(); - tokens.add(new SpecialTokens.Token("c+")); - tokens.add(new SpecialTokens.Token("c++")); - tokens.add(new SpecialTokens.Token(".net")); - tokens.add(new SpecialTokens.Token("tcp/ip")); - tokens.add(new SpecialTokens.Token("i/o")); - tokens.add(new SpecialTokens.Token("c#")); - tokens.add(new SpecialTokens.Token("AS/400")); - tokens.add(new SpecialTokens.Token("....")); - tokens.add(new SpecialTokens.Token("b.s.d.")); - tokens.add(new SpecialTokens.Token("with space")); - tokens.add(new SpecialTokens.Token("dvd\\xB1r")); - SpecialTokens defaultTokens = new SpecialTokens("default", tokens); - - tokens = new ArrayList<>(); - tokens.add(new SpecialTokens.Token("[huh]")); - tokens.add(new SpecialTokens.Token("&&&%%%")); - tokens.add(new SpecialTokens.Token("------")); - tokens.add(new SpecialTokens.Token("!!!***")); - SpecialTokens otherTokens = new SpecialTokens("other", tokens); - - tokens = new ArrayList<>(); - tokens.add(new SpecialTokens.Token("....")); - tokens.add(new SpecialTokens.Token("c++", "cpp")); - tokens.add(new SpecialTokens.Token("b.s.d.")); - tokens.add(new SpecialTokens.Token("with space", "with-space")); - tokens.add(new SpecialTokens.Token("c#")); - tokens.add(new SpecialTokens.Token("know", "knuwww")); - SpecialTokens replacingTokens = new SpecialTokens("replacing", tokens); - - return new SpecialTokenRegistry(List.of(defaultTokens, otherTokens, replacingTokens)); + private SpecialTokens createSpecialTokens() { + SpecialTokens tokens = new SpecialTokens("default"); + + tokens.addSpecialToken("c+", null); + tokens.addSpecialToken("c++", null); + tokens.addSpecialToken(".net", null); + tokens.addSpecialToken("tcp/ip", null); + tokens.addSpecialToken("i/o", null); + tokens.addSpecialToken("c#", null); + tokens.addSpecialToken("AS/400", null); + return tokens; } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg new file mode 100644 index 00000000000..6a189de0164 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg @@ -0,0 +1,12 @@ +tokenlist[1] +tokenlist[0].name default +tokenlist[0].tokens[6] +tokenlist[0].tokens[0].token .... +tokenlist[0].tokens[1].token c++ +tokenlist[0].tokens[1].replace cpp +tokenlist[0].tokens[2].token b.s.d. +tokenlist[0].tokens[3].token with space +tokenlist[0].tokens[3].replace with-space +tokenlist[0].tokens[4].token c# +tokenlist[0].tokens[5].token know +tokenlist[0].tokens[5].replace knuwww diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java index 08146bbe069..5508c2a73a7 100644 --- a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java @@ -8,7 +8,7 @@ import org.junit.Test; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Item; -import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.search.Query; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.Execution.Context; diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index b77b03664d4..58b838d7332 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -427,57 +427,6 @@ ], "fields": [] }, - "com.yahoo.language.process.SpecialTokenRegistry": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void ()", - "public void (com.yahoo.vespa.configdefinition.SpecialtokensConfig)", - "public void (java.util.List)", - "public com.yahoo.language.process.SpecialTokens getSpecialTokens(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.language.process.SpecialTokens$Token": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void (java.lang.String)", - "public void (java.lang.String, java.lang.String)", - "public java.lang.String token()", - "public java.lang.String replacement()", - "public int compareTo(com.yahoo.language.process.SpecialTokens$Token)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.language.process.SpecialTokens": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void (java.lang.String, java.util.List)", - "public java.lang.String name()", - "public java.util.Map asMap()", - "public com.yahoo.language.process.SpecialTokens$Token tokenize(java.lang.String, boolean)", - "public static com.yahoo.language.process.SpecialTokens empty()" - ], - "fields": [] - }, "com.yahoo.language.process.StemList": { "superClass": "java.util.AbstractList", "interfaces": [], diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 73518876c3f..e1185cb2457 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -4,7 +4,6 @@ package com.yahoo.language.opennlp; import com.yahoo.language.Language; import com.yahoo.language.LinguisticsCase; import com.yahoo.language.process.Normalizer; -import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import com.yahoo.language.process.TokenType; @@ -33,21 +32,15 @@ public class OpenNlpTokenizer implements Tokenizer { private final Normalizer normalizer; private final Transformer transformer; private final SimpleTokenizer simpleTokenizer; - private final SpecialTokenRegistry specialTokenRegistry; public OpenNlpTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); } public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { - this(normalizer, transformer, new SpecialTokenRegistry(List.of())); - } - - public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; - this.specialTokenRegistry = specialTokenRegistry; - this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); + simpleTokenizer = new SimpleTokenizer(normalizer, transformer); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java deleted file mode 100644 index b6335d67967..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics - * and be tokenized into a single token. - * - * @author bratseth - */ -public class SpecialTokenRegistry { - - /** - * The current special token lists, indexed on name. - * These lists are unmodifiable and used directly by clients of this - */ - private final Map specialTokenMap; - - /** Creates an empty special token registry */ - public SpecialTokenRegistry() { - this(List.of()); - } - - /** Create a special token registry from a configuration object. */ - public SpecialTokenRegistry(SpecialtokensConfig config) { - this(specialTokensFrom(config)); - } - - public SpecialTokenRegistry(List specialTokensList) { - specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t)); - } - - private static List specialTokensFrom(SpecialtokensConfig config) { - List specialTokensList = new ArrayList<>(); - for (Iterator i = config.tokenlist().iterator(); i.hasNext();) { - Tokenlist tokenListConfig = i.next(); - - List tokenList = new ArrayList<>(); - for (Iterator j = tokenListConfig.tokens().iterator(); j.hasNext();) { - Tokens tokenConfig = j.next(); - tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace())); - } - specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList)); - } - return specialTokensList; - } - - /** - * Returns the list of special tokens for a given name. - * - * @param name the name of the special tokens to return - * null, the empty string or the string "default" returns - * the default ones - * @return a read-only list of SpecialToken instances, an empty list if this name - * has no special tokens - */ - public SpecialTokens getSpecialTokens(String name) { - if (name == null || name.trim().equals("")) - name = "default"; - return specialTokenMap.getOrDefault(name, SpecialTokens.empty()); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java deleted file mode 100644 index 465d9b754b3..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.yahoo.language.LinguisticsCase.toLowerCase; - -/** - * An immutable list of special tokens - strings which should override the normal tokenizer semantics - * and be tokenized into a single token. Special tokens are case insensitive. - * - * @author bratseth - */ -public class SpecialTokens { - - private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of()); - - private final String name; - private final int maximumLength; - private final List tokens; - private final Map tokenMap; - - public SpecialTokens(String name, List tokens) { - tokens.stream().peek(token -> token.validate()); - List mutableTokens = new ArrayList<>(tokens); - Collections.sort(mutableTokens); - this.name = name; - this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0); - this.tokens = List.copyOf(mutableTokens); - this.tokenMap = tokens.stream().collect(Collectors.toUnmodifiableMap(t -> t.token(), t -> t.replacement())); - } - - /** Returns the name of this special tokens list */ - public String name() { - return name; - } - - /** - * Returns the tokens of this as an immutable map from token to replacement. - * Tokens which do not have a replacement token maps to themselves. - */ - public Map asMap() { return tokenMap; } - - /** - * Returns the special token starting at the start of the given string, or null if no - * special token starts at this string - * - * @param string the string to search for a special token at the start position - * @param substring true to allow the special token to be followed by a character which does not - * mark the end of a token - */ - public Token tokenize(String string, boolean substring) { - // XXX detonator pattern token.length may be != the length of the - // matching data in string, ref caseIndependentLength(String) - String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength))); - for (Iterator i = tokens.iterator(); i.hasNext();) { - Token special = i.next(); - - if (input.startsWith(special.token())) { - if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) - return special; - } - } - return null; - } - - private boolean tokenEndsAt(int position, String string) { - return !Character.isLetterOrDigit(string.charAt(position)); - } - - public static SpecialTokens empty() { return empty; } - - /** An immutable special token */ - public final static class Token implements Comparable { - - private final String token; - private final String replacement; - - /** Creates a special token */ - public Token(String token) { - this(token, null); - } - - /** Creates a special token which will be represented by the given replacement token */ - public Token(String token, String replacement) { - this.token = toLowerCase(token); - if (replacement == null || replacement.trim().equals("")) - this.replacement = this.token; - else - this.replacement = toLowerCase(replacement); - } - - /** Returns the special token */ - public String token() { return token; } - - /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */ - public String replacement() { return replacement; } - - @Override - public int compareTo(Token other) { - if (this.token().length() < other.token().length()) return 1; - if (this.token().length() == other.token().length()) return 0; - return -1; - } - - @Override - public boolean equals(Object other) { - if (other == this) return true; - if ( ! (other instanceof Token)) return false; - return Objects.equals(this.token, ((Token)other).token); - } - - @Override - public int hashCode() { return token.hashCode(); } - - @Override - public String toString() { - return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'"); - } - - private void validate() { - // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... - String asLow = toLowerCase(token); - // TODO: Put along with the global toLowerCase - String asHigh = token.toUpperCase(Locale.ENGLISH); - if (asLow.length() != token.length() || asHigh.length() != token.length()) { - throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " + - "Please report this to the Vespa team."); - } - } - - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java index ad154d1b003..57a5b6edb68 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java @@ -4,7 +4,7 @@ package com.yahoo.language.process; /** * An enumeration of token types. * - * @author Mathias Mølster Lidal + * @author Mathias Mølster Lidal */ public enum TokenType { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 4ffe2a866d8..e1a04b2985d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -11,14 +11,10 @@ import com.yahoo.language.process.GramSplitter; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.language.process.SegmenterImpl; -import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; - -import java.util.List; /** * Factory of simple linguistic processor implementations. @@ -35,7 +31,6 @@ public class SimpleLinguistics implements Linguistics { private final Detector detector; private final CharacterClasses characterClasses; private final GramSplitter gramSplitter; - private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of()); @Inject public SimpleLinguistics() { @@ -50,7 +45,7 @@ public class SimpleLinguistics implements Linguistics { public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); } @Override - public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); } + public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); } @Override public Normalizer getNormalizer() { return normalizer; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 740307c0cca..7df432f496d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -23,13 +23,11 @@ import java.util.logging.Level; */ public class SimpleTokenizer implements Tokenizer { - private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); private final static int SPACE_CODE = 32; - private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); - private final SpecialTokenRegistry specialTokenRegistry; + private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -40,13 +38,8 @@ public class SimpleTokenizer implements Tokenizer { } public SimpleTokenizer(Normalizer normalizer, Transformer transformer) { - this(normalizer, transformer, new SpecialTokenRegistry(List.of())); - } - - public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; - this.specialTokenRegistry = specialTokenRegistry; } @Override @@ -63,8 +56,8 @@ public class SimpleTokenizer implements Tokenizer { String original = input.substring(prev, next); String token = processToken(original, language, stemMode, removeAccents); tokens.add(new SimpleToken(original).setOffset(prev) - .setType(prevType) - .setTokenString(token)); + .setType(prevType) + .setTokenString(token)); prev = next; prevType = nextType; } @@ -74,20 +67,20 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { - String original = token; - log.log(Level.FINEST, () -> "processToken '" + original + "'"); + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { - String oldToken = token; + final String oldToken = token; token = stemmer.stem(token); - String newToken = token; - log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'"); + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); } - String result = token; - log.log(Level.FINEST, () -> "processed token is: " + result); + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); return result; } diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java deleted file mode 100644 index 47c3ba7933c..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class SpecialTokensTestCase { - - @Test - public void testSpecialTokensConfig() { - var builder = new SpecialtokensConfig.Builder(); - var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder(); - tokenBuilder.name("default"); - - var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); - tokenListBuilder1.token("c++"); - tokenListBuilder1.replace("cpp"); - tokenBuilder.tokens(tokenListBuilder1); - - var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); - tokenListBuilder2.token("..."); - tokenBuilder.tokens(tokenListBuilder2); - - builder.tokenlist(tokenBuilder); - - var registry = new SpecialTokenRegistry(builder.build()); - - var defaultTokens = registry.getSpecialTokens("default"); - assertEquals("default", defaultTokens.name()); - assertEquals(2, defaultTokens.asMap().size()); - assertEquals("cpp", defaultTokens.asMap().get("c++")); - assertEquals("...", defaultTokens.asMap().get("...")); - } - -} -- cgit v1.2.3