diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 15:34:49 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 15:34:49 +0200 |
commit | f0e85a3bda91ccff4ecc15735ad269310b1b29f9 (patch) | |
tree | 57e0a4c543e44d1e6f2a1c52aaf36e9c529c1f5e /container-search/src | |
parent | e63dd633b8a12d34db1f779cc67d9f005cf9d445 (diff) |
Make immutable
Diffstat (limited to 'container-search/src')
8 files changed, 186 insertions, 247 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java index f1798948361..9c735a031d7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; @@ -11,55 +10,36 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.logging.Logger; +import java.util.stream.Collectors; /** - * A <i>registry</i> which is responsible for knowing the current - * set of special tokens. The default registry returns empty token lists - * for all names. Usage of this registry is multithread safe. + * A registry which is responsible for knowing the current + * set of special tokens.Usage of this registry is multithread safe. * * @author bratseth */ public class SpecialTokenRegistry { - /** The log of this */ - private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName()); - - private static final SpecialTokens nullSpecialTokens = new SpecialTokens(); - /** * The current special token lists, indexed on name. * These lists are unmodifiable and used directly by clients of this */ - private Map<String, SpecialTokens> specialTokenMap = new HashMap<>(); + private Map<String, SpecialTokens> specialTokenMap; private boolean frozen = false; - /** - * Creates an empty special token registry which - * does not subscribe to any configuration - */ - public SpecialTokenRegistry() {} - - /** - * Create a special token registry which subscribes to the specialtokens - * configuration. Only used for testing. - */ - public SpecialTokenRegistry(String configId) { - try { - build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId)); - } catch (Exception e) { - log.config("No special tokens are configured (" + e.getMessage() + ")"); - } + /** Creates an empty special token registry */ + public SpecialTokenRegistry() { + this(List.of()); } - /** - * Create a special token registry from a configuration object. This is the production code path. - */ + /** Create a special token registry from a configuration object. */ public SpecialTokenRegistry(SpecialtokensConfig config) { - if (config != null) { - build(config); - } + this(specialTokensFrom(config)); + } + + public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) { + specialTokenMap = specialTokensList.stream().collect(Collectors.toMap(t -> t.name(), t -> t)); freeze(); } @@ -67,35 +47,19 @@ public class SpecialTokenRegistry { frozen = true; } - private void build(SpecialtokensConfig config) { - List<SpecialTokens> list = new ArrayList<>(); + private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) { + List<SpecialTokens> specialTokensList = new ArrayList<>(); for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) { - Tokenlist tokenList = i.next(); - SpecialTokens tokens = new SpecialTokens(tokenList.name()); + Tokenlist tokenListConfig = i.next(); - for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext();) { - Tokens token = j.next(); - tokens.addSpecialToken(token.token(), token.replace()); + List<SpecialTokens.Token> tokenList = new ArrayList<>(); + for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) { + Tokens tokenConfig = j.next(); + tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace())); } - tokens.freeze(); - list.add(tokens); + specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList)); } - addSpecialTokens(list); - } - - /** - * Adds a SpecialTokens instance to the registry. That is, add the - * tokens contained for the name of the SpecialTokens instance - * given. - * - * @param specialTokens the SpecialTokens object to add - */ - public void addSpecialTokens(SpecialTokens specialTokens) { - ensureNotFrozen(); - List<SpecialTokens> list = new ArrayList<>(); - list.add(specialTokens); - addSpecialTokens(list); - + return specialTokensList; } private void ensureNotFrozen() { @@ -104,14 +68,6 @@ public class SpecialTokenRegistry { } } - private void addSpecialTokens(List<SpecialTokens> list) { - HashMap<String,SpecialTokens> tokens = new HashMap<>(specialTokenMap); - for(SpecialTokens t: list) { - tokens.put(t.getName(),t); - } - specialTokenMap = tokens; - } - /** * Returns the list of special tokens for a given name. * @@ -122,15 +78,9 @@ public class SpecialTokenRegistry { * has no special tokens */ public SpecialTokens getSpecialTokens(String name) { - if (name == null || name.trim().equals("")) { + if (name == null || name.trim().equals("")) name = "default"; - } - SpecialTokens specialTokens = specialTokenMap.get(name); - - if (specialTokens == null) { - return nullSpecialTokens; - } - return specialTokens; + return specialTokenMap.getOrDefault(name, SpecialTokens.empty()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java index 6f1fad1a6c1..4b29b50f095 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -1,11 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import java.util.logging.Level; -import com.yahoo.prelude.query.Substring; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; -import java.util.*; -import java.util.logging.Logger; +import com.yahoo.prelude.query.Substring; import static com.yahoo.language.LinguisticsCase.toLowerCase; @@ -17,61 +20,28 @@ import static com.yahoo.language.LinguisticsCase.toLowerCase; */ public class SpecialTokens { - private static final Logger log = Logger.getLogger(SpecialTokens.class.getName()); + private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of()); private final String name; - - private final List<SpecialToken> specialTokens = new ArrayList<>(); - - private boolean frozen = false; - - private int currentMaximumLength = 0; - - /** Creates a null list of special tokens */ - public SpecialTokens() { - this.name = "(null)"; - } - - public SpecialTokens(String name) { + private final List<Token> tokens; + private final int maximumLength; + + public SpecialTokens(String name, List<Token> tokens) { + tokens.stream().peek(token -> token.validate()); + List<Token> mutableTokens = new ArrayList<>(tokens); + Collections.sort(mutableTokens); + this.tokens = List.copyOf(mutableTokens); this.name = name; + this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0); } /** Returns the name of this special tokens list */ - public String getName() { + public String name() { return name; } - /** - * Adds a special token to this - * - * @param token the special token string to add - * @param replace the token to replace instances of the special token with, or null to keep the token - */ - public void addSpecialToken(String token, String replace) { - ensureNotFrozen(); - if (!caseIndependentLength(token)) { - return; - } - // TODO: Are special tokens correctly unicode normalized in regards to query parsing? - final SpecialToken specialTokenToAdd = new SpecialToken(token, replace); - currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length()); - specialTokens.add(specialTokenToAdd); - Collections.sort(specialTokens); - } - - private boolean caseIndependentLength(String token) { - // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... - String asLow = toLowerCase(token); - // TODO: Put along with the global toLowerCase - String asHigh = token.toUpperCase(Locale.ENGLISH); - if (asLow.length() != token.length() || asHigh.length() != token.length()) { - log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + - " Please report this message in a bug to the Vespa team."); - return false; - } else { - return true; - } - } + /** Returns a sorted immutable list of the special tokens in this */ + public List<Token> tokens() { return tokens; } /** * Returns the special token starting at the start of the given string, or null if no @@ -81,12 +51,12 @@ public class SpecialTokens { * @param substring true to allow the special token to be followed by a character which does not * mark the end of a token */ - public SpecialToken tokenize(String string, boolean substring) { + public Token tokenize(String string, boolean substring) { // XXX detonator pattern token.length may be != the length of the // matching data in string, ref caseIndependentLength(String) - String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength))); - for (Iterator<SpecialToken> i = specialTokens.iterator(); i.hasNext();) { - SpecialTokens.SpecialToken special = i.next(); + String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength))); + for (Iterator<Token> i = tokens.iterator(); i.hasNext();) { + Token special = i.next(); if (input.startsWith(special.token())) { if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) @@ -100,49 +70,36 @@ public class SpecialTokens { return !Character.isLetterOrDigit(string.charAt(position)); } - /** Returns the number of special tokens in this */ - public int size() { - return specialTokens.size(); - } - - private void ensureNotFrozen() { - if (frozen) { - throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance."); - } - } - - public void freeze() { - frozen = true; - } + public static SpecialTokens empty() { return empty; } /** An immutable special token */ - public final static class SpecialToken implements Comparable<SpecialToken> { + public final static class Token implements Comparable<Token> { private final String token; + private final String replacement; - private final String replace; + /** Creates a special token */ + public Token(String token) { + this(token, null); + } - public SpecialToken(String token, String replace) { + /** Creates a special token which will be represented by the given replacement token */ + public Token(String token, String replacement) { this.token = toLowerCase(token); - if (replace == null || replace.trim().equals("")) { - this.replace = this.token; - } else { - this.replace = toLowerCase(replace); - } + if (replacement == null || replacement.trim().equals("")) + this.replacement = this.token; + else + this.replacement = toLowerCase(replacement); } /** Returns the special token */ - public String token() { - return token; - } + public String token() { return token; } - /** Returns the right replace value, never null or an empty string */ - public String replace() { - return replace; - } + /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */ + public String replacement() { return replacement; } @Override - public int compareTo(SpecialToken other) { + public int compareTo(Token other) { if (this.token().length() < other.token().length()) return 1; if (this.token().length() == other.token().length()) return 0; return -1; @@ -151,15 +108,27 @@ public class SpecialTokens { @Override public boolean equals(Object other) { if (other == this) return true; - if ( ! (other instanceof SpecialToken)) return false; - return Objects.equals(this.token, ((SpecialToken)other).token); + if ( ! (other instanceof Token)) return false; + return Objects.equals(this.token, ((Token)other).token); } @Override public int hashCode() { return token.hashCode(); } - public Token toToken(int start, String rawSource) { - return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe? + @Override + public String toString() { + return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'"); + } + + private void validate() { + // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... + String asLow = toLowerCase(token); + // TODO: Put along with the global toLowerCase + String asHigh = token.toUpperCase(Locale.ENGLISH); + if (asLow.length() != token.length() || asHigh.length() != token.length()) { + throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " + + "Please report this to the Vespa team."); + } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 686ceefc460..f0656efa59a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -200,7 +200,7 @@ public final class Tokenizer { } StringBuilder tmp = new StringBuilder(); for (int i = 0; i < tokencnt; i++) { - Token useToken = tokens.get(backtrack+i); + Token useToken = tokens.get(backtrack + i); tmp.append(useToken.image); } String indexName = tmp.toString(); @@ -216,13 +216,13 @@ public final class Tokenizer { } private int consumeSpecialToken(int start) { - SpecialTokens.SpecialToken specialToken = getSpecialToken(start); - if (specialToken == null) return start; - tokens.add(specialToken.toToken(start, source)); - return start + specialToken.token().length(); + SpecialTokens.Token token = getSpecialToken(start); + if (token == null) return start; + tokens.add(toToken(token, start, source)); + return start + token.token().length(); } - private SpecialTokens.SpecialToken getSpecialToken(int start) { + private SpecialTokens.Token getSpecialToken(int start) { if (specialTokens == null) return null; return specialTokens.tokenize(source.substring(start), substringSpecialTokens); } @@ -467,7 +467,7 @@ public final class Tokenizer { /** Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number */ private int consumeWordOrNumber(int start, Index currentIndex) { int tokenEnd = start; - SpecialTokens.SpecialToken substringSpecialToken = null; + SpecialTokens.Token substringToken = null; boolean digitsOnly = true; // int underscores = 0; // boolean underscoresOnly = true; @@ -475,8 +475,8 @@ public final class Tokenizer { while (tokenEnd < source.length()) { if (substringSpecialTokens) { - substringSpecialToken = getSpecialToken(tokenEnd); - if (substringSpecialToken != null) break; + substringToken = getSpecialToken(tokenEnd); + if (substringToken != null) break; } int c = source.codePointAt(tokenEnd); @@ -524,11 +524,11 @@ public final class Tokenizer { } } - if (substringSpecialToken == null) + if (substringToken == null) return --tokenEnd; // TODO: test the logic around tokenEnd with friends - addToken(substringSpecialToken.toToken(tokenEnd, source)); - return --tokenEnd + substringSpecialToken.token().length(); + addToken(toToken(substringToken, tokenEnd, source)); + return --tokenEnd + substringToken.token().length(); } private void addToken(Token.Kind kind, String word, int start, int end) { @@ -539,4 +539,11 @@ public final class Tokenizer { tokens.add(token); } + public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) { + return new Token(Token.Kind.WORD, + specialToken.replacement(), + true, + new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe? + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 94b9bf6ce65..1b35296082e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -18,7 +18,7 @@ public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); private Linguistics linguistics = new SimpleLinguistics(); - private SpecialTokens specialTokens = new SpecialTokens(); + private SpecialTokens specialTokens = SpecialTokens.empty(); public IndexFacts getIndexFacts() { return indexFacts; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 84fe88d0292..7adfccc2ed1 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -17,8 +17,6 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; -import java.util.logging.Logger; - /** * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread) * of the searchers of a search chain.</p> diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 6afea895f3a..eb86af993d7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -1639,7 +1639,7 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { - ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); + ParsingTester customTester = new ParsingTester(SpecialTokens.empty()); customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java index 17155fff5de..a17d791f906 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -20,6 +20,9 @@ import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.query.parser.ParserFactory; +import java.util.ArrayList; +import java.util.List; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -32,7 +35,7 @@ import static org.junit.Assert.assertTrue; public class ParsingTester { private static final Linguistics linguistics = new SimpleLinguistics(); - private IndexFacts indexFacts; + private final IndexFacts indexFacts; private SpecialTokenRegistry tokenRegistry; public ParsingTester() { @@ -49,11 +52,10 @@ public class ParsingTester { public ParsingTester(IndexFacts indexFacts, SpecialTokens specialTokens) { indexFacts.freeze(); - specialTokens.freeze(); this.indexFacts = indexFacts; tokenRegistry = new SpecialTokenRegistry(); - tokenRegistry.addSpecialTokens(specialTokens); + tokenRegistry = new SpecialTokenRegistry(List.of(specialTokens)); } /** @@ -72,13 +74,13 @@ public class ParsingTester { * This can be used to add new tokens and passing the resulting special tokens to the constructor of this. */ public static SpecialTokens createSpecialTokens() { - SpecialTokens tokens = new SpecialTokens("default"); - tokens.addSpecialToken("c++", null); - tokens.addSpecialToken(".net", "dotnet"); - tokens.addSpecialToken("tcp/ip", null); - tokens.addSpecialToken("c#", null); - tokens.addSpecialToken("special-token-fs","firstsecond"); - return tokens; + List<SpecialTokens.Token> tokens = new ArrayList<>(); + tokens.add(new SpecialTokens.Token("c++")); + tokens.add(new SpecialTokens.Token(".net", "dotnet")); + tokens.add(new SpecialTokens.Token("tcp/ip")); + tokens.add(new SpecialTokens.Token("c#")); + tokens.add(new SpecialTokens.Token("special-token-fs","firstsecond")); + return new SpecialTokens("default", tokens); } /** diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index aa2e9dbcf75..5634cf67212 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -10,8 +10,10 @@ import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.prelude.query.parser.Token; import com.yahoo.prelude.query.parser.Tokenizer; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import org.junit.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -39,13 +41,11 @@ import static org.junit.Assert.assertTrue; */ public class TokenizerTestCase { - private SpecialTokenRegistry defaultRegistry = new SpecialTokenRegistry("file:src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg"); - @Test public void testPlainTokenization() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a"); assertEquals(new Token(WORD, "drive"), tokens.get(0)); @@ -87,7 +87,7 @@ public class TokenizerTestCase { public void testOneSpecialToken() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("c++ lovers, please apply"); assertEquals(new Token(WORD, "c++"), tokens.get(0)); @@ -97,7 +97,7 @@ public class TokenizerTestCase { public void testSpecialTokenCombination() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("c#, c++ or .net know, not tcp/ip"); assertEquals(new Token(WORD, "c#"), tokens.get(0)); @@ -123,10 +123,9 @@ public class TokenizerTestCase { */ @Test public void testSpecialTokenCJK() { - assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); tokenizer.setSubstringSpecialTokens(true); - tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing")); List<?> tokens = tokenizer.tokenize("fooc#bar,c++with spacebarknowknowknow,knowknownot know"); assertEquals(new Token(WORD, "foo"), tokens.get(0)); @@ -151,7 +150,7 @@ public class TokenizerTestCase { public void testSpecialTokenCaseInsensitive() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("The AS/400 is great"); assertEquals(new Token(WORD, "The"), tokens.get(0)); @@ -167,7 +166,7 @@ public class TokenizerTestCase { public void testSpecialTokenNonMatch() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("c++ c+ aS/400 i/o .net i/ooo ap.net"); assertEquals(new Token(WORD, "c++"), tokens.get(0)); @@ -190,18 +189,9 @@ public class TokenizerTestCase { @Test public void testSpecialTokenConfigurationDefault() { - String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; - - SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); - assertEquals("Special tokens configured", 6, - r.getSpecialTokens("default").size()); - assertEquals("Special tokens configured", 4, - r.getSpecialTokens("other").size()); - Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens( - r.getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize( "with space, c++ or .... know, not b.s.d."); @@ -224,18 +214,9 @@ public class TokenizerTestCase { @Test public void testSpecialTokenConfigurationOther() { - String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; - - SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); - assertEquals("Special tokens configured", 6, - r.getSpecialTokens("default").size()); - assertEquals("Special tokens configured", 4, - r.getSpecialTokens("other").size()); - Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens( - r.getSpecialTokens("other")); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("other")); List<?> tokens = tokenizer.tokenize( "with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d."); @@ -267,26 +248,9 @@ public class TokenizerTestCase { } @Test - public void testSpecialTokenConfigurationMissing() { - String tokenFile = "file:source/bogus/specialtokens.cfg"; - - SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); - - Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - - tokenizer.setSpecialTokens(r.getSpecialTokens("other")); - List<?> tokens = tokenizer.tokenize("c++"); - - assertEquals(new Token(WORD, "c"), tokens.get(0)); - assertEquals(new Token(PLUS, "+"), tokens.get(1)); - assertEquals(new Token(PLUS, "+"), tokens.get(2)); - } - - @Test public void testTokenReplacing() { - assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing")); List<?> tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d."); assertEquals(new Token(WORD, "with-space"), tokens.get(0)); @@ -745,7 +709,7 @@ public class TokenizerTestCase { public void testSingleQuoteAsWordCharacter() { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - tokenizer.setSpecialTokens(createSpecialTokens()); + tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default")); List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'"); assertEquals(new Token(WORD, "drive"), tokens.get(0)); @@ -781,17 +745,66 @@ public class TokenizerTestCase { assertEquals(new Token(WORD, "a'"), tokens.get(30)); } - private SpecialTokens createSpecialTokens() { - SpecialTokens tokens = new SpecialTokens("default"); - - tokens.addSpecialToken("c+", null); - tokens.addSpecialToken("c++", null); - tokens.addSpecialToken(".net", null); - tokens.addSpecialToken("tcp/ip", null); - tokens.addSpecialToken("i/o", null); - tokens.addSpecialToken("c#", null); - tokens.addSpecialToken("AS/400", null); - return tokens; + @Test + public void testSpecialTokensConfig() { + var builder = new SpecialtokensConfig.Builder(); + var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder(); + tokenBuilder.name("default"); + + var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder1.token("c++"); + tokenListBuilder1.replace("cpp"); + tokenBuilder.tokens(tokenListBuilder1); + + var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder2.token("..."); + tokenBuilder.tokens(tokenListBuilder2); + + builder.tokenlist(tokenBuilder); + + var registry = new SpecialTokenRegistry(builder.build()); + + var defaultTokens = registry.getSpecialTokens("default"); + assertEquals("default", defaultTokens.name()); + assertEquals(2, defaultTokens.tokens().size()); + assertEquals("c++", defaultTokens.tokens().get(0).token()); + assertEquals("cpp", defaultTokens.tokens().get(0).replacement()); + assertEquals("...", defaultTokens.tokens().get(1).token()); + assertEquals("...", defaultTokens.tokens().get(1).replacement()); + } + + private SpecialTokenRegistry createSpecialTokens() { + List<SpecialTokens.Token> tokens = new ArrayList<>(); + tokens.add(new SpecialTokens.Token("c+")); + tokens.add(new SpecialTokens.Token("c++")); + tokens.add(new SpecialTokens.Token(".net")); + tokens.add(new SpecialTokens.Token("tcp/ip")); + tokens.add(new SpecialTokens.Token("i/o")); + tokens.add(new SpecialTokens.Token("c#")); + tokens.add(new SpecialTokens.Token("AS/400")); + tokens.add(new SpecialTokens.Token("....")); + tokens.add(new SpecialTokens.Token("b.s.d.")); + tokens.add(new SpecialTokens.Token("with space")); + tokens.add(new SpecialTokens.Token("dvd\\xB1r")); + SpecialTokens defaultTokens = new SpecialTokens("default", tokens); + + tokens = new ArrayList<>(); + tokens.add(new SpecialTokens.Token("[huh]")); + tokens.add(new SpecialTokens.Token("&&&%%%")); + tokens.add(new SpecialTokens.Token("------")); + tokens.add(new SpecialTokens.Token("!!!***")); + SpecialTokens otherTokens = new SpecialTokens("other", tokens); + + tokens = new ArrayList<>(); + tokens.add(new SpecialTokens.Token("....")); + tokens.add(new SpecialTokens.Token("c++", "cpp")); + tokens.add(new SpecialTokens.Token("b.s.d.")); + tokens.add(new SpecialTokens.Token("with space", "with-space")); + tokens.add(new SpecialTokens.Token("c#")); + tokens.add(new SpecialTokens.Token("know", "knuwww")); + SpecialTokens replacingTokens = new SpecialTokens("replacing", tokens); + + return new SpecialTokenRegistry(List.of(defaultTokens, otherTokens, replacingTokens)); } } |