diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 22:55:00 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-04 22:55:00 +0200 |
commit | 4c1ac8706f9aa9a8dbf5d8a73ee87650cf6620c5 (patch) | |
tree | e3553d597f6998695ec10dfb5c5db17979d0513a /linguistics/src/main | |
parent | 8e1475c69c80e937cfa3eb47372c8008786196af (diff) |
Revert "Bratseth/special tokens"
Diffstat (limited to 'linguistics/src/main')
6 files changed, 13 insertions, 245 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 73518876c3f..e1185cb2457 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -4,7 +4,6 @@ package com.yahoo.language.opennlp; import com.yahoo.language.Language; import com.yahoo.language.LinguisticsCase; import com.yahoo.language.process.Normalizer; -import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import com.yahoo.language.process.TokenType; @@ -33,21 +32,15 @@ public class OpenNlpTokenizer implements Tokenizer { private final Normalizer normalizer; private final Transformer transformer; private final SimpleTokenizer simpleTokenizer; - private final SpecialTokenRegistry specialTokenRegistry; public OpenNlpTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); } public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { - this(normalizer, transformer, new SpecialTokenRegistry(List.of())); - } - - public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; - this.specialTokenRegistry = specialTokenRegistry; - this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); + simpleTokenizer = new SimpleTokenizer(normalizer, transformer); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java deleted file mode 100644 index b6335d67967..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics - * and be tokenized into a single token. - * - * @author bratseth - */ -public class SpecialTokenRegistry { - - /** - * The current special token lists, indexed on name. - * These lists are unmodifiable and used directly by clients of this - */ - private final Map<String, SpecialTokens> specialTokenMap; - - /** Creates an empty special token registry */ - public SpecialTokenRegistry() { - this(List.of()); - } - - /** Create a special token registry from a configuration object. */ - public SpecialTokenRegistry(SpecialtokensConfig config) { - this(specialTokensFrom(config)); - } - - public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) { - specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t)); - } - - private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) { - List<SpecialTokens> specialTokensList = new ArrayList<>(); - for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) { - Tokenlist tokenListConfig = i.next(); - - List<SpecialTokens.Token> tokenList = new ArrayList<>(); - for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) { - Tokens tokenConfig = j.next(); - tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace())); - } - specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList)); - } - return specialTokensList; - } - - /** - * Returns the list of special tokens for a given name. - * - * @param name the name of the special tokens to return - * null, the empty string or the string "default" returns - * the default ones - * @return a read-only list of SpecialToken instances, an empty list if this name - * has no special tokens - */ - public SpecialTokens getSpecialTokens(String name) { - if (name == null || name.trim().equals("")) - name = "default"; - return specialTokenMap.getOrDefault(name, SpecialTokens.empty()); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java deleted file mode 100644 index 465d9b754b3..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.yahoo.language.LinguisticsCase.toLowerCase; - -/** - * An immutable list of special tokens - strings which should override the normal tokenizer semantics - * and be tokenized into a single token. Special tokens are case insensitive. - * - * @author bratseth - */ -public class SpecialTokens { - - private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of()); - - private final String name; - private final int maximumLength; - private final List<Token> tokens; - private final Map<String, String> tokenMap; - - public SpecialTokens(String name, List<Token> tokens) { - tokens.stream().peek(token -> token.validate()); - List<Token> mutableTokens = new ArrayList<>(tokens); - Collections.sort(mutableTokens); - this.name = name; - this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0); - this.tokens = List.copyOf(mutableTokens); - this.tokenMap = tokens.stream().collect(Collectors.toUnmodifiableMap(t -> t.token(), t -> t.replacement())); - } - - /** Returns the name of this special tokens list */ - public String name() { - return name; - } - - /** - * Returns the tokens of this as an immutable map from token to replacement. - * Tokens which do not have a replacement token maps to themselves. - */ - public Map<String, String> asMap() { return tokenMap; } - - /** - * Returns the special token starting at the start of the given string, or null if no - * special token starts at this string - * - * @param string the string to search for a special token at the start position - * @param substring true to allow the special token to be followed by a character which does not - * mark the end of a token - */ - public Token tokenize(String string, boolean substring) { - // XXX detonator pattern token.length may be != the length of the - // matching data in string, ref caseIndependentLength(String) - String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength))); - for (Iterator<Token> i = tokens.iterator(); i.hasNext();) { - Token special = i.next(); - - if (input.startsWith(special.token())) { - if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) - return special; - } - } - return null; - } - - private boolean tokenEndsAt(int position, String string) { - return !Character.isLetterOrDigit(string.charAt(position)); - } - - public static SpecialTokens empty() { return empty; } - - /** An immutable special token */ - public final static class Token implements Comparable<Token> { - - private final String token; - private final String replacement; - - /** Creates a special token */ - public Token(String token) { - this(token, null); - } - - /** Creates a special token which will be represented by the given replacement token */ - public Token(String token, String replacement) { - this.token = toLowerCase(token); - if (replacement == null || replacement.trim().equals("")) - this.replacement = this.token; - else - this.replacement = toLowerCase(replacement); - } - - /** Returns the special token */ - public String token() { return token; } - - /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */ - public String replacement() { return replacement; } - - @Override - public int compareTo(Token other) { - if (this.token().length() < other.token().length()) return 1; - if (this.token().length() == other.token().length()) return 0; - return -1; - } - - @Override - public boolean equals(Object other) { - if (other == this) return true; - if ( ! (other instanceof Token)) return false; - return Objects.equals(this.token, ((Token)other).token); - } - - @Override - public int hashCode() { return token.hashCode(); } - - @Override - public String toString() { - return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'"); - } - - private void validate() { - // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... - String asLow = toLowerCase(token); - // TODO: Put along with the global toLowerCase - String asHigh = token.toUpperCase(Locale.ENGLISH); - if (asLow.length() != token.length() || asHigh.length() != token.length()) { - throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " + - "Please report this to the Vespa team."); - } - } - - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java index ad154d1b003..57a5b6edb68 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java @@ -4,7 +4,7 @@ package com.yahoo.language.process; /** * An enumeration of token types. * - * @author Mathias Mølster Lidal + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> */ public enum TokenType { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 4ffe2a866d8..e1a04b2985d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -11,14 +11,10 @@ import com.yahoo.language.process.GramSplitter; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.language.process.SegmenterImpl; -import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; - -import java.util.List; /** * Factory of simple linguistic processor implementations. @@ -35,7 +31,6 @@ public class SimpleLinguistics implements Linguistics { private final Detector detector; private final CharacterClasses characterClasses; private final GramSplitter gramSplitter; - private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of()); @Inject public SimpleLinguistics() { @@ -50,7 +45,7 @@ public class SimpleLinguistics implements Linguistics { public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); } @Override - public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); } + public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); } @Override public Normalizer getNormalizer() { return normalizer; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 740307c0cca..7df432f496d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -23,13 +23,11 @@ import java.util.logging.Level; */ public class SimpleTokenizer implements Tokenizer { - private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); private final static int SPACE_CODE = 32; - private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); - private final SpecialTokenRegistry specialTokenRegistry; + private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -40,13 +38,8 @@ public class SimpleTokenizer implements Tokenizer { } public SimpleTokenizer(Normalizer normalizer, Transformer transformer) { - this(normalizer, transformer, new SpecialTokenRegistry(List.of())); - } - - public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; - this.specialTokenRegistry = specialTokenRegistry; } @Override @@ -63,8 +56,8 @@ public class SimpleTokenizer implements Tokenizer { String original = input.substring(prev, next); String token = processToken(original, language, stemMode, removeAccents); tokens.add(new SimpleToken(original).setOffset(prev) - .setType(prevType) - .setTokenString(token)); + .setType(prevType) + .setTokenString(token)); prev = next; prevType = nextType; } @@ -74,20 +67,20 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { - String original = token; - log.log(Level.FINEST, () -> "processToken '" + original + "'"); + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { - String oldToken = token; + final String oldToken = token; token = stemmer.stem(token); - String newToken = token; - log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'"); + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); } - String result = token; - log.log(Level.FINEST, () -> "processed token is: " + result); + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); return result; } |