From 346c5a523fec4ec7cecb373e480e30846f5e33e5 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 5 May 2021 21:18:50 +0200 Subject: Revert "Merge pull request #17754 from vespa-engine/revert-17747-bratseth/special-tokens-take-2" This reverts commit a2c9cd4bc04f1a3eaa31524b3970b96be5c2eda9, reversing changes made to 8c61a373af0066fbdf1cca354c24b197c7347321. --- .../yahoo/language/opennlp/OpenNlpTokenizer.java | 9 +- .../language/process/SpecialTokenRegistry.java | 72 +++++++++++ .../com/yahoo/language/process/SpecialTokens.java | 141 +++++++++++++++++++++ .../java/com/yahoo/language/process/TokenType.java | 2 +- .../yahoo/language/simple/SimpleLinguistics.java | 7 +- .../com/yahoo/language/simple/SimpleTokenizer.java | 27 ++-- 6 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java create mode 100644 linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java (limited to 'linguistics/src/main/java/com') diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index e1185cb2457..73518876c3f 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -4,6 +4,7 @@ package com.yahoo.language.opennlp; import com.yahoo.language.Language; import com.yahoo.language.LinguisticsCase; import com.yahoo.language.process.Normalizer; +import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import com.yahoo.language.process.TokenType; @@ -32,15 +33,21 @@ public class OpenNlpTokenizer implements Tokenizer { private final Normalizer normalizer; private final Transformer transformer; private final SimpleTokenizer simpleTokenizer; + private final SpecialTokenRegistry specialTokenRegistry; public OpenNlpTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); } public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { + this(normalizer, transformer, new SpecialTokenRegistry(List.of())); + } + + public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; - simpleTokenizer = new SimpleTokenizer(normalizer, transformer); + this.specialTokenRegistry = specialTokenRegistry; + this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java new file mode 100644 index 00000000000..b6335d67967 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java @@ -0,0 +1,72 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics + * and be tokenized into a single token. + * + * @author bratseth + */ +public class SpecialTokenRegistry { + + /** + * The current special token lists, indexed on name. + * These lists are unmodifiable and used directly by clients of this + */ + private final Map specialTokenMap; + + /** Creates an empty special token registry */ + public SpecialTokenRegistry() { + this(List.of()); + } + + /** Create a special token registry from a configuration object. */ + public SpecialTokenRegistry(SpecialtokensConfig config) { + this(specialTokensFrom(config)); + } + + public SpecialTokenRegistry(List specialTokensList) { + specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t)); + } + + private static List specialTokensFrom(SpecialtokensConfig config) { + List specialTokensList = new ArrayList<>(); + for (Iterator i = config.tokenlist().iterator(); i.hasNext();) { + Tokenlist tokenListConfig = i.next(); + + List tokenList = new ArrayList<>(); + for (Iterator j = tokenListConfig.tokens().iterator(); j.hasNext();) { + Tokens tokenConfig = j.next(); + tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace())); + } + specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList)); + } + return specialTokensList; + } + + /** + * Returns the list of special tokens for a given name. + * + * @param name the name of the special tokens to return + * null, the empty string or the string "default" returns + * the default ones + * @return a read-only list of SpecialToken instances, an empty list if this name + * has no special tokens + */ + public SpecialTokens getSpecialTokens(String name) { + if (name == null || name.trim().equals("")) + name = "default"; + return specialTokenMap.getOrDefault(name, SpecialTokens.empty()); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java new file mode 100644 index 00000000000..465d9b754b3 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java @@ -0,0 +1,141 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * An immutable list of special tokens - strings which should override the normal tokenizer semantics + * and be tokenized into a single token. Special tokens are case insensitive. + * + * @author bratseth + */ +public class SpecialTokens { + + private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of()); + + private final String name; + private final int maximumLength; + private final List tokens; + private final Map tokenMap; + + public SpecialTokens(String name, List tokens) { + tokens.stream().peek(token -> token.validate()); + List mutableTokens = new ArrayList<>(tokens); + Collections.sort(mutableTokens); + this.name = name; + this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0); + this.tokens = List.copyOf(mutableTokens); + this.tokenMap = tokens.stream().collect(Collectors.toUnmodifiableMap(t -> t.token(), t -> t.replacement())); + } + + /** Returns the name of this special tokens list */ + public String name() { + return name; + } + + /** + * Returns the tokens of this as an immutable map from token to replacement. + * Tokens which do not have a replacement token maps to themselves. + */ + public Map asMap() { return tokenMap; } + + /** + * Returns the special token starting at the start of the given string, or null if no + * special token starts at this string + * + * @param string the string to search for a special token at the start position + * @param substring true to allow the special token to be followed by a character which does not + * mark the end of a token + */ + public Token tokenize(String string, boolean substring) { + // XXX detonator pattern token.length may be != the length of the + // matching data in string, ref caseIndependentLength(String) + String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength))); + for (Iterator i = tokens.iterator(); i.hasNext();) { + Token special = i.next(); + + if (input.startsWith(special.token())) { + if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) + return special; + } + } + return null; + } + + private boolean tokenEndsAt(int position, String string) { + return !Character.isLetterOrDigit(string.charAt(position)); + } + + public static SpecialTokens empty() { return empty; } + + /** An immutable special token */ + public final static class Token implements Comparable { + + private final String token; + private final String replacement; + + /** Creates a special token */ + public Token(String token) { + this(token, null); + } + + /** Creates a special token which will be represented by the given replacement token */ + public Token(String token, String replacement) { + this.token = toLowerCase(token); + if (replacement == null || replacement.trim().equals("")) + this.replacement = this.token; + else + this.replacement = toLowerCase(replacement); + } + + /** Returns the special token */ + public String token() { return token; } + + /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */ + public String replacement() { return replacement; } + + @Override + public int compareTo(Token other) { + if (this.token().length() < other.token().length()) return 1; + if (this.token().length() == other.token().length()) return 0; + return -1; + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof Token)) return false; + return Objects.equals(this.token, ((Token)other).token); + } + + @Override + public int hashCode() { return token.hashCode(); } + + @Override + public String toString() { + return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'"); + } + + private void validate() { + // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... + String asLow = toLowerCase(token); + // TODO: Put along with the global toLowerCase + String asHigh = token.toUpperCase(Locale.ENGLISH); + if (asLow.length() != token.length() || asHigh.length() != token.length()) { + throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " + + "Please report this to the Vespa team."); + } + } + + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java index 57a5b6edb68..ad154d1b003 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java @@ -4,7 +4,7 @@ package com.yahoo.language.process; /** * An enumeration of token types. * - * @author Mathias Mølster Lidal + * @author Mathias Mølster Lidal */ public enum TokenType { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index e1a04b2985d..4ffe2a866d8 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -11,10 +11,14 @@ import com.yahoo.language.process.GramSplitter; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.language.process.SegmenterImpl; +import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; + +import java.util.List; /** * Factory of simple linguistic processor implementations. @@ -31,6 +35,7 @@ public class SimpleLinguistics implements Linguistics { private final Detector detector; private final CharacterClasses characterClasses; private final GramSplitter gramSplitter; + private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of()); @Inject public SimpleLinguistics() { @@ -45,7 +50,7 @@ public class SimpleLinguistics implements Linguistics { public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); } @Override - public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); } + public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); } @Override public Normalizer getNormalizer() { return normalizer; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 7df432f496d..740307c0cca 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -23,11 +23,13 @@ import java.util.logging.Level; */ public class SimpleTokenizer implements Tokenizer { + private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); private final static int SPACE_CODE = 32; + private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); - private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); + private final SpecialTokenRegistry specialTokenRegistry; public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -38,8 +40,13 @@ public class SimpleTokenizer implements Tokenizer { } public SimpleTokenizer(Normalizer normalizer, Transformer transformer) { + this(normalizer, transformer, new SpecialTokenRegistry(List.of())); + } + + public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { this.normalizer = normalizer; this.transformer = transformer; + this.specialTokenRegistry = specialTokenRegistry; } @Override @@ -56,8 +63,8 @@ public class SimpleTokenizer implements Tokenizer { String original = input.substring(prev, next); String token = processToken(original, language, stemMode, removeAccents); tokens.add(new SimpleToken(original).setOffset(prev) - .setType(prevType) - .setTokenString(token)); + .setType(prevType) + .setTokenString(token)); prev = next; prevType = nextType; } @@ -67,20 +74,20 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { - final String original = token; - log.log(Level.FINEST, () -> "processToken '"+original+"'"); + String original = token; + log.log(Level.FINEST, () -> "processToken '" + original + "'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { - final String oldToken = token; + String oldToken = token; token = stemmer.stem(token); - final String newToken = token; - log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); + String newToken = token; + log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'"); } - final String result = token; - log.log(Level.FINEST, () -> "processed token is: "+result); + String result = token; + log.log(Level.FINEST, () -> "processed token is: " + result); return result; } -- cgit v1.2.3