diff options
Diffstat (limited to 'linguistics/src/main/java')
6 files changed, 248 insertions, 10 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 5e28213d524..035de415aa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -41,7 +41,12 @@ public interface Linguistics { CHARACTER_CLASSES } - /** The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. */ + /** + * The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. + * + * @deprecated use new com.yahoo.language.simple.SimpleLinguistics() + */ + @Deprecated // TODO: Remove this field on Vespa 7 Linguistics SIMPLE = new SimpleLinguistics(); /** diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java new file mode 100644 index 00000000000..12de309a2d3 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -0,0 +1,11 @@ +package com.yahoo.language.opennlp; + +import com.yahoo.language.process.Tokenizer; +import com.yahoo.language.simple.SimpleLinguistics; + +public class OpenNlpLinguistics extends SimpleLinguistics { + @Override + public Tokenizer getTokenizer() { + return new OpenNlpTokenizer(getNormalizer(), getTransformer()); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java new file mode 100644 index 00000000000..5d5f5cbfba9 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -0,0 +1,135 @@ +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.LinguisticsCase; +import com.yahoo.language.process.*; +import com.yahoo.language.simple.*; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.snowball.SnowballStemmer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class OpenNlpTokenizer implements Tokenizer { + private final static int SPACE_CODE = 32; + private final Normalizer normalizer; + private final Transformer transformer; + private final SimpleTokenizer simpleTokenizer; + + public OpenNlpTokenizer() { + this(new SimpleNormalizer(), new SimpleTransformer()); + } + + public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { + this.normalizer = normalizer; + this.transformer = transformer; + simpleTokenizer = new SimpleTokenizer(normalizer, transformer); + } + + @Override + public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { + if (input.isEmpty()) return Collections.emptyList(); + Stemmer stemmer = getStemmerForLanguage(language, stemMode); + if (stemmer == null) { + return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); + } + + List<Token> tokens = new ArrayList<>(); + int nextCode = input.codePointAt(0); + TokenType prevType = SimpleTokenType.valueOf(nextCode); + for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) { + nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE; + TokenType nextType = SimpleTokenType.valueOf(nextCode); + if (!prevType.isIndexable() || !nextType.isIndexable()) { + String original = input.substring(prev, next); + String token = processToken(original, language, stemMode, removeAccents, stemmer); + tokens.add(new SimpleToken(original).setOffset(prev) + .setType(prevType) + .setTokenString(token)); + prev = next; + prevType = nextType; + } + next += Character.charCount(nextCode); + } + return tokens; + } + + private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) { + if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) { + return null; + } + SnowballStemmer.ALGORITHM alg; + switch (language) { + case DANISH: + alg = SnowballStemmer.ALGORITHM.DANISH; + break; + case DUTCH: + alg = SnowballStemmer.ALGORITHM.DUTCH; + break; + case FINNISH: + alg = SnowballStemmer.ALGORITHM.FINNISH; + break; + case FRENCH: + alg = SnowballStemmer.ALGORITHM.FRENCH; + break; + case GERMAN: + alg = SnowballStemmer.ALGORITHM.GERMAN; + break; + case HUNGARIAN: + alg = SnowballStemmer.ALGORITHM.HUNGARIAN; + break; + case IRISH: + alg = SnowballStemmer.ALGORITHM.IRISH; + break; + case ITALIAN: + alg = SnowballStemmer.ALGORITHM.ITALIAN; + break; + case NORWEGIAN_BOKMAL: + case NORWEGIAN_NYNORSK: + alg = SnowballStemmer.ALGORITHM.NORWEGIAN; + break; + case PORTUGUESE: + alg = SnowballStemmer.ALGORITHM.PORTUGUESE; + break; + case ROMANIAN: + alg = SnowballStemmer.ALGORITHM.ROMANIAN; + break; + case RUSSIAN: + alg = SnowballStemmer.ALGORITHM.RUSSIAN; + break; + case SPANISH: + alg = SnowballStemmer.ALGORITHM.SPANISH; + break; + case SWEDISH: + alg = SnowballStemmer.ALGORITHM.SWEDISH; + break; + case TURKISH: + alg = SnowballStemmer.ALGORITHM.TURKISH; + break; + case ENGLISH: + alg = SnowballStemmer.ALGORITHM.ENGLISH; + break; + default: + return null; + + } + return new SnowballStemmer(alg); + } + + private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, + Stemmer stemmer) { + token = normalizer.normalize(token); + token = LinguisticsCase.toLowerCase(token); + if (removeAccents) + token = transformer.accentDrop(token, language); + if (stemMode != StemMode.NONE) { + token = doStemming(token, stemmer); + } + return token; + } + + private String doStemming(String token, Stemmer stemmer) { + return stemmer.stem(token).toString(); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java b/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java new file mode 100644 index 00000000000..2bdd315418f --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.language.opennlp; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index e6ce4eddb59..2b31f95675b 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -1,17 +1,30 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; +import com.google.common.base.Optional; +import com.optimaize.langdetect.LanguageDetector; +import com.optimaize.langdetect.LanguageDetectorBuilder; +import com.optimaize.langdetect.i18n.LdLocale; +import com.optimaize.langdetect.ngram.NgramExtractors; +import com.optimaize.langdetect.profiles.LanguageProfile; +import com.optimaize.langdetect.profiles.LanguageProfileReader; +import com.optimaize.langdetect.text.CommonTextObjectFactories; +import com.optimaize.langdetect.text.TextObject; +import com.optimaize.langdetect.text.TextObjectFactory; import com.yahoo.language.Language; import com.yahoo.language.detect.Detection; import com.yahoo.language.detect.Detector; import com.yahoo.language.detect.Hint; import com.yahoo.text.Utf8; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Locale; /** - * Includes functionality for determining the langCode from a sample or from the encoding. Currently only Chinese, - * Japanese and Korean are supported. There are two ways to guess a String's langCode, by encoding and by character + * Includes functionality for determining the langCode from a sample or from the encoding. + * There are two ways to guess a String's langCode, by encoding and by character * set. If the encoding is available this is a very good indication of the langCode. If the encoding is not available, * then the actual characters in the string can be used to make an educated guess at the String's langCode. Recall a * String in Java is unicode. Therefore, we can simply look at the unicode blocks of the characters in the string. @@ -21,8 +34,40 @@ import java.nio.ByteBuffer; * character blocks, so if there are no definitive signs of Japanese then it is assumed that the String is Chinese. * * @author Rich Pito + * @author bjorncs */ public class SimpleDetector implements Detector { + static private TextObjectFactory textObjectFactory; + static private LanguageDetector languageDetector; + + static { + // origin: https://github.com/optimaize/language-detector + //load all languages: + List<LanguageProfile> languageProfiles; + try { + languageProfiles = new LanguageProfileReader().readAllBuiltIn(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + //build language detector: + languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) + .withProfiles(languageProfiles) + .build(); + + //create a text object factory + textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); + } + + private final boolean enableOptimaize; + + public SimpleDetector() { + this.enableOptimaize = true; + } + + public SimpleDetector(SimpleLinguisticsConfig.Detector detector) { + this.enableOptimaize = detector.enableOptimaize(); + } @Override public Detection detect(byte[] input, int offset, int length, Hint hint) { @@ -41,11 +86,11 @@ public class SimpleDetector implements Detector { return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); } - public static Language guessLanguage(byte[] buf, int offset, int length) { + public Language guessLanguage(byte[] buf, int offset, int length) { return guessLanguage(Utf8.toString(buf, offset, length)); } - public static Language guessLanguage(String input) { + public Language guessLanguage(String input) { if (input == null || input.length() == 0) { return Language.UNKNOWN; } @@ -109,10 +154,26 @@ public class SimpleDetector implements Detector { return Language.THAI; } } + if (enableOptimaize && Language.UNKNOWN.equals(soFar)){ + return detectLangOptimaize(input); + } // got to the end, so return the current best guess return soFar; } + private static Language detectLangOptimaize(String input) { + if (input == null || input.length() == 0) { + return Language.UNKNOWN; + } + TextObject textObject = textObjectFactory.forText(input); + Optional<LdLocale> lang = languageDetector.detect(textObject); + if (lang.isPresent()) { + String language = lang.get().getLanguage(); + return Language.fromLocale(new Locale(language)); + } + return Language.UNKNOWN; + } + private boolean isTrailingOctet(byte i) { return ((i >>> 6) & 3) == 2; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index ad855a18088..cdfd5b4cb58 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; +import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.language.Linguistics; @@ -19,15 +20,35 @@ import com.yahoo.language.process.Transformer; * Factory of pure Java linguistic processor implementations. * * @author bratseth + * @author bjorncs */ public class SimpleLinguistics implements Linguistics { // Threadsafe instances - private final static Normalizer normalizer = new SimpleNormalizer(); - private final static Transformer transformer = new SimpleTransformer(); - private final static Detector detector = new SimpleDetector(); - private final static CharacterClasses characterClasses = new CharacterClasses(); - private final static GramSplitter gramSplitter = new GramSplitter(characterClasses); + private final Normalizer normalizer; + private final Transformer transformer; + private final Detector detector; + private final CharacterClasses characterClasses; + private final GramSplitter gramSplitter; + + @Inject + public SimpleLinguistics() { + CharacterClasses characterClasses = new CharacterClasses(); + this.normalizer = new SimpleNormalizer(); + this.transformer = new SimpleTransformer(); + this.detector = new SimpleDetector(); + this.characterClasses = new CharacterClasses(); + this.gramSplitter = new GramSplitter(characterClasses); + } + + public SimpleLinguistics(SimpleLinguisticsConfig config) { + CharacterClasses characterClasses = new CharacterClasses(); + this.normalizer = new SimpleNormalizer(); + this.transformer = new SimpleTransformer(); + this.detector = new SimpleDetector(config.detector()); + this.characterClasses = new CharacterClasses(); + this.gramSplitter = new GramSplitter(characterClasses); + } @Override public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); } |