diff options
Diffstat (limited to 'linguistics/src/main/java/com')
7 files changed, 131 insertions, 163 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java deleted file mode 100644 index aa4387bcc45..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator; -import opennlp.tools.langdetect.LanguageDetectorContextGenerator; -import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer; -import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer; -import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer; -import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer; - -/** - * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 - * - * @author jonmv - */ -@SuppressWarnings("unused") // Loaded by black magic. -public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { - - @Override - public LanguageDetectorContextGenerator getContextGenerator() { - return new DefaultLanguageDetectorContextGenerator(1, 3, - EmojiCharSequenceNormalizer.getInstance(), - UrlCharSequenceNormalizer.getInstance(), - TwitterCharSequenceNormalizer.getInstance(), - NumberCharSequenceNormalizer.getInstance(), - ShrinkCharSequenceNormalizer.getInstance()); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java deleted file mode 100644 index 849452aeafd..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detection; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.detect.Hint; -import com.yahoo.language.simple.SimpleDetector; -import opennlp.tools.langdetect.LanguageDetectorConfig; -import opennlp.tools.langdetect.LanguageDetectorME; -import opennlp.tools.langdetect.LanguageDetectorModel; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Detects the language of some sample text using {@link SimpleDetector} for CJK input, and OpenNLP otherwise. - * - * @author jonmv - */ -class OpenNlpDetector implements Detector { - - private static final Object monitor = new Object(); - private static LanguageDetectorModel model; - - private final SimpleDetector simple = new SimpleDetector(); - private final Map<String, Language> languagesByISO3 = new HashMap<>(); - private final LanguageDetectorME detector; - private final LanguageDetectorConfig config; - - OpenNlpDetector() { - detector = new LanguageDetectorME(loadModel()); - config = new LanguageDetectorConfig(); - config.setMinDiff(0.02); - config.setChunkSize(64); - for (Locale locale : Locale.getAvailableLocales()) - languagesByISO3.put(locale.getISO3Language(), Language.fromLocale(locale)); - } - - private static LanguageDetectorModel loadModel() { - synchronized (monitor) { - if (model == null) { - try { - model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin")); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - return model; - } - - @Override - public Detection detect(byte[] input, int offset, int length, Hint hint) { - Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length)); - return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false); - } - - @Override - public Detection detect(ByteBuffer input, Hint hint) { - if (input.hasArray()) - return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint); - - byte[] buffer = new byte[input.remaining()]; - input.get(buffer); - return detect(buffer, 0, buffer.length, hint); - } - - @Override - public Detection detect(String input, Hint hint) { - return new Detection(detectLanguage(input), UTF_8.name(), false); - } - - private Language detectLanguage(String input) { - Language simpleGuess = simple.guessLanguage(input); - if (simpleGuess != Language.UNKNOWN) - return simpleGuess; - - var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0]; - return prediction.getConfidence() > 0.03 ? languagesByISO3.getOrDefault(prediction.getLang(), Language.UNKNOWN) - : Language.UNKNOWN; - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index c749679024a..a27e726cda8 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -7,21 +7,36 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; -import opennlp.tools.langdetect.LanguageDetectorModel; +import java.util.logging.Logger; +import java.util.logging.Level; /** - * Returns a linguistics implementation based on OpenNlp. + * Returns a linguistics implementation based on OpenNlp, + * and (optionally, default on) Optimaize for language detection. * * @author bratseth - * @author jonmv */ public class OpenNlpLinguistics extends SimpleLinguistics { + private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; - @Inject public OpenNlpLinguistics() { - this.detector = new OpenNlpDetector(); + this(true); + } + + @Inject + public OpenNlpLinguistics(OpennlpLinguisticsConfig config) { + this(config.detector().enableOptimaize()); + } + + public OpenNlpLinguistics(boolean enableOptimaize) { + this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); + log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); + } + + private OpenNlpLinguistics(Detector detector) { + this.detector = detector; } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java new file mode 100644 index 00000000000..83947c795fb --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -0,0 +1,107 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.google.common.base.Optional; +import com.optimaize.langdetect.LanguageDetector; +import com.optimaize.langdetect.LanguageDetectorBuilder; +import com.optimaize.langdetect.i18n.LdLocale; +import com.optimaize.langdetect.ngram.NgramExtractors; +import com.optimaize.langdetect.profiles.LanguageProfile; +import com.optimaize.langdetect.profiles.LanguageProfileReader; +import com.optimaize.langdetect.text.CommonTextObjectFactories; +import com.optimaize.langdetect.text.TextObjectFactory; +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detection; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.detect.Hint; +import com.yahoo.language.simple.SimpleDetector; +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Locale; +import java.util.logging.Logger; +import java.util.logging.Level; + +/** + * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. + * + * @author bratseth + */ +public class OptimaizeDetector implements Detector { + + private static final Object initGuard = new Object(); + private static TextObjectFactory textObjectFactory = null; + private static LanguageDetector languageDetector = null; + private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); + + static private void initOptimaize() { + synchronized (initGuard) { + if ((textObjectFactory != null) && (languageDetector != null)) return; + + // origin: https://github.com/optimaize/language-detector + // load all languages: + List<LanguageProfile> languageProfiles; + try { + languageProfiles = new LanguageProfileReader().readAllBuiltIn(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + //build language detector: + languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) + .withProfiles(languageProfiles) + .build(); + + //create a text object factory + textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); + } + } + + private final SimpleDetector simpleDetector = new SimpleDetector(); + + public OptimaizeDetector() { + initOptimaize(); + } + + @Override + public Detection detect(byte[] input, int offset, int length, Hint hint) { + return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false); + } + + @Override + public Detection detect(ByteBuffer input, Hint hint) { + byte[] buf = new byte[input.remaining()]; + input.get(buf, 0, buf.length); + return detect(buf, 0, buf.length, hint); + } + + @Override + public Detection detect(String input, Hint hint) { + return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); + } + + private Language guessLanguage(byte[] buf, int offset, int length) { + return guessLanguage(Utf8.toString(buf, offset, length)); + } + + public Language guessLanguage(String input) { + if (input == null || input.length() == 0) return Language.UNKNOWN; + + Language result = simpleDetector.guessLanguage(input); + if (result != Language.UNKNOWN) return result; + + return guessLanguageUsingOptimaize(input); + } + + private static Language guessLanguageUsingOptimaize(String input) { + Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input)); + if ( ! result.isPresent()) return Language.UNKNOWN; + log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); + + return Language.fromLocale(new Locale(result.get().getLanguage())); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java deleted file mode 100644 index 883319e2f8b..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.util.normalizer.CharSequenceNormalizer; - -import java.util.regex.Pattern; - -/** - * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. - * - * @author jonmv - */ -public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { - - private static final Pattern URL_REGEX = - Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); - private static final Pattern MAIL_REGEX = - Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); - - private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); - - public static UrlCharSequenceNormalizer getInstance() { - return INSTANCE; - } - - public CharSequence normalize(CharSequence text) { - String modified = URL_REGEX.matcher(text).replaceAll(" "); - return MAIL_REGEX.matcher(modified).replaceAll(" "); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 61d446cd8d0..53b8ad7ad70 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -130,14 +130,10 @@ public class SimpleDetector implements Detector { } public String guessEncoding(byte[] input) { - return guessEncoding(input, 0, input.length); - } - - public String guessEncoding(byte[] input, int offset, int length) { boolean isUtf8 = true; boolean hasHighs = false; scan: - for (int i = offset; i < offset + length; i++) { + for (int i = 0; i < input.length; i++) { final int l = isLeadingFor(input[i]); if (l < 0 || i + l >= input.length) { hasHighs = true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index b10beb8c9af..3ca46dcc4f1 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -2,7 +2,8 @@ package com.yahoo.language.simple; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.CharacterClasses; @@ -15,6 +16,7 @@ import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import java.util.List; |