diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2021-12-20 08:59:09 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2021-12-20 08:59:09 +0100 |
commit | 2994e2a0f96e87d9428a156fe3263efab1f7a654 (patch) | |
tree | c3a567beeaeb47ddb1ee89d5c80f4fd448987505 /linguistics | |
parent | a744d811f50fb521c89810a705597b059348543f (diff) |
Revert "Merge pull request #20578 from vespa-engine/revert-20568-jonmv/replace-optimaize-with-lingua"
This reverts commit 5476504932cd90eb2dad82dbab633e3ffa2034c3, reversing
changes made to 235a78cc4707f78d18c6818a577de1b7507f5e40.
Diffstat (limited to 'linguistics')
13 files changed, 246 insertions, 176 deletions
diff --git a/linguistics/pom.xml b/linguistics/pom.xml index a09f2ecb031..d9ab942a0b8 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -22,6 +22,7 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -61,10 +62,6 @@ <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java new file mode 100644 index 00000000000..aa4387bcc45 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator; +import opennlp.tools.langdetect.LanguageDetectorContextGenerator; +import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer; +import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer; +import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer; +import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer; + +/** + * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 + * + * @author jonmv + */ +@SuppressWarnings("unused") // Loaded by black magic. +public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { + + @Override + public LanguageDetectorContextGenerator getContextGenerator() { + return new DefaultLanguageDetectorContextGenerator(1, 3, + EmojiCharSequenceNormalizer.getInstance(), + UrlCharSequenceNormalizer.getInstance(), + TwitterCharSequenceNormalizer.getInstance(), + NumberCharSequenceNormalizer.getInstance(), + ShrinkCharSequenceNormalizer.getInstance()); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java new file mode 100644 index 00000000000..849452aeafd --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detection; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.detect.Hint; +import com.yahoo.language.simple.SimpleDetector; +import opennlp.tools.langdetect.LanguageDetectorConfig; +import opennlp.tools.langdetect.LanguageDetectorME; +import opennlp.tools.langdetect.LanguageDetectorModel; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Detects the language of some sample text using {@link SimpleDetector} for CJK input, and OpenNLP otherwise. + * + * @author jonmv + */ +class OpenNlpDetector implements Detector { + + private static final Object monitor = new Object(); + private static LanguageDetectorModel model; + + private final SimpleDetector simple = new SimpleDetector(); + private final Map<String, Language> languagesByISO3 = new HashMap<>(); + private final LanguageDetectorME detector; + private final LanguageDetectorConfig config; + + OpenNlpDetector() { + detector = new LanguageDetectorME(loadModel()); + config = new LanguageDetectorConfig(); + config.setMinDiff(0.02); + config.setChunkSize(64); + for (Locale locale : Locale.getAvailableLocales()) + languagesByISO3.put(locale.getISO3Language(), Language.fromLocale(locale)); + } + + private static LanguageDetectorModel loadModel() { + synchronized (monitor) { + if (model == null) { + try { + model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin")); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return model; + } + + @Override + public Detection detect(byte[] input, int offset, int length, Hint hint) { + Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length)); + return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false); + } + + @Override + public Detection detect(ByteBuffer input, Hint hint) { + if (input.hasArray()) + return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint); + + byte[] buffer = new byte[input.remaining()]; + input.get(buffer); + return detect(buffer, 0, buffer.length, hint); + } + + @Override + public Detection detect(String input, Hint hint) { + return new Detection(detectLanguage(input), UTF_8.name(), false); + } + + private Language detectLanguage(String input) { + Language simpleGuess = simple.guessLanguage(input); + if (simpleGuess != Language.UNKNOWN) + return simpleGuess; + + var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0]; + return prediction.getConfidence() > 0.03 ? languagesByISO3.getOrDefault(prediction.getLang(), Language.UNKNOWN) + : Language.UNKNOWN; + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index a27e726cda8..c749679024a 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -7,36 +7,21 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; -import java.util.logging.Logger; -import java.util.logging.Level; +import opennlp.tools.langdetect.LanguageDetectorModel; /** - * Returns a linguistics implementation based on OpenNlp, - * and (optionally, default on) Optimaize for language detection. + * Returns a linguistics implementation based on OpenNlp. * * @author bratseth + * @author jonmv */ public class OpenNlpLinguistics extends SimpleLinguistics { - private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; - public OpenNlpLinguistics() { - this(true); - } - @Inject - public OpenNlpLinguistics(OpennlpLinguisticsConfig config) { - this(config.detector().enableOptimaize()); - } - - public OpenNlpLinguistics(boolean enableOptimaize) { - this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); - log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); - } - - private OpenNlpLinguistics(Detector detector) { - this.detector = detector; + public OpenNlpLinguistics() { + this.detector = new OpenNlpDetector(); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java deleted file mode 100644 index 83947c795fb..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.google.common.base.Optional; -import com.optimaize.langdetect.LanguageDetector; -import com.optimaize.langdetect.LanguageDetectorBuilder; -import com.optimaize.langdetect.i18n.LdLocale; -import com.optimaize.langdetect.ngram.NgramExtractors; -import com.optimaize.langdetect.profiles.LanguageProfile; -import com.optimaize.langdetect.profiles.LanguageProfileReader; -import com.optimaize.langdetect.text.CommonTextObjectFactories; -import com.optimaize.langdetect.text.TextObjectFactory; -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detection; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.detect.Hint; -import com.yahoo.language.simple.SimpleDetector; -import com.yahoo.text.Utf8; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Locale; -import java.util.logging.Logger; -import java.util.logging.Level; - -/** - * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. - * - * @author bratseth - */ -public class OptimaizeDetector implements Detector { - - private static final Object initGuard = new Object(); - private static TextObjectFactory textObjectFactory = null; - private static LanguageDetector languageDetector = null; - private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); - - static private void initOptimaize() { - synchronized (initGuard) { - if ((textObjectFactory != null) && (languageDetector != null)) return; - - // origin: https://github.com/optimaize/language-detector - // load all languages: - List<LanguageProfile> languageProfiles; - try { - languageProfiles = new LanguageProfileReader().readAllBuiltIn(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - //build language detector: - languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) - .withProfiles(languageProfiles) - .build(); - - //create a text object factory - textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); - } - } - - private final SimpleDetector simpleDetector = new SimpleDetector(); - - public OptimaizeDetector() { - initOptimaize(); - } - - @Override - public Detection detect(byte[] input, int offset, int length, Hint hint) { - return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false); - } - - @Override - public Detection detect(ByteBuffer input, Hint hint) { - byte[] buf = new byte[input.remaining()]; - input.get(buf, 0, buf.length); - return detect(buf, 0, buf.length, hint); - } - - @Override - public Detection detect(String input, Hint hint) { - return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); - } - - private Language guessLanguage(byte[] buf, int offset, int length) { - return guessLanguage(Utf8.toString(buf, offset, length)); - } - - public Language guessLanguage(String input) { - if (input == null || input.length() == 0) return Language.UNKNOWN; - - Language result = simpleDetector.guessLanguage(input); - if (result != Language.UNKNOWN) return result; - - return guessLanguageUsingOptimaize(input); - } - - private static Language guessLanguageUsingOptimaize(String input) { - Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input)); - if ( ! result.isPresent()) return Language.UNKNOWN; - log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); - - return Language.fromLocale(new Locale(result.get().getLanguage())); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java new file mode 100644 index 00000000000..883319e2f8b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.regex.Pattern; + +/** + * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. + * + * @author jonmv + */ +public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { + + private static final Pattern URL_REGEX = + Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); + private static final Pattern MAIL_REGEX = + Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); + + private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); + + public static UrlCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + public CharSequence normalize(CharSequence text) { + String modified = URL_REGEX.matcher(text).replaceAll(" "); + return MAIL_REGEX.matcher(modified).replaceAll(" "); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 53b8ad7ad70..61d446cd8d0 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -130,10 +130,14 @@ public class SimpleDetector implements Detector { } public String guessEncoding(byte[] input) { + return guessEncoding(input, 0, input.length); + } + + public String guessEncoding(byte[] input, int offset, int length) { boolean isUtf8 = true; boolean hasHighs = false; scan: - for (int i = 0; i < input.length; i++) { + for (int i = offset; i < offset + length; i++) { final int l = isLeadingFor(input[i]); if (l < 0 || i + l >= input.length) { hasHighs = true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 3ca46dcc4f1..b10beb8c9af 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -2,8 +2,7 @@ package com.yahoo.language.simple; import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.Version; +import com.yahoo.component.AbstractComponent; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.CharacterClasses; @@ -16,7 +15,6 @@ import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import java.util.List; diff --git a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def b/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def deleted file mode 100644 index 361a8a5f50c..00000000000 --- a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=language.opennlp - -# Enable Optimaize language detector -detector.enableOptimaize bool default=true - diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin Binary files differnew file mode 100644 index 00000000000..c3cde217050 --- /dev/null +++ b/linguistics/src/main/resources/models/langdetect-183.bin diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java new file mode 100644 index 00000000000..aaa6b2a6484 --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detector; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class OpenNlpDetectorTestCase { + + @Test + public void testDetection() { + Detector detector = new OpenNlpDetector(); + + assertLanguage(Language.UNKNOWN, + "", + detector); + + assertLanguage(Language.UNKNOWN, + "Hello!", + detector); + + // from https://en.wikipedia.org/wiki/Yahoo + assertLanguage(Language.ENGLISH, + "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.", + detector); + + // from https://de.wikipedia.org/wiki/Yahoo + assertLanguage(Language.GERMAN, + "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.", + detector); + + // from https://fr.wikipedia.org/wiki/Yahoo + assertLanguage(Language.FRENCH, + "À l'origine, Yahoo! était uniquement un annuaire Web.", + detector); + + // Test fallback to SimpleDetector + assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input + "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002", + detector); + + // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F + assertLanguage(Language.RUSSIAN, + "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.", + detector); + + // https://he.wikipedia.org/wiki/Yahoo! + assertLanguage(Language.HEBREW, + "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום", + detector); + } + + private void assertLanguage(Language language, String input, Detector detector) { + assertEquals(language, detector.detect(input, null).getLanguage()); + } + +} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java deleted file mode 100644 index 20b5de3b165..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detector; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class OptimaizeDetectorTestCase { - - private static final Detector detector = new OptimaizeDetector(); - - @Test - public void testDetection() { - assertLanguage(Language.UNKNOWN, "Hello!"); - - // Test fallback to SimpleDetector - assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input - "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002"); - - // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F - assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии"); - // https://he.wikipedia.org/wiki/Yahoo! - assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום"); - } - - private static void assertLanguage(Language language, String input) { - assertEquals(language, detector.detect(input, null).getLanguage()); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java new file mode 100644 index 00000000000..a8c637bc6ec --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class UrlCharSequenceNormalizerTest { + + @Test + public void testNormalization() { + String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo"; + assertEquals(" foo _bax a@b § ", + UrlCharSequenceNormalizer.getInstance().normalize(text)); + } + +} |