Replace optimaize with OpenNLP language detector

8 files changed, 102 insertions, 170 deletions

diff --git a/linguistics/pom.xml b/linguistics/pom.xml
index a09f2ecb031..c48054911a2 100644
--- a/linguistics/pom.xml
+++ b/linguistics/pom.xml
@@ -61,10 +61,6 @@
       <groupId>org.apache.opennlp</groupId>
       <artifactId>opennlp-tools</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.optimaize.languagedetector</groupId>
-      <artifactId>language-detector</artifactId>
-    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LangDetectModel.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LangDetectModel.java
new file mode 100644
index 00000000000..70664b8f79f
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LangDetectModel.java
@@ -0,0 +1,13 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+/**
+ * @author jonmv
+ */
+public interface LangDetectModel {
+
+    LanguageDetectorModel load();
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
new file mode 100644
index 00000000000..e0c0960b920
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
@@ -0,0 +1,77 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detection;
+import com.yahoo.language.detect.Detector;
+import com.yahoo.language.detect.Hint;
+import com.yahoo.language.simple.SimpleDetector;
+import opennlp.tools.cmdline.langdetect.LanguageDetectorModelLoader;
+import opennlp.tools.langdetect.LanguageDetectorConfig;
+import opennlp.tools.langdetect.LanguageDetectorME;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Detects the language of some sample text using {@link SimpleDetector} for CJK input, and OpenNLP otherwise.
+ *
+ * @author jonmv
+ */
+class OpenNlpDetector implements Detector {
+
+    private final SimpleDetector simple = new SimpleDetector();
+    private final Map<String, Language> languagesByISO3 = new HashMap<>();
+    private final LanguageDetectorME detector;
+    private final LanguageDetectorConfig config;
+
+    OpenNlpDetector(LanguageDetectorModel model) {
+        detector = new LanguageDetectorME(model);
+        config = new LanguageDetectorConfig();
+        config.setMinDiff(0.02);
+        config.setChunkSize(64);
+        for (Locale locale : Locale.getAvailableLocales())
+            languagesByISO3.put(locale.getISO3Language(), Language.fromLocale(locale));
+    }
+
+    @Override
+    public Detection detect(byte[] input, int offset, int length, Hint hint) {
+        Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length));
+        return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false);
+    }
+
+    @Override
+    public Detection detect(ByteBuffer input, Hint hint) {
+        if (input.hasArray())
+            return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint);
+
+        byte[] buffer = new byte[input.remaining()];
+        input.get(buffer);
+        return detect(buffer, 0, buffer.length, hint);
+    }
+
+    @Override
+    public Detection detect(String input, Hint hint) {
+        return new Detection(detectLanguage(input), UTF_8.name(), false);
+    }
+
+    private Language detectLanguage(String input) {
+        Language simpleGuess = simple.guessLanguage(input);
+        if (simpleGuess != Language.UNKNOWN)
+            return simpleGuess;
+
+        var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0];
+        return prediction.getConfidence() > 0.03 ? languagesByISO3.getOrDefault(prediction.getLang(), Language.UNKNOWN)
+                                                 : Language.UNKNOWN;
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
index a27e726cda8..7ee17559d88 100644
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
@@ -7,36 +7,22 @@ import com.yahoo.language.detect.Detector;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.simple.SimpleDetector;
 import com.yahoo.language.simple.SimpleLinguistics;
-import java.util.logging.Logger;
-import java.util.logging.Level;
+import opennlp.tools.langdetect.LanguageDetectorModel;
 
 /**
- * Returns a linguistics implementation based on OpenNlp,
- * and (optionally, default on) Optimaize for language detection.
+ * Returns a linguistics implementation based on OpenNlp.
  *
  * @author bratseth
+ * @author jonmv
  */
 public class OpenNlpLinguistics extends SimpleLinguistics {
 
-    private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName());
     private final Detector detector;
 
-    public OpenNlpLinguistics() {
-        this(true);
-    }
-
     @Inject
-    public OpenNlpLinguistics(OpennlpLinguisticsConfig config) {
-        this(config.detector().enableOptimaize());
-    }
-
-    public OpenNlpLinguistics(boolean enableOptimaize) {
-        this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector());
-        log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector");
-    }
-
-    private OpenNlpLinguistics(Detector detector) {
-        this.detector = detector;
+    public OpenNlpLinguistics(LangDetectModel model) {
+        LanguageDetectorModel loaded = model.load();
+        this.detector = loaded != null ? new OpenNlpDetector(loaded) : new SimpleDetector();
     }
 
     @Override
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java
deleted file mode 100644
index 83947c795fb..00000000000
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.opennlp;
-
-import com.google.common.base.Optional;
-import com.optimaize.langdetect.LanguageDetector;
-import com.optimaize.langdetect.LanguageDetectorBuilder;
-import com.optimaize.langdetect.i18n.LdLocale;
-import com.optimaize.langdetect.ngram.NgramExtractors;
-import com.optimaize.langdetect.profiles.LanguageProfile;
-import com.optimaize.langdetect.profiles.LanguageProfileReader;
-import com.optimaize.langdetect.text.CommonTextObjectFactories;
-import com.optimaize.langdetect.text.TextObjectFactory;
-import com.yahoo.language.Language;
-import com.yahoo.language.detect.Detection;
-import com.yahoo.language.detect.Detector;
-import com.yahoo.language.detect.Hint;
-import com.yahoo.language.simple.SimpleDetector;
-import com.yahoo.text.Utf8;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Locale;
-import java.util.logging.Logger;
-import java.util.logging.Level;
-
-/**
- * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise.
- *
- * @author bratseth
- */
-public class OptimaizeDetector implements Detector {
-
-    private static final Object initGuard = new Object();
-    private static TextObjectFactory textObjectFactory = null;
-    private static LanguageDetector languageDetector = null;
-    private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName());
-
-    static private void initOptimaize() {
-        synchronized (initGuard) {
-            if ((textObjectFactory != null) && (languageDetector != null)) return;
-
-            // origin: https://github.com/optimaize/language-detector
-            // load all languages:
-            List<LanguageProfile> languageProfiles;
-            try {
-                languageProfiles = new LanguageProfileReader().readAllBuiltIn();
-            } catch (IOException e) {
-                throw new UncheckedIOException(e);
-            }
-
-            //build language detector:
-            languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
-                                                      .withProfiles(languageProfiles)
-                                                      .build();
-
-            //create a text object factory
-            textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
-        }
-    }
-
-    private final SimpleDetector simpleDetector = new SimpleDetector();
-
-    public OptimaizeDetector() {
-        initOptimaize();
-    }
-
-    @Override
-    public Detection detect(byte[] input, int offset, int length, Hint hint) {
-        return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false);
-    }
-
-    @Override
-    public Detection detect(ByteBuffer input, Hint hint) {
-        byte[] buf = new byte[input.remaining()];
-        input.get(buf, 0, buf.length);
-        return detect(buf, 0, buf.length, hint);
-    }
-
-    @Override
-    public Detection detect(String input, Hint hint) {
-        return new Detection(guessLanguage(input), Utf8.getCharset().name(), false);
-    }
-
-    private Language guessLanguage(byte[] buf, int offset, int length) {
-        return guessLanguage(Utf8.toString(buf, offset, length));
-    }
-
-    public Language guessLanguage(String input) {
-        if (input == null || input.length() == 0) return Language.UNKNOWN;
-
-        Language result = simpleDetector.guessLanguage(input);
-        if (result != Language.UNKNOWN) return result;
-
-        return guessLanguageUsingOptimaize(input);
-    }
-
-    private static Language guessLanguageUsingOptimaize(String input) {
-        Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input));
-        if ( ! result.isPresent()) return Language.UNKNOWN;
-        log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input);
-
-        return Language.fromLocale(new Locale(result.get().getLanguage()));
-    }
-
-}
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
index 53b8ad7ad70..61d446cd8d0 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
@@ -130,10 +130,14 @@ public class SimpleDetector implements Detector {
     }
 
     public String guessEncoding(byte[] input) {
+        return guessEncoding(input, 0, input.length);
+    }
+
+    public String guessEncoding(byte[] input, int offset, int length) {
         boolean isUtf8 = true;
         boolean hasHighs = false;
         scan:
-        for (int i = 0; i < input.length; i++) {
+        for (int i = offset; i < offset + length; i++) {
             final int l = isLeadingFor(input[i]);
             if (l < 0 || i + l >= input.length) {
                 hasHighs = true;
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index 3ca46dcc4f1..b10beb8c9af 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -2,8 +2,7 @@
 package com.yahoo.language.simple;
 
 import com.google.inject.Inject;
-import com.yahoo.collections.Tuple2;
-import com.yahoo.component.Version;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.detect.Detector;
 import com.yahoo.language.process.CharacterClasses;
@@ -16,7 +15,6 @@ import com.yahoo.language.process.Stemmer;
 import com.yahoo.language.process.StemmerImpl;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.process.Transformer;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 
 import java.util.List;
 
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java
deleted file mode 100644
index 20b5de3b165..00000000000
--- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.opennlp;
-
-import com.yahoo.language.Language;
-import com.yahoo.language.detect.Detector;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class OptimaizeDetectorTestCase {
-
-    private static final Detector detector = new OptimaizeDetector();
-
-    @Test
-    public void testDetection() {
-        assertLanguage(Language.UNKNOWN, "Hello!");
-
-        // Test fallback to SimpleDetector
-        assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input
-                       "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002");
-
-        // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F
-        assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии");
-        // https://he.wikipedia.org/wiki/Yahoo!
-        assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום");
-    }
-
-    private static void assertLanguage(Language language, String input) {
-        assertEquals(language, detector.detect(input, null).getLanguage());
-    }
-
-}