Revert "Merge pull request #20578 from vespa-engine/revert-20568-jonmv/replace-optimaize-with-lingua"

This reverts commit 5476504932cd90eb2dad82dbab633e3ffa2034c3, reversing
changes made to 235a78cc4707f78d18c6818a577de1b7507f5e40.

13 files changed, 246 insertions, 176 deletions

diff --git a/linguistics/pom.xml b/linguistics/pom.xml
index a09f2ecb031..d9ab942a0b8 100644
--- a/linguistics/pom.xml
+++ b/linguistics/pom.xml
@@ -22,6 +22,7 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
@@ -61,10 +62,6 @@
       <groupId>org.apache.opennlp</groupId>
       <artifactId>opennlp-tools</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.optimaize.languagedetector</groupId>
-      <artifactId>language-detector</artifactId>
-    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
new file mode 100644
index 00000000000..aa4387bcc45
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator;
+import opennlp.tools.langdetect.LanguageDetectorContextGenerator;
+import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer;
+
+/**
+ * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350
+ *
+ * @author jonmv
+ */
+@SuppressWarnings("unused") // Loaded by black magic.
+public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory {
+
+    @Override
+    public LanguageDetectorContextGenerator getContextGenerator() {
+        return new DefaultLanguageDetectorContextGenerator(1, 3,
+                                                           EmojiCharSequenceNormalizer.getInstance(),
+                                                           UrlCharSequenceNormalizer.getInstance(),
+                                                           TwitterCharSequenceNormalizer.getInstance(),
+                                                           NumberCharSequenceNormalizer.getInstance(),
+                                                           ShrinkCharSequenceNormalizer.getInstance());
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
new file mode 100644
index 00000000000..849452aeafd
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
@@ -0,0 +1,92 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detection;
+import com.yahoo.language.detect.Detector;
+import com.yahoo.language.detect.Hint;
+import com.yahoo.language.simple.SimpleDetector;
+import opennlp.tools.langdetect.LanguageDetectorConfig;
+import opennlp.tools.langdetect.LanguageDetectorME;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Detects the language of some sample text using {@link SimpleDetector} for CJK input, and OpenNLP otherwise.
+ *
+ * @author jonmv
+ */
+class OpenNlpDetector implements Detector {
+
+    private static final Object monitor = new Object();
+    private static LanguageDetectorModel model;
+
+    private final SimpleDetector simple = new SimpleDetector();
+    private final Map<String, Language> languagesByISO3 = new HashMap<>();
+    private final LanguageDetectorME detector;
+    private final LanguageDetectorConfig config;
+
+    OpenNlpDetector() {
+        detector = new LanguageDetectorME(loadModel());
+        config = new LanguageDetectorConfig();
+        config.setMinDiff(0.02);
+        config.setChunkSize(64);
+        for (Locale locale : Locale.getAvailableLocales())
+            languagesByISO3.put(locale.getISO3Language(), Language.fromLocale(locale));
+    }
+
+    private static LanguageDetectorModel loadModel() {
+        synchronized (monitor) {
+            if (model == null) {
+                try {
+                    model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin"));
+                }
+                catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+        return model;
+    }
+
+    @Override
+    public Detection detect(byte[] input, int offset, int length, Hint hint) {
+        Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length));
+        return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false);
+    }
+
+    @Override
+    public Detection detect(ByteBuffer input, Hint hint) {
+        if (input.hasArray())
+            return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint);
+
+        byte[] buffer = new byte[input.remaining()];
+        input.get(buffer);
+        return detect(buffer, 0, buffer.length, hint);
+    }
+
+    @Override
+    public Detection detect(String input, Hint hint) {
+        return new Detection(detectLanguage(input), UTF_8.name(), false);
+    }
+
+    private Language detectLanguage(String input) {
+        Language simpleGuess = simple.guessLanguage(input);
+        if (simpleGuess != Language.UNKNOWN)
+            return simpleGuess;
+
+        var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0];
+        return prediction.getConfidence() > 0.03 ? languagesByISO3.getOrDefault(prediction.getLang(), Language.UNKNOWN)
+                                                 : Language.UNKNOWN;
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
index a27e726cda8..c749679024a 100644
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
@@ -7,36 +7,21 @@ import com.yahoo.language.detect.Detector;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.simple.SimpleDetector;
 import com.yahoo.language.simple.SimpleLinguistics;
-import java.util.logging.Logger;
-import java.util.logging.Level;
+import opennlp.tools.langdetect.LanguageDetectorModel;
 
 /**
- * Returns a linguistics implementation based on OpenNlp,
- * and (optionally, default on) Optimaize for language detection.
+ * Returns a linguistics implementation based on OpenNlp.
  *
  * @author bratseth
+ * @author jonmv
  */
 public class OpenNlpLinguistics extends SimpleLinguistics {
 
-    private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName());
     private final Detector detector;
 
-    public OpenNlpLinguistics() {
-        this(true);
-    }
-
     @Inject
-    public OpenNlpLinguistics(OpennlpLinguisticsConfig config) {
-        this(config.detector().enableOptimaize());
-    }
-
-    public OpenNlpLinguistics(boolean enableOptimaize) {
-        this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector());
-        log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector");
-    }
-
-    private OpenNlpLinguistics(Detector detector) {
-        this.detector = detector;
+    public OpenNlpLinguistics() {
+        this.detector = new OpenNlpDetector();
     }
 
     @Override
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java
deleted file mode 100644
index 83947c795fb..00000000000
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.opennlp;
-
-import com.google.common.base.Optional;
-import com.optimaize.langdetect.LanguageDetector;
-import com.optimaize.langdetect.LanguageDetectorBuilder;
-import com.optimaize.langdetect.i18n.LdLocale;
-import com.optimaize.langdetect.ngram.NgramExtractors;
-import com.optimaize.langdetect.profiles.LanguageProfile;
-import com.optimaize.langdetect.profiles.LanguageProfileReader;
-import com.optimaize.langdetect.text.CommonTextObjectFactories;
-import com.optimaize.langdetect.text.TextObjectFactory;
-import com.yahoo.language.Language;
-import com.yahoo.language.detect.Detection;
-import com.yahoo.language.detect.Detector;
-import com.yahoo.language.detect.Hint;
-import com.yahoo.language.simple.SimpleDetector;
-import com.yahoo.text.Utf8;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Locale;
-import java.util.logging.Logger;
-import java.util.logging.Level;
-
-/**
- * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise.
- *
- * @author bratseth
- */
-public class OptimaizeDetector implements Detector {
-
-    private static final Object initGuard = new Object();
-    private static TextObjectFactory textObjectFactory = null;
-    private static LanguageDetector languageDetector = null;
-    private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName());
-
-    static private void initOptimaize() {
-        synchronized (initGuard) {
-            if ((textObjectFactory != null) && (languageDetector != null)) return;
-
-            // origin: https://github.com/optimaize/language-detector
-            // load all languages:
-            List<LanguageProfile> languageProfiles;
-            try {
-                languageProfiles = new LanguageProfileReader().readAllBuiltIn();
-            } catch (IOException e) {
-                throw new UncheckedIOException(e);
-            }
-
-            //build language detector:
-            languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
-                                                      .withProfiles(languageProfiles)
-                                                      .build();
-
-            //create a text object factory
-            textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
-        }
-    }
-
-    private final SimpleDetector simpleDetector = new SimpleDetector();
-
-    public OptimaizeDetector() {
-        initOptimaize();
-    }
-
-    @Override
-    public Detection detect(byte[] input, int offset, int length, Hint hint) {
-        return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false);
-    }
-
-    @Override
-    public Detection detect(ByteBuffer input, Hint hint) {
-        byte[] buf = new byte[input.remaining()];
-        input.get(buf, 0, buf.length);
-        return detect(buf, 0, buf.length, hint);
-    }
-
-    @Override
-    public Detection detect(String input, Hint hint) {
-        return new Detection(guessLanguage(input), Utf8.getCharset().name(), false);
-    }
-
-    private Language guessLanguage(byte[] buf, int offset, int length) {
-        return guessLanguage(Utf8.toString(buf, offset, length));
-    }
-
-    public Language guessLanguage(String input) {
-        if (input == null || input.length() == 0) return Language.UNKNOWN;
-
-        Language result = simpleDetector.guessLanguage(input);
-        if (result != Language.UNKNOWN) return result;
-
-        return guessLanguageUsingOptimaize(input);
-    }
-
-    private static Language guessLanguageUsingOptimaize(String input) {
-        Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input));
-        if ( ! result.isPresent()) return Language.UNKNOWN;
-        log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input);
-
-        return Language.fromLocale(new Locale(result.get().getLanguage()));
-    }
-
-}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
new file mode 100644
index 00000000000..883319e2f8b
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
@@ -0,0 +1,31 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+
+import java.util.regex.Pattern;
+
+/**
+ * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex.
+ *
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
+
+    private static final Pattern URL_REGEX =
+            Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
+    private static final Pattern MAIL_REGEX =
+            Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+    private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer();
+
+    public static UrlCharSequenceNormalizer getInstance() {
+        return INSTANCE;
+    }
+
+    public CharSequence normalize(CharSequence text) {
+        String modified = URL_REGEX.matcher(text).replaceAll(" ");
+        return MAIL_REGEX.matcher(modified).replaceAll(" ");
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
index 53b8ad7ad70..61d446cd8d0 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
@@ -130,10 +130,14 @@ public class SimpleDetector implements Detector {
     }
 
     public String guessEncoding(byte[] input) {
+        return guessEncoding(input, 0, input.length);
+    }
+
+    public String guessEncoding(byte[] input, int offset, int length) {
         boolean isUtf8 = true;
         boolean hasHighs = false;
         scan:
-        for (int i = 0; i < input.length; i++) {
+        for (int i = offset; i < offset + length; i++) {
             final int l = isLeadingFor(input[i]);
             if (l < 0 || i + l >= input.length) {
                 hasHighs = true;
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index 3ca46dcc4f1..b10beb8c9af 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -2,8 +2,7 @@
 package com.yahoo.language.simple;
 
 import com.google.inject.Inject;
-import com.yahoo.collections.Tuple2;
-import com.yahoo.component.Version;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.detect.Detector;
 import com.yahoo.language.process.CharacterClasses;
@@ -16,7 +15,6 @@ import com.yahoo.language.process.Stemmer;
 import com.yahoo.language.process.StemmerImpl;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.process.Transformer;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 
 import java.util.List;
 
diff --git a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def b/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def
deleted file mode 100644
index 361a8a5f50c..00000000000
--- a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def
+++ /dev/null
@@ -1,6 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-namespace=language.opennlp
-
-# Enable Optimaize language detector
-detector.enableOptimaize bool  default=true
-
diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin
new file mode 100644
index 00000000000..c3cde217050
--- /dev/null
+++ b/linguistics/src/main/resources/models/langdetect-183.bin
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
new file mode 100644
index 00000000000..aaa6b2a6484
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
@@ -0,0 +1,62 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class OpenNlpDetectorTestCase {
+
+    @Test
+    public void testDetection() {
+        Detector detector = new OpenNlpDetector();
+
+        assertLanguage(Language.UNKNOWN,
+                       "",
+                       detector);
+
+        assertLanguage(Language.UNKNOWN,
+                       "Hello!",
+                       detector);
+
+        // from https://en.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.ENGLISH,
+                       "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.",
+                       detector);
+
+        // from https://de.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.GERMAN,
+                       "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.",
+                       detector);
+
+        // from https://fr.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.FRENCH,
+                       "À l'origine, Yahoo! était uniquement un annuaire Web.",
+                       detector);
+
+        // Test fallback to SimpleDetector
+        assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input
+                       "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002",
+                       detector);
+
+        // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F
+        assertLanguage(Language.RUSSIAN,
+                       "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.",
+                       detector);
+
+        // https://he.wikipedia.org/wiki/Yahoo!
+        assertLanguage(Language.HEBREW,
+                       "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום",
+                       detector);
+    }
+
+    private void assertLanguage(Language language, String input, Detector detector) {
+        assertEquals(language, detector.detect(input, null).getLanguage());
+    }
+
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java
deleted file mode 100644
index 20b5de3b165..00000000000
--- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.opennlp;
-
-import com.yahoo.language.Language;
-import com.yahoo.language.detect.Detector;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class OptimaizeDetectorTestCase {
-
-    private static final Detector detector = new OptimaizeDetector();
-
-    @Test
-    public void testDetection() {
-        assertLanguage(Language.UNKNOWN, "Hello!");
-
-        // Test fallback to SimpleDetector
-        assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input
-                       "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002");
-
-        // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F
-        assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии");
-        // https://he.wikipedia.org/wiki/Yahoo!
-        assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום");
-    }
-
-    private static void assertLanguage(Language language, String input) {
-        assertEquals(language, detector.detect(input, null).getLanguage());
-    }
-
-}
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
new file mode 100644
index 00000000000..a8c637bc6ec
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
@@ -0,0 +1,20 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizerTest {
+
+    @Test
+    public void testNormalization() {
+        String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo";
+        assertEquals("  foo  _bax   a@b § ",
+                     UrlCharSequenceNormalizer.getInstance().normalize(text));
+    }
+
+}