Re-add files

5 files changed, 142 insertions, 0 deletions

diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
new file mode 100644
index 00000000000..aa4387bcc45
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator;
+import opennlp.tools.langdetect.LanguageDetectorContextGenerator;
+import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer;
+
+/**
+ * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350
+ *
+ * @author jonmv
+ */
+@SuppressWarnings("unused") // Loaded by black magic.
+public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory {
+
+    @Override
+    public LanguageDetectorContextGenerator getContextGenerator() {
+        return new DefaultLanguageDetectorContextGenerator(1, 3,
+                                                           EmojiCharSequenceNormalizer.getInstance(),
+                                                           UrlCharSequenceNormalizer.getInstance(),
+                                                           TwitterCharSequenceNormalizer.getInstance(),
+                                                           NumberCharSequenceNormalizer.getInstance(),
+                                                           ShrinkCharSequenceNormalizer.getInstance());
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
new file mode 100644
index 00000000000..883319e2f8b
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
@@ -0,0 +1,31 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+
+import java.util.regex.Pattern;
+
+/**
+ * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex.
+ *
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
+
+    private static final Pattern URL_REGEX =
+            Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
+    private static final Pattern MAIL_REGEX =
+            Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+    private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer();
+
+    public static UrlCharSequenceNormalizer getInstance() {
+        return INSTANCE;
+    }
+
+    public CharSequence normalize(CharSequence text) {
+        String modified = URL_REGEX.matcher(text).replaceAll(" ");
+        return MAIL_REGEX.matcher(modified).replaceAll(" ");
+    }
+
+}
diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin
new file mode 100644
index 00000000000..c3cde217050
--- /dev/null
+++ b/linguistics/src/main/resources/models/langdetect-183.bin
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
new file mode 100644
index 00000000000..aaa6b2a6484
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
@@ -0,0 +1,62 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class OpenNlpDetectorTestCase {
+
+    @Test
+    public void testDetection() {
+        Detector detector = new OpenNlpDetector();
+
+        assertLanguage(Language.UNKNOWN,
+                       "",
+                       detector);
+
+        assertLanguage(Language.UNKNOWN,
+                       "Hello!",
+                       detector);
+
+        // from https://en.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.ENGLISH,
+                       "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.",
+                       detector);
+
+        // from https://de.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.GERMAN,
+                       "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.",
+                       detector);
+
+        // from https://fr.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.FRENCH,
+                       "À l'origine, Yahoo! était uniquement un annuaire Web.",
+                       detector);
+
+        // Test fallback to SimpleDetector
+        assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input
+                       "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002",
+                       detector);
+
+        // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F
+        assertLanguage(Language.RUSSIAN,
+                       "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.",
+                       detector);
+
+        // https://he.wikipedia.org/wiki/Yahoo!
+        assertLanguage(Language.HEBREW,
+                       "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום",
+                       detector);
+    }
+
+    private void assertLanguage(Language language, String input, Detector detector) {
+        assertEquals(language, detector.detect(input, null).getLanguage());
+    }
+
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
new file mode 100644
index 00000000000..a8c637bc6ec
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
@@ -0,0 +1,20 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizerTest {
+
+    @Test
+    public void testNormalization() {
+        String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo";
+        assertEquals("  foo  _bax   a@b § ",
+                     UrlCharSequenceNormalizer.getInstance().normalize(text));
+    }
+
+}