Split out opennlp-linguistics

15 files changed, 863 insertions, 0 deletions

diff --git a/opennlp-linguistics/abi-spec.json b/opennlp-linguistics/abi-spec.json
new file mode 100644
index 00000000000..6f31cf5a2e6
--- /dev/null
+++ b/opennlp-linguistics/abi-spec.json
@@ -0,0 +1 @@
+{ }
\ No newline at end of file
diff --git a/opennlp-linguistics/pom.xml b/opennlp-linguistics/pom.xml
new file mode 100644
index 00000000000..40f1e95f4f4
--- /dev/null
+++ b/opennlp-linguistics/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0"?>
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>parent</artifactId>
+    <version>8-SNAPSHOT</version>
+    <relativePath>../parent/pom.xml</relativePath>
+  </parent>
+  <artifactId>opennlp-linguistics</artifactId>
+  <packaging>container-plugin</packaging>
+  <version>8-SNAPSHOT</version>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>component</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>config-bundle</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>annotations</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>configdefinitions</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>vespajlib</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>linguistics</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <scope>provided</scope>
+      <classifier>no_aop</classifier>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.opennlp</groupId>
+      <artifactId>opennlp-tools</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
+        <extensions>true</extensions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>abi-check-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java
new file mode 100644
index 00000000000..27c23d8d3e6
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java
@@ -0,0 +1,32 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.ngram.NGramCharModel;
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Avoids using the unnecessarily slow {@link NGramCharModel}.
+ *
+ * @author jonmv
+ */
+public class DefaultLanguageDetectorContextGenerator extends opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator {
+
+    public DefaultLanguageDetectorContextGenerator(int minLength, int maxLength, CharSequenceNormalizer... normalizers) {
+        super(minLength, maxLength, normalizers);
+    }
+
+    @Override
+    public String[] getContext(CharSequence document) {
+        int[] normalized = normalizer.normalize(document).codePoints().map(Character::toLowerCase).toArray();
+        Set<String> grams = new HashSet<>();
+        for (int i = 0; i < normalized.length; i++)
+            for (int j = minLength; j <= maxLength && i + j < normalized.length; j++)
+                grams.add(new String(normalized, i, j));
+
+        return grams.toArray(new String[grams.size()]);
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
new file mode 100644
index 00000000000..0cf4634c6c3
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java
@@ -0,0 +1,20 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.langdetect.LanguageDetectorContextGenerator;
+
+/**
+ * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350
+ *
+ * @author jonmv
+ */
+@SuppressWarnings("unused") // Loaded by black magic: specified in properties in the loaded model.
+public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory {
+
+    @Override
+    public LanguageDetectorContextGenerator getContextGenerator() {
+        return new DefaultLanguageDetectorContextGenerator(1, 3,
+                                                           VespaCharSequenceNormalizer.getInstance());
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
new file mode 100644
index 00000000000..d7a7d3a4744
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java
@@ -0,0 +1,92 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detection;
+import com.yahoo.language.detect.Detector;
+import com.yahoo.language.detect.Hint;
+import com.yahoo.language.simple.SimpleDetector;
+import opennlp.tools.langdetect.LanguageDetectorConfig;
+import opennlp.tools.langdetect.LanguageDetectorME;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Detects text language using patched OpenNLP, with fallback to {@link SimpleDetector} for undetected CJK input.
+ *
+ * @author jonmv
+ */
+class OpenNlpDetector implements Detector {
+
+    private static final Object monitor = new Object();
+    private static LanguageDetectorModel model;
+
+    private final SimpleDetector simple = new SimpleDetector();
+    private final Map<String, Language> languagesByISO3 = new HashMap<>();
+    private final LanguageDetectorME detector;
+    private final LanguageDetectorConfig config;
+
+    OpenNlpDetector() {
+        detector = new LanguageDetectorME(loadModel());
+        config = new LanguageDetectorConfig();
+        config.setMinDiff(0.02);
+        config.setChunkSize(32);
+        config.setMaxLength(256);
+        for (Locale locale : Locale.getAvailableLocales()) {
+            Language language = Language.fromLocale(locale);
+            if (language != null)
+                languagesByISO3.put(locale.getISO3Language(), language);
+        }
+    }
+
+    private static LanguageDetectorModel loadModel() {
+        synchronized (monitor) {
+            if (model == null) {
+                try {
+                    model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin"));
+                }
+                catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+        return model;
+    }
+
+    @Override
+    public Detection detect(byte[] input, int offset, int length, Hint hint) {
+        Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length));
+        return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false);
+    }
+
+    @Override
+    public Detection detect(ByteBuffer input, Hint hint) {
+        if (input.hasArray())
+            return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint);
+
+        byte[] buffer = new byte[input.remaining()];
+        input.get(buffer);
+        return detect(buffer, 0, buffer.length, hint);
+    }
+
+    @Override
+    public Detection detect(String input, Hint hint) {
+        return new Detection(detectLanguage(input), UTF_8.name(), false);
+    }
+
+    private Language detectLanguage(String input) {
+        var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0];
+        var result = prediction.getConfidence() > 0.02 ? languagesByISO3.get(prediction.getLang()) : null;
+        return result != null ? result : simple.guessLanguage(input.substring(0, Math.min(input.length(), 256)));
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
new file mode 100644
index 00000000000..1d96d8a0cdf
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
@@ -0,0 +1,36 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.component.annotation.Inject;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.detect.Detector;
+import com.yahoo.language.process.Tokenizer;
+import com.yahoo.language.simple.SimpleLinguistics;
+
+/**
+ * Returns a linguistics implementation based on OpenNlp.
+ *
+ * @author bratseth
+ * @author jonmv
+ */
+public class OpenNlpLinguistics extends SimpleLinguistics {
+
+    private final Detector detector;
+
+    @Inject
+    public OpenNlpLinguistics() {
+        this.detector = new OpenNlpDetector();
+    }
+
+    @Override
+    public Tokenizer getTokenizer() {
+        return new OpenNlpTokenizer(getNormalizer(), getTransformer());
+    }
+
+    @Override
+    public Detector getDetector() { return detector; }
+
+    @Override
+    public boolean equals(Linguistics other) { return (other instanceof OpenNlpLinguistics); }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
new file mode 100644
index 00000000000..8080dc92729
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
@@ -0,0 +1,99 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.LinguisticsCase;
+import com.yahoo.language.process.Normalizer;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.Tokenizer;
+import com.yahoo.language.process.Transformer;
+import com.yahoo.language.simple.SimpleNormalizer;
+import com.yahoo.language.simple.SimpleTokenizer;
+import com.yahoo.language.simple.SimpleTransformer;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import java.util.List;
+
+/**
+ * Tokenizer using OpenNlp
+ *
+ * @author matskin
+ * @author bratseth
+ */
+public class OpenNlpTokenizer implements Tokenizer {
+
+    private final static int SPACE_CODE = 32;
+    private final Normalizer normalizer;
+    private final Transformer transformer;
+    private final SimpleTokenizer simpleTokenizer;
+    private final SpecialTokenRegistry specialTokenRegistry;
+
+    public OpenNlpTokenizer() {
+        this(new SimpleNormalizer(), new SimpleTransformer());
+    }
+
+    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
+        this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
+    }
+
+    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
+        this.normalizer = normalizer;
+        this.transformer = transformer;
+        this.specialTokenRegistry = specialTokenRegistry;
+        this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry);
+    }
+
+    @Override
+    public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
+        Stemmer stemmer = stemmerFor(language, stemMode);
+        if (stemmer == null)
+            return simpleTokenizer.tokenize(input, language, stemMode, removeAccents);
+        else
+            return simpleTokenizer.tokenize(input, token -> processToken(token, language, stemMode, removeAccents, stemmer));
+    }
+
+    private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
+                                Stemmer stemmer) {
+        token = normalizer.normalize(token);
+        token = LinguisticsCase.toLowerCase(token);
+        if (removeAccents)
+            token = transformer.accentDrop(token, language);
+        if (stemMode != StemMode.NONE)
+            token = stemmer.stem(token).toString();
+        return token;
+    }
+
+    private Stemmer stemmerFor(Language language, StemMode stemMode) {
+        if (language == null || language == Language.ENGLISH || stemMode == StemMode.NONE) return null;
+        SnowballStemmer.ALGORITHM algorithm = algorithmFor(language);
+        if (algorithm == null) return null;
+        return new SnowballStemmer(algorithm);
+    }
+
+    private SnowballStemmer.ALGORITHM algorithmFor(Language language) {
+        switch (language) {
+            case DANISH: return SnowballStemmer.ALGORITHM.DANISH;
+            case DUTCH: return SnowballStemmer.ALGORITHM.DUTCH;
+            case FINNISH: return SnowballStemmer.ALGORITHM.FINNISH;
+            case FRENCH: return SnowballStemmer.ALGORITHM.FRENCH;
+            case GERMAN: return SnowballStemmer.ALGORITHM.GERMAN;
+            case HUNGARIAN: return SnowballStemmer.ALGORITHM.HUNGARIAN;
+            case IRISH: return SnowballStemmer.ALGORITHM.IRISH;
+            case ITALIAN: return SnowballStemmer.ALGORITHM.ITALIAN;
+            case NORWEGIAN_BOKMAL: return SnowballStemmer.ALGORITHM.NORWEGIAN;
+            case NORWEGIAN_NYNORSK: return SnowballStemmer.ALGORITHM.NORWEGIAN;
+            case PORTUGUESE: return SnowballStemmer.ALGORITHM.PORTUGUESE;
+            case ROMANIAN: return SnowballStemmer.ALGORITHM.ROMANIAN;
+            case RUSSIAN: return SnowballStemmer.ALGORITHM.RUSSIAN;
+            case SPANISH: return SnowballStemmer.ALGORITHM.SPANISH;
+            case SWEDISH: return SnowballStemmer.ALGORITHM.SWEDISH;
+            case TURKISH: return SnowballStemmer.ALGORITHM.TURKISH;
+            case ENGLISH: return SnowballStemmer.ALGORITHM.ENGLISH;
+            default: return null;
+        }
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
new file mode 100644
index 00000000000..883319e2f8b
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java
@@ -0,0 +1,31 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+
+import java.util.regex.Pattern;
+
+/**
+ * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex.
+ *
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
+
+    private static final Pattern URL_REGEX =
+            Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
+    private static final Pattern MAIL_REGEX =
+            Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+    private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer();
+
+    public static UrlCharSequenceNormalizer getInstance() {
+        return INSTANCE;
+    }
+
+    public CharSequence normalize(CharSequence text) {
+        String modified = URL_REGEX.matcher(text).replaceAll(" ");
+        return MAIL_REGEX.matcher(modified).replaceAll(" ");
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java
new file mode 100644
index 00000000000..df8f3fad520
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+
+import java.util.function.IntConsumer;
+import java.util.stream.IntStream;
+
+/**
+ * Simple normalizer
+ *
+ * @author arnej
+ */
+public class VespaCharSequenceNormalizer implements CharSequenceNormalizer {
+
+    private static final VespaCharSequenceNormalizer INSTANCE = new VespaCharSequenceNormalizer();
+
+    public static VespaCharSequenceNormalizer getInstance() {
+        return INSTANCE;
+    }
+
+    // filter replacing sequences of non-letters with a single space
+    static class OnlyLetters implements IntStream.IntMapMultiConsumer {
+        boolean addSpace = false;
+        public void accept(int codepoint, IntConsumer target) {
+            if (WordCharDetector.isWordChar(codepoint)) {
+                if (addSpace) {
+                    target.accept(' ');
+                    addSpace = false;
+                }
+                target.accept(Character.toLowerCase(codepoint));
+            } else {
+                addSpace = true;
+            }
+        }
+    }
+
+    public CharSequence normalize(CharSequence text) {
+        if (text.isEmpty()) {
+            return text;
+        }
+        var r = text
+                .codePoints()
+                .mapMulti(new OnlyLetters())
+                .collect(StringBuilder::new,
+                         StringBuilder::appendCodePoint,
+                         StringBuilder::append);
+        return r;
+    }
+
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java
new file mode 100644
index 00000000000..d7e3f88ae8d
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java
@@ -0,0 +1,48 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+class WordCharDetector {
+    public static boolean isWordChar(int codepoint) {
+        int unicodeGeneralCategory = Character.getType(codepoint);
+        switch (unicodeGeneralCategory) {
+        case Character.LOWERCASE_LETTER:
+        case Character.OTHER_LETTER:
+        case Character.TITLECASE_LETTER:
+        case Character.UPPERCASE_LETTER:
+        case Character.MODIFIER_LETTER:
+            return true;
+/*
+ * these are the other categories, currently considered non-word-chars:
+ *
+        case Character.CONNECTOR_PUNCTUATION:
+        case Character.CONTROL:
+        case Character.CURRENCY_SYMBOL:
+        case Character.DASH_PUNCTUATION:
+        case Character.ENCLOSING_MARK:
+        case Character.END_PUNCTUATION:
+        case Character.FINAL_QUOTE_PUNCTUATION:
+        case Character.FORMAT:
+        case Character.INITIAL_QUOTE_PUNCTUATION:
+        case Character.MATH_SYMBOL:
+        case Character.MODIFIER_SYMBOL:
+        case Character.NON_SPACING_MARK:
+        case Character.OTHER_PUNCTUATION:
+        case Character.OTHER_SYMBOL:
+        case Character.PRIVATE_USE:
+        case Character.START_PUNCTUATION:
+        case Character.SURROGATE:
+        case Character.UNASSIGNED:
+        case Character.DECIMAL_DIGIT_NUMBER:
+        case Character.LETTER_NUMBER:
+        case Character.OTHER_NUMBER:
+        case Character.COMBINING_SPACING_MARK:
+        case Character.LINE_SEPARATOR:
+        case Character.SPACE_SEPARATOR:
+        case Character.PARAGRAPH_SEPARATOR:
+ *
+ */
+        default:
+            return false;
+        }
+    }
+}
diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java
new file mode 100644
index 00000000000..9606578b3ac
--- /dev/null
+++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.language.opennlp;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/opennlp-linguistics/src/main/resources/models/langdetect-183.bin b/opennlp-linguistics/src/main/resources/models/langdetect-183.bin
new file mode 100644
index 00000000000..c3cde217050
--- /dev/null
+++ b/opennlp-linguistics/src/main/resources/models/langdetect-183.bin
diff --git a/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
new file mode 100644
index 00000000000..746ed10da1c
--- /dev/null
+++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java
@@ -0,0 +1,87 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.detect.Detector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class OpenNlpDetectorTestCase {
+
+    @Test
+    public void testDetection() {
+        Detector detector = new OpenNlpDetector();
+
+        assertLanguage(Language.UNKNOWN,
+                       "",
+                       detector);
+
+        assertLanguage(Language.UNKNOWN,
+                       "Hello!",
+                       detector);
+
+        // from https://en.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.ENGLISH,
+                       "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.",
+                       detector);
+
+        // from https://de.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.GERMAN,
+                       "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.",
+                       detector);
+
+        // from https://fr.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.FRENCH,
+                       "À l'origine, Yahoo! était uniquement un annuaire Web.",
+                       detector);
+
+        // Test fallback to SimpleDetector
+        assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input
+                       "我能吞下玻璃而不伤身体。",
+                       detector);
+
+        // from https://zh.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.CHINESE_TRADITIONAL,
+                       "Yahoo! Next是一个展示雅虎新技术、新产品的场所，目前在测试阶段。",
+                       detector);
+
+        // from https://th.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.THAI,
+                       "เดือนกรกฎาคม 2012 Yahoo! ก็ได้ประธานเจ้าหน้าที่บริหารคนใหม่ \"มาริสสา เมเยอร์\" อดีตผู้บริหารจาก Google มาทำหน้าที่พลิกฟื้นบริษัท",
+                       detector);
+
+        // from https://ar.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.ARABIC,
+                       "وفقًا لمزودي تحليلات الويب دائما كأليكسا وسميلارويب،وصل موقع ياهولأكثر من 7 مليارات مشاهدة شهريًا - حيث احتل المرتبة السادسة بين أكثر مواقع الويب زيارة على مستوى العالم في عام 2016.",
+                       detector);
+
+        // from https://ko.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.KOREAN,
+                       "야후!의 전신인 디렉터리 사이트는 1994년 1월에 스탠퍼드 대학교 출신의 제리 양과 데이비드 파일로가 만들었으며, 회사는 1995년 3월 2일에 설립되었다.",
+                       detector);
+
+        // from https://ja.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.JAPANESE,
+                       "日本では、ヤフー株式会社がYahoo!（後にベライゾンがアルタバに売却）とソフトバンクの合弁会社として1996年に設立した。",
+                       detector);
+
+        // from https://ru.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.RUSSIAN,
+                       "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.",
+                       detector);
+
+        // from https://he.wikipedia.org/wiki/Yahoo
+        assertLanguage(Language.HEBREW,
+                       "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום",
+                       detector);
+    }
+
+    private void assertLanguage(Language language, String input, Detector detector) {
+        assertEquals(language, detector.detect(input, null).getLanguage());
+    }
+
+}
diff --git a/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
new file mode 100644
index 00000000000..cd2a0f73895
--- /dev/null
+++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
@@ -0,0 +1,258 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.TokenType;
+import com.yahoo.language.process.Tokenizer;
+import org.junit.Test;
+
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test of tokenization, with stemming and accent removal
+ *
+ * @author matskin
+ */
+public class OpenNlpTokenizationTestCase {
+
+    private final Tokenizer tokenizer = new OpenNlpTokenizer();
+
+    @Test
+    public void testTokenizer() {
+        assertTokenize("This is a test, 123",
+                       List.of("this", "is", "a", "test", "123"),
+                       List.of("This", " ", "is", " ", "a", " ", "test", ",", " ", "123"));
+    }
+
+    @Test
+    public void testUnderScoreTokenization() {
+        assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, List.of("ugcapi", "1"), null);
+    }
+
+    @Test
+    public void testPhrasesWithPunctuation() {
+        assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE,
+                       false,
+                       List.of("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"),
+                       null);
+        assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, List.of("phy", "101"), null);
+        assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, List.of("101", "3"), null);
+    }
+
+    @Test
+    public void testDoubleWidthTokenization() {
+        // "sony"
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.NONE, false,
+                       List.of("sony"), null);
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.SHORTEST, false,
+                       List.of("sony"), null);
+        // "SONY"
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.NONE, false,
+                       List.of("sony"), null);
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.SHORTEST, false,
+                       List.of("sony"), null);
+        // "on"
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.NONE, false,
+                       List.of("on"), null);
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       List.of("on"), null);
+        // "ON"
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.NONE, false,
+                       List.of("on"), null);
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       List.of("on"), null);
+        assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false,
+                       List.of("наименован"), null);
+    }
+
+    @Test
+    public void testLargeTextTokenization() {
+        String input = "teststring ".repeat(100000);
+        int numTokens = 0;
+        List<Long> pos = new ArrayList<>();
+        for (Token t : tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false)) {
+            numTokens++;
+            if ((numTokens % 100) == 0) {
+                pos.add(t.getOffset());
+            }
+        }
+
+        assertEquals("Check that all tokens have been tokenized", numTokens, 200000);
+        assertTrue("Increasing token pos", assertMonoIncr(pos));
+    }
+
+    @Test
+    public void testLargeTokenGuard() {
+        String input = "ab".repeat(128 * 256);
+        Iterator<Token> it = tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertTrue(it.hasNext());
+        assertNotNull(it.next().getTokenString());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testTokenIterator() {
+        Iterator<Token> it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+        try {
+            it.next();
+            fail();
+        } catch (NoSuchElementException e) {
+            // success
+        }
+
+        it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+
+        it = tokenizer.tokenize("one two three", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testGetOffsetLength() {
+        String input = "Deka-Chef Weber r\u00e4umt Kommunikationsfehler ein";
+        long[] expOffset = { 0, 4, 5, 9, 10, 15, 16, 21, 22, 42, 43 };
+        int[] len = { 4, 1, 4, 1, 5, 1, 5, 1, 20, 1, 3 };
+
+        int idx = 0;
+        for (Token token : tokenizer.tokenize(input, Language.GERMAN, StemMode.SHORTEST, false)) {
+            assertEquals("Token offset for token #" + idx, expOffset[idx], token.getOffset());
+            assertEquals("Token len for token #" + idx, len[idx], token.getOrig().length());
+            idx++;
+        }
+    }
+
+    @Test
+    public void testRecursiveDecompose() {
+        for (Token t : tokenizer.tokenize("\u00a510%", Language.ENGLISH, StemMode.SHORTEST, false)) {
+            recurseDecompose(t);
+        }
+    }
+
+    @Test
+    public void testIndexability() {
+        String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652";
+        for (StemMode stemMode : new StemMode[] { StemMode.NONE,
+                StemMode.SHORTEST }) {
+            for (Language l : List.of(Language.INDONESIAN, Language.ENGLISH, Language.ARABIC)) {
+                for (boolean accentDrop : new boolean[] { true, false }) {
+                    for (Token token : tokenizer.tokenize(input, l, stemMode, accentDrop)) {
+                        if (token.getTokenString().length() == 0) {
+                            assertFalse(token.isIndexable());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testTokenTypes() {
+        testTokenTypes(Language.ENGLISH);
+        testTokenTypes(Language.SPANISH);
+    }
+
+    public void testTokenTypes(Language language) {
+        assertEquals(TokenType.ALPHABETIC, tokenize("word", language).iterator().next().getType());
+        assertEquals(TokenType.NUMERIC, tokenize("123", language).iterator().next().getType());
+        assertEquals(TokenType.SPACE, tokenize(" ", language).iterator().next().getType());
+        assertEquals(TokenType.PUNCTUATION, tokenize(".", language).iterator().next().getType());
+        assertEquals(TokenType.ALPHABETIC, tokenize("123word", language).iterator().next().getType());
+
+        var tokens = tokenize("123 123word word123", language).iterator();
+        assertEquals(TokenType.NUMERIC, tokens.next().getType());
+        assertEquals(TokenType.SPACE, tokens.next().getType());
+        assertEquals(TokenType.ALPHABETIC, tokens.next().getType());
+        assertEquals(TokenType.SPACE, tokens.next().getType());
+        assertEquals(TokenType.ALPHABETIC, tokens.next().getType());
+    }
+
+    private Iterable<Token> tokenize(String input, Language language) {
+        return tokenizer.tokenize(input, language, StemMode.SHORTEST, true);
+    }
+
+    private void recurseDecompose(Token t) {
+        assertTrue(t.getOffset() >= 0);
+        assertTrue(t.getOrig().length() >= 0);
+
+        int numComp = t.getNumComponents();
+        for (int i = 0; i < numComp; i++) {
+            Token comp = t.getComponent(i);
+            recurseDecompose(comp);
+        }
+    }
+
+    private boolean assertMonoIncr(Iterable<Long> n) {
+        long trailing = -1;
+        for (long i : n) {
+            if (i < trailing) {
+                return false;
+            }
+            trailing = i;
+        }
+        return true;
+    }
+
+    private void assertTokenize(String input, List<String> indexed, List<String> orig) {
+        assertTokenize(input, Language.ENGLISH, StemMode.NONE, false, indexed, orig);
+    }
+
+    /**
+     * Compare the results of running an input string through the tokenizer with an "index" truth, and an optional
+     * "orig" truth.
+     *
+     * @param input      the text to process, passed to tokenizer
+     * @param language   the language tag, passed to tokenizer
+     * @param stemMode   if stemMode != NONE, test will silently succeed if tokenizer does not do stemming
+     * @param accentDrop passed to the tokenizer
+     * @param indexed    compared to the "TokenString" result from the tokenizer
+     * @param orig       compared to the "Orig" result from the tokenizer
+     */
+    private void assertTokenize(String input, Language language, StemMode stemMode, boolean accentDrop,
+                                List<String> indexed, List<String> orig) {
+        int i = 0;
+        int j = 0;
+        for (Token token : tokenizer.tokenize(input, language, stemMode, accentDrop)) {
+            // System.err.println("got token orig '"+token.getOrig()+"'");
+            // System.err.println("got token stem '"+token.getTokenString(stemMode)+"'");
+            if (token.getNumComponents() > 0) {
+                for (int comp = 0; comp < token.getNumComponents(); comp++) {
+                    Token t = token.getComponent(comp);
+                    if (t.getType().isIndexable()) {
+                        assertEquals("comp index: " + i, indexed.get(i++), toLowerCase(t.getTokenString()));
+                    }
+                }
+            } else {
+                if (token.getType().isIndexable()) {
+                    assertEquals("exp index: " + i, indexed.get(i++), toLowerCase(token.getTokenString()));
+                }
+            }
+            if (orig != null) {
+                assertEquals("orig index: " + j, token.getOrig(), orig.get(j++));
+            }
+        }
+        assertEquals("indexed length", indexed.size(), i);
+        if (orig != null) {
+            assertEquals("orig length", orig.size(), j);
+        }
+    }
+
+}
diff --git a/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
new file mode 100644
index 00000000000..a8c637bc6ec
--- /dev/null
+++ b/opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java
@@ -0,0 +1,20 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class UrlCharSequenceNormalizerTest {
+
+    @Test
+    public void testNormalization() {
+        String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo";
+        assertEquals("  foo  _bax   a@b § ",
+                     UrlCharSequenceNormalizer.getInstance().normalize(text));
+    }
+
+}