Merge pull request #6407 from jefimm/master

add lang detection and opennlp stemmers

7 files changed, 400 insertions, 0 deletions

diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml
index bad78ce4182..60b293ca1d9 100644
--- a/container-dependencies-enforcer/pom.xml
+++ b/container-dependencies-enforcer/pom.xml
@@ -136,6 +136,7 @@
                                                 <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include>
                                                 <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include>
                                                 <include>xml-apis:xml-apis:[1.4.01]:jar:provided</include>
+                                                <include>org.apache.opennlp:opennlp-tools:1.8.4:jar:provided</include>
                                             </includes>
                                         </bannedDependencies>
                                     </rules>
diff --git a/linguistics/pom.xml b/linguistics/pom.xml
index e4aa7c3049e..1785befbc39 100644
--- a/linguistics/pom.xml
+++ b/linguistics/pom.xml
@@ -62,6 +62,10 @@
       <scope>provided</scope>
       <classifier>no_aop</classifier>
     </dependency>
+    <dependency>
+      <groupId>org.apache.opennlp</groupId>
+      <artifactId>opennlp-tools</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
new file mode 100644
index 00000000000..12de309a2d3
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
@@ -0,0 +1,11 @@
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.process.Tokenizer;
+import com.yahoo.language.simple.SimpleLinguistics;
+
+public class OpenNlpLinguistics extends SimpleLinguistics {
+    @Override
+    public Tokenizer getTokenizer() {
+        return new OpenNlpTokenizer(getNormalizer(), getTransformer());
+    }
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
new file mode 100644
index 00000000000..5d5f5cbfba9
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
@@ -0,0 +1,135 @@
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.LinguisticsCase;
+import com.yahoo.language.process.*;
+import com.yahoo.language.simple.*;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class OpenNlpTokenizer implements Tokenizer {
+    private final static int SPACE_CODE = 32;
+    private final Normalizer normalizer;
+    private final Transformer transformer;
+    private final SimpleTokenizer simpleTokenizer;
+
+    public OpenNlpTokenizer() {
+        this(new SimpleNormalizer(), new SimpleTransformer());
+    }
+
+    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
+        this.normalizer = normalizer;
+        this.transformer = transformer;
+        simpleTokenizer = new SimpleTokenizer(normalizer, transformer);
+    }
+
+    @Override
+    public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
+        if (input.isEmpty()) return Collections.emptyList();
+        Stemmer stemmer = getStemmerForLanguage(language, stemMode);
+        if (stemmer == null) {
+            return simpleTokenizer.tokenize(input, language, stemMode, removeAccents);
+        }
+
+        List<Token> tokens = new ArrayList<>();
+        int nextCode = input.codePointAt(0);
+        TokenType prevType = SimpleTokenType.valueOf(nextCode);
+        for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
+            nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
+            TokenType nextType = SimpleTokenType.valueOf(nextCode);
+            if (!prevType.isIndexable() || !nextType.isIndexable()) {
+                String original = input.substring(prev, next);
+                String token = processToken(original, language, stemMode, removeAccents, stemmer);
+                tokens.add(new SimpleToken(original).setOffset(prev)
+                        .setType(prevType)
+                        .setTokenString(token));
+                prev = next;
+                prevType = nextType;
+            }
+            next += Character.charCount(nextCode);
+        }
+        return tokens;
+    }
+
+    private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) {
+        if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) {
+            return null;
+        }
+        SnowballStemmer.ALGORITHM alg;
+        switch (language) {
+            case DANISH:
+                alg = SnowballStemmer.ALGORITHM.DANISH;
+                break;
+            case DUTCH:
+                alg = SnowballStemmer.ALGORITHM.DUTCH;
+                break;
+            case FINNISH:
+                alg = SnowballStemmer.ALGORITHM.FINNISH;
+                break;
+            case FRENCH:
+                alg = SnowballStemmer.ALGORITHM.FRENCH;
+                break;
+            case GERMAN:
+                alg = SnowballStemmer.ALGORITHM.GERMAN;
+                break;
+            case HUNGARIAN:
+                alg = SnowballStemmer.ALGORITHM.HUNGARIAN;
+                break;
+            case IRISH:
+                alg = SnowballStemmer.ALGORITHM.IRISH;
+                break;
+            case ITALIAN:
+                alg = SnowballStemmer.ALGORITHM.ITALIAN;
+                break;
+            case NORWEGIAN_BOKMAL:
+            case NORWEGIAN_NYNORSK:
+                alg = SnowballStemmer.ALGORITHM.NORWEGIAN;
+                break;
+            case PORTUGUESE:
+                alg = SnowballStemmer.ALGORITHM.PORTUGUESE;
+                break;
+            case ROMANIAN:
+                alg = SnowballStemmer.ALGORITHM.ROMANIAN;
+                break;
+            case RUSSIAN:
+                alg = SnowballStemmer.ALGORITHM.RUSSIAN;
+                break;
+            case SPANISH:
+                alg = SnowballStemmer.ALGORITHM.SPANISH;
+                break;
+            case SWEDISH:
+                alg = SnowballStemmer.ALGORITHM.SWEDISH;
+                break;
+            case TURKISH:
+                alg = SnowballStemmer.ALGORITHM.TURKISH;
+                break;
+            case ENGLISH:
+                alg = SnowballStemmer.ALGORITHM.ENGLISH;
+                break;
+            default:
+                return null;
+
+        }
+        return new SnowballStemmer(alg);
+    }
+
+    private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
+                                Stemmer stemmer) {
+        token = normalizer.normalize(token);
+        token = LinguisticsCase.toLowerCase(token);
+        if (removeAccents)
+            token = transformer.accentDrop(token, language);
+        if (stemMode != StemMode.NONE) {
+            token = doStemming(token, stemmer);
+        }
+        return token;
+    }
+
+    private String doStemming(String token, Stemmer stemmer) {
+        return stemmer.stem(token).toString();
+    }
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
new file mode 100644
index 00000000000..914e3817568
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
@@ -0,0 +1,237 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.Tokenizer;
+import org.junit.Test;
+
+import java.util.*;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+
+/**
+ * Test of tokenization, with stemming and accent removal
+ *
+ * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a>
+ */
+public class OpenNlpTokenizationTestCase {
+
+    private final Tokenizer tokenizer = new OpenNlpTokenizer();
+
+    @Test
+    public void testTokenizer() {
+        assertTokenize("This is a test, 123",
+                       Arrays.asList("this", "is", "a", "test", "123"),
+                       Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123"));
+    }
+
+    @Test
+    public void testUnderScoreTokenization() {
+        assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null);
+    }
+
+    @Test
+    public void testPhrasesWithPunctuation() {
+        assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE,
+                       false,
+                       Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"),
+                       null);
+        assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null);
+        assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null);
+    }
+
+    @Test
+    public void testDoubleWidthTokenization() {
+        // "sony"
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("sony"), null);
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("sony"), null);
+        // "SONY"
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("sony"), null);
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("sony"), null);
+        // "on"
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("on"), null);
+        // "ON"
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false,
+                Arrays.asList("наименован"), null);
+    }
+
+    @Test
+    public void testLargeTextTokenization() {
+        StringBuilder sb = new StringBuilder();
+        String s = "teststring ";
+        for (int i = 0; i < 100000; i++) {
+            sb.append(s);
+        }
+
+        String input = sb.toString();
+
+        int numTokens = 0;
+        List<Long> pos = new ArrayList<>();
+        for (Token t : tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false)) {
+            numTokens++;
+            if ((numTokens % 100) == 0) {
+                pos.add(t.getOffset());
+            }
+        }
+
+        assertEquals("Check that all tokens have been tokenized", numTokens, 200000);
+        assertTrue("Increasing token pos", assertMonoIncr(pos));
+    }
+
+    @Test
+    public void testLargeTokenGuard() {
+        StringBuilder str = new StringBuilder();
+        for (int i = 0; i < 128 * 256; i++) {
+            str.append("ab");
+        }
+        Iterator<Token> it = tokenizer.tokenize(str.toString(), Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertTrue(it.hasNext());
+        assertNotNull(it.next().getTokenString());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testTokenIterator() {
+        Iterator<Token> it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+        try {
+            it.next();
+            fail();
+        } catch (NoSuchElementException e) {
+            // success
+        }
+
+        it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+
+        it = tokenizer.tokenize("one two three", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testGetOffsetLength() {
+        String input = "Deka-Chef Weber r\u00e4umt Kommunikationsfehler ein";
+        long[] expOffset = { 0, 4, 5, 9, 10, 15, 16, 21, 22, 42, 43 };
+        int[] len = { 4, 1, 4, 1, 5, 1, 5, 1, 20, 1, 3 };
+
+        int idx = 0;
+        for (Token token : tokenizer.tokenize(input, Language.GERMAN, StemMode.SHORTEST, false)) {
+            assertThat("Token offset for token #" + idx, token.getOffset(), is(expOffset[idx]));
+            assertThat("Token len for token #" + idx, token.getOrig().length(), is(len[idx]));
+            idx++;
+        }
+    }
+
+    @Test
+    public void testRecursiveDecompose() {
+        for (Token t : tokenizer.tokenize("\u00a510%", Language.ENGLISH, StemMode.SHORTEST, false)) {
+            recurseDecompose(t);
+        }
+    }
+
+    @Test
+    public void testIndexability() {
+        String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652";
+        for (StemMode stemMode : new StemMode[] { StemMode.NONE,
+                StemMode.SHORTEST }) {
+            for (Language l : new Language[] { Language.INDONESIAN,
+                    Language.ENGLISH, Language.ARABIC }) {
+                for (boolean accentDrop : new boolean[] { true, false }) {
+                    for (Token token : tokenizer.tokenize(input,
+                            l, stemMode, accentDrop)) {
+                        if (token.getTokenString().length() == 0) {
+                            assertFalse(token.isIndexable());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private void recurseDecompose(Token t) {
+        assertTrue(t.getOffset() >= 0);
+        assertTrue(t.getOrig().length() >= 0);
+
+        int numComp = t.getNumComponents();
+        for (int i = 0; i < numComp; i++) {
+            Token comp = t.getComponent(i);
+            recurseDecompose(comp);
+        }
+    }
+
+    private boolean assertMonoIncr(Iterable<Long> n) {
+        long trailing = -1;
+        for (long i : n) {
+            if (i < trailing) {
+                return false;
+            }
+            trailing = i;
+        }
+        return true;
+    }
+
+    private void assertTokenize(String input, List<String> indexed, List<String> orig) {
+        assertTokenize(input, Language.ENGLISH, StemMode.NONE, false, indexed, orig);
+    }
+
+    /**
+     * <p>Compare the results of running an input string through the tokenizer with an "index" truth, and an optional
+     * "orig" truth.</p>
+     *
+     * @param input      The text to process, passed to tokenizer.
+     * @param language   The language tag, passed to tokenizer.
+     * @param stemMode   If stemMode != NONE, test will silently succeed if tokenizer does not do stemming.
+     * @param accentDrop Passed to the tokenizer.
+     * @param indexed    Compared to the "TokenString" result from the tokenizer.
+     * @param orig       Compared to the "Orig" result from the tokenizer.
+     */
+    private void assertTokenize(String input, Language language, StemMode stemMode, boolean accentDrop,
+                                List<String> indexed, List<String> orig) {
+        int i = 0;
+        int j = 0;
+        for (Token token : tokenizer.tokenize(input, language, stemMode, accentDrop)) {
+            // System.err.println("got token orig '"+token.getOrig()+"'");
+            // System.err.println("got token stem '"+token.getTokenString(stemMode)+"'");
+            if (token.getNumComponents() > 0) {
+                for (int comp = 0; comp < token.getNumComponents(); comp++) {
+                    Token t = token.getComponent(comp);
+                    if (t.getType().isIndexable()) {
+                        assertThat("comp index: " + i, toLowerCase(t.getTokenString()), is(indexed.get(i++)));
+                    }
+                }
+            } else {
+                if (token.getType().isIndexable()) {
+                    assertThat("exp index: " + i, toLowerCase(token.getTokenString()), is(indexed.get(i++)));
+                }
+            }
+            if (orig != null) {
+                assertThat("orig index: " + j, token.getOrig(), is(orig.get(j++)));
+            }
+        }
+        assertThat("indexed length", i, is(indexed.size()));
+        if (orig != null) {
+            assertThat("orig length", j, is(orig.size()));
+        }
+    }
+
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
index e36d90b3206..72dd6f8ce58 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
@@ -64,6 +64,8 @@ public class TokenizationTestCase {
                        Arrays.asList("on"), null);
         assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false,
                        Arrays.asList("on"), null);
+
+
     }
 
     @Test
diff --git a/parent/pom.xml b/parent/pom.xml
index 0b141046d8a..34f6a4e1523 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -663,6 +663,16 @@
                 <artifactId>wiremock-standalone</artifactId>
                 <version>2.6.0</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.opennlp</groupId>
+                <artifactId>opennlp-tools</artifactId>
+                <version>1.8.4</version>
+            </dependency>
+            <dependency>
+                <groupId>com.optimaize.languagedetector</groupId>
+                <artifactId>language-detector</artifactId>
+                <version>0.6</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>