diff options
author | Jefim Matskin <jefimm@wix.com> | 2018-07-18 16:47:07 +0300 |
---|---|---|
committer | Jefim Matskin <jefimm@wix.com> | 2018-07-18 16:47:07 +0300 |
commit | 7ea9da4d2ac02e42bb3a0ecdff575090e590e136 (patch) | |
tree | 82ca16f88b471604f25cee62fd3f07e1c9d45782 | |
parent | 174abd98383bae0cde12349723cfdfcdf0799bce (diff) |
add opennlp stemmers - revert previous changes
https://github.com/vespa-engine/vespa/issues/6403
10 files changed, 392 insertions, 139 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 960ca6dac97..bab2f37e667 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -53,7 +53,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>18.0</version> + <version>13.0.1</version> <scope>provided</scope> </dependency> <dependency> @@ -308,14 +308,6 @@ <artifactId>tensorflow</artifactId> <version>1.6.0</version> </dependency> - <dependency> - <groupId>org.apache.opennlp</groupId> - <artifactId>opennlp-tools</artifactId> - </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </dependency> </dependencies> <build> diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 0de866563cd..2a3b76a42b9 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -136,10 +136,7 @@ <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> <include>xml-apis:xml-apis:[1.4.01]:jar:provided</include> - <include>com.optimaize.languagedetector:language-detector:0.6:jar:provided</include> <include>org.apache.opennlp:opennlp-tools:1.8.4:jar:provided</include> - <include>net.arnx:jsonic:1.2.11:jar:provided</include> - <include>com.intellij:annotations:12.0:jar:provided</include> </includes> </bannedDependencies> </rules> diff --git a/linguistics/pom.xml b/linguistics/pom.xml index f743348dde3..1785befbc39 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -66,10 +66,6 @@ <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java new file mode 100644 index 00000000000..12de309a2d3 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -0,0 +1,11 @@ +package com.yahoo.language.opennlp; + +import com.yahoo.language.process.Tokenizer; +import com.yahoo.language.simple.SimpleLinguistics; + +public class OpenNlpLinguistics extends SimpleLinguistics { + @Override + public Tokenizer getTokenizer() { + return new OpenNlpTokenizer(getNormalizer(), getTransformer()); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java new file mode 100644 index 00000000000..5d5f5cbfba9 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -0,0 +1,135 @@ +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.LinguisticsCase; +import com.yahoo.language.process.*; +import com.yahoo.language.simple.*; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.snowball.SnowballStemmer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class OpenNlpTokenizer implements Tokenizer { + private final static int SPACE_CODE = 32; + private final Normalizer normalizer; + private final Transformer transformer; + private final SimpleTokenizer simpleTokenizer; + + public OpenNlpTokenizer() { + this(new SimpleNormalizer(), new SimpleTransformer()); + } + + public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { + this.normalizer = normalizer; + this.transformer = transformer; + simpleTokenizer = new SimpleTokenizer(normalizer, transformer); + } + + @Override + public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { + if (input.isEmpty()) return Collections.emptyList(); + Stemmer stemmer = getStemmerForLanguage(language, stemMode); + if (stemmer == null) { + return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); + } + + List<Token> tokens = new ArrayList<>(); + int nextCode = input.codePointAt(0); + TokenType prevType = SimpleTokenType.valueOf(nextCode); + for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) { + nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE; + TokenType nextType = SimpleTokenType.valueOf(nextCode); + if (!prevType.isIndexable() || !nextType.isIndexable()) { + String original = input.substring(prev, next); + String token = processToken(original, language, stemMode, removeAccents, stemmer); + tokens.add(new SimpleToken(original).setOffset(prev) + .setType(prevType) + .setTokenString(token)); + prev = next; + prevType = nextType; + } + next += Character.charCount(nextCode); + } + return tokens; + } + + private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) { + if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) { + return null; + } + SnowballStemmer.ALGORITHM alg; + switch (language) { + case DANISH: + alg = SnowballStemmer.ALGORITHM.DANISH; + break; + case DUTCH: + alg = SnowballStemmer.ALGORITHM.DUTCH; + break; + case FINNISH: + alg = SnowballStemmer.ALGORITHM.FINNISH; + break; + case FRENCH: + alg = SnowballStemmer.ALGORITHM.FRENCH; + break; + case GERMAN: + alg = SnowballStemmer.ALGORITHM.GERMAN; + break; + case HUNGARIAN: + alg = SnowballStemmer.ALGORITHM.HUNGARIAN; + break; + case IRISH: + alg = SnowballStemmer.ALGORITHM.IRISH; + break; + case ITALIAN: + alg = SnowballStemmer.ALGORITHM.ITALIAN; + break; + case NORWEGIAN_BOKMAL: + case NORWEGIAN_NYNORSK: + alg = SnowballStemmer.ALGORITHM.NORWEGIAN; + break; + case PORTUGUESE: + alg = SnowballStemmer.ALGORITHM.PORTUGUESE; + break; + case ROMANIAN: + alg = SnowballStemmer.ALGORITHM.ROMANIAN; + break; + case RUSSIAN: + alg = SnowballStemmer.ALGORITHM.RUSSIAN; + break; + case SPANISH: + alg = SnowballStemmer.ALGORITHM.SPANISH; + break; + case SWEDISH: + alg = SnowballStemmer.ALGORITHM.SWEDISH; + break; + case TURKISH: + alg = SnowballStemmer.ALGORITHM.TURKISH; + break; + case ENGLISH: + alg = SnowballStemmer.ALGORITHM.ENGLISH; + break; + default: + return null; + + } + return new SnowballStemmer(alg); + } + + private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, + Stemmer stemmer) { + token = normalizer.normalize(token); + token = LinguisticsCase.toLowerCase(token); + if (removeAccents) + token = transformer.accentDrop(token, language); + if (stemMode != StemMode.NONE) { + token = doStemming(token, stemmer); + } + return token; + } + + private String doStemming(String token, Stemmer stemmer) { + return stemmer.stem(token).toString(); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 8ac3237a953..e6ce4eddb59 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -1,26 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; -import com.google.common.base.Optional; -import com.optimaize.langdetect.LanguageDetector; -import com.optimaize.langdetect.LanguageDetectorBuilder; -import com.optimaize.langdetect.i18n.LdLocale; -import com.optimaize.langdetect.ngram.NgramExtractors; -import com.optimaize.langdetect.profiles.LanguageProfile; -import com.optimaize.langdetect.profiles.LanguageProfileReader; -import com.optimaize.langdetect.text.CommonTextObjectFactories; -import com.optimaize.langdetect.text.TextObject; -import com.optimaize.langdetect.text.TextObjectFactory; import com.yahoo.language.Language; import com.yahoo.language.detect.Detection; import com.yahoo.language.detect.Detector; import com.yahoo.language.detect.Hint; import com.yahoo.text.Utf8; -import java.io.IOException; import java.nio.ByteBuffer; -import java.util.List; -import java.util.Locale; /** * Includes functionality for determining the langCode from a sample or from the encoding. Currently only Chinese, @@ -36,27 +23,6 @@ import java.util.Locale; * @author Rich Pito */ public class SimpleDetector implements Detector { - static private TextObjectFactory textObjectFactory; - static private LanguageDetector languageDetector; - - static { - // origin: https://github.com/optimaize/language-detector - //load all languages: - List<LanguageProfile> languageProfiles; - try { - languageProfiles = new LanguageProfileReader().readAllBuiltIn(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - //build language detector: - languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) - .withProfiles(languageProfiles) - .build(); - - //create a text object factory - textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); - } @Override public Detection detect(byte[] input, int offset, int length, Hint hint) { @@ -143,26 +109,10 @@ public class SimpleDetector implements Detector { return Language.THAI; } } - if (Language.UNKNOWN.equals(soFar)){ - return detectLangOptimaize(input); - } // got to the end, so return the current best guess return soFar; } - private static Language detectLangOptimaize(String input) { - if (input == null || input.length() == 0) { - return Language.UNKNOWN; - } - TextObject textObject = textObjectFactory.forText(input); - Optional<LdLocale> lang = languageDetector.detect(textObject); - if (lang.isPresent()) { - String language = lang.get().getLanguage(); - return Language.fromLocale(new Locale(language)); - } - return Language.UNKNOWN; - } - private boolean isTrailingOctet(byte i) { return ((i >>> 6) & 3) == 2; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 4888fd8676f..068fc0126d7 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -5,8 +5,6 @@ import com.yahoo.language.Language; import com.yahoo.language.LinguisticsCase; import com.yahoo.language.process.*; import com.yahoo.language.simple.kstem.KStemmer; -import opennlp.tools.stemmer.Stemmer; -import opennlp.tools.stemmer.snowball.SnowballStemmer; import java.util.ArrayList; import java.util.Collections; @@ -26,6 +24,7 @@ public class SimpleTokenizer implements Tokenizer { private final static int SPACE_CODE = 32; private final Normalizer normalizer; private final Transformer transformer; + private final KStemmer stemmer = new KStemmer(); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -44,8 +43,6 @@ public class SimpleTokenizer implements Tokenizer { public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { if (input.isEmpty()) return Collections.emptyList(); - opennlp.tools.stemmer.Stemmer stemmer = getStemmerForLanguage(language); - List<Token> tokens = new ArrayList<>(); int nextCode = input.codePointAt(0); TokenType prevType = SimpleTokenType.valueOf(nextCode); @@ -54,10 +51,10 @@ public class SimpleTokenizer implements Tokenizer { TokenType nextType = SimpleTokenType.valueOf(nextCode); if (!prevType.isIndexable() || !nextType.isIndexable()) { String original = input.substring(prev, next); - String token = processToken(original, language, stemMode, removeAccents, stemmer); + String token = processToken(original, language, stemMode, removeAccents); tokens.add(new SimpleToken(original).setOffset(prev) - .setType(prevType) - .setTokenString(token)); + .setType(prevType) + .setTokenString(token)); prev = next; prevType = nextType; } @@ -66,72 +63,14 @@ public class SimpleTokenizer implements Tokenizer { return tokens; } - private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { + private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); - if (stemMode != StemMode.NONE && token != null) - token = stemmer.stem(token).toString(); + if (stemMode != StemMode.NONE) + token = stemmer.stem(token); return token; } - private static Stemmer getStemmerForLanguage(Language language) { - Stemmer stemmer = charSequence -> charSequence == null ? null : new KStemmer().stem(charSequence.toString()); - if (language == null) { - return stemmer; - } - SnowballStemmer.ALGORITHM alg; - switch (language) { - case DANISH: - alg = SnowballStemmer.ALGORITHM.DANISH; - break; - case DUTCH: - alg = SnowballStemmer.ALGORITHM.DUTCH; - break; - case FINNISH: - alg = SnowballStemmer.ALGORITHM.FINNISH; - break; - case FRENCH: - alg = SnowballStemmer.ALGORITHM.FRENCH; - break; - case GERMAN: - alg = SnowballStemmer.ALGORITHM.GERMAN; - break; - case HUNGARIAN: - alg = SnowballStemmer.ALGORITHM.HUNGARIAN; - break; - case IRISH: - alg = SnowballStemmer.ALGORITHM.IRISH; - break; - case ITALIAN: - alg = SnowballStemmer.ALGORITHM.ITALIAN; - break; - case NORWEGIAN_BOKMAL: - case NORWEGIAN_NYNORSK: - alg = SnowballStemmer.ALGORITHM.NORWEGIAN; - break; - case PORTUGUESE: - alg = SnowballStemmer.ALGORITHM.PORTUGUESE; - break; - case ROMANIAN: - alg = SnowballStemmer.ALGORITHM.ROMANIAN; - break; - case RUSSIAN: - alg = SnowballStemmer.ALGORITHM.RUSSIAN; - break; - case SPANISH: - alg = SnowballStemmer.ALGORITHM.SPANISH; - break; - case SWEDISH: - alg = SnowballStemmer.ALGORITHM.SWEDISH; - break; - case TURKISH: - alg = SnowballStemmer.ALGORITHM.TURKISH; - break; - default: - return stemmer; - } - return new SnowballStemmer(alg); - } } diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java new file mode 100644 index 00000000000..914e3817568 --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java @@ -0,0 +1,237 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; +import com.yahoo.language.process.Tokenizer; +import org.junit.Test; + +import java.util.*; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * Test of tokenization, with stemming and accent removal + * + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + */ +public class OpenNlpTokenizationTestCase { + + private final Tokenizer tokenizer = new OpenNlpTokenizer(); + + @Test + public void testTokenizer() { + assertTokenize("This is a test, 123", + Arrays.asList("this", "is", "a", "test", "123"), + Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); + } + + @Test + public void testUnderScoreTokenization() { + assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null); + } + + @Test + public void testPhrasesWithPunctuation() { + assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE, + false, + Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), + null); + assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null); + assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null); + } + + @Test + public void testDoubleWidthTokenization() { + // "sony" + assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.NONE, false, + Arrays.asList("sony"), null); + assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.SHORTEST, false, + Arrays.asList("sony"), null); + // "SONY" + assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.NONE, false, + Arrays.asList("sony"), null); + assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.SHORTEST, false, + Arrays.asList("sony"), null); + // "on" + assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.NONE, false, + Arrays.asList("on"), null); + assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.SHORTEST, false, + Arrays.asList("on"), null); + // "ON" + assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.NONE, false, + Arrays.asList("on"), null); + assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false, + Arrays.asList("on"), null); + assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false, + Arrays.asList("наименован"), null); + } + + @Test + public void testLargeTextTokenization() { + StringBuilder sb = new StringBuilder(); + String s = "teststring "; + for (int i = 0; i < 100000; i++) { + sb.append(s); + } + + String input = sb.toString(); + + int numTokens = 0; + List<Long> pos = new ArrayList<>(); + for (Token t : tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false)) { + numTokens++; + if ((numTokens % 100) == 0) { + pos.add(t.getOffset()); + } + } + + assertEquals("Check that all tokens have been tokenized", numTokens, 200000); + assertTrue("Increasing token pos", assertMonoIncr(pos)); + } + + @Test + public void testLargeTokenGuard() { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < 128 * 256; i++) { + str.append("ab"); + } + Iterator<Token> it = tokenizer.tokenize(str.toString(), Language.ENGLISH, StemMode.NONE, false).iterator(); + assertTrue(it.hasNext()); + assertNotNull(it.next().getTokenString()); + assertFalse(it.hasNext()); + } + + @Test + public void testTokenIterator() { + Iterator<Token> it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator(); + assertFalse(it.hasNext()); + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + // success + } + + it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator(); + assertFalse(it.hasNext()); + + it = tokenizer.tokenize("one two three", Language.ENGLISH, StemMode.NONE, false).iterator(); + assertNotNull(it.next()); + assertNotNull(it.next()); + assertNotNull(it.next()); + assertNotNull(it.next()); + assertNotNull(it.next()); + assertFalse(it.hasNext()); + } + + @Test + public void testGetOffsetLength() { + String input = "Deka-Chef Weber r\u00e4umt Kommunikationsfehler ein"; + long[] expOffset = { 0, 4, 5, 9, 10, 15, 16, 21, 22, 42, 43 }; + int[] len = { 4, 1, 4, 1, 5, 1, 5, 1, 20, 1, 3 }; + + int idx = 0; + for (Token token : tokenizer.tokenize(input, Language.GERMAN, StemMode.SHORTEST, false)) { + assertThat("Token offset for token #" + idx, token.getOffset(), is(expOffset[idx])); + assertThat("Token len for token #" + idx, token.getOrig().length(), is(len[idx])); + idx++; + } + } + + @Test + public void testRecursiveDecompose() { + for (Token t : tokenizer.tokenize("\u00a510%", Language.ENGLISH, StemMode.SHORTEST, false)) { + recurseDecompose(t); + } + } + + @Test + public void testIndexability() { + String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652"; + for (StemMode stemMode : new StemMode[] { StemMode.NONE, + StemMode.SHORTEST }) { + for (Language l : new Language[] { Language.INDONESIAN, + Language.ENGLISH, Language.ARABIC }) { + for (boolean accentDrop : new boolean[] { true, false }) { + for (Token token : tokenizer.tokenize(input, + l, stemMode, accentDrop)) { + if (token.getTokenString().length() == 0) { + assertFalse(token.isIndexable()); + } + } + } + } + } + } + + private void recurseDecompose(Token t) { + assertTrue(t.getOffset() >= 0); + assertTrue(t.getOrig().length() >= 0); + + int numComp = t.getNumComponents(); + for (int i = 0; i < numComp; i++) { + Token comp = t.getComponent(i); + recurseDecompose(comp); + } + } + + private boolean assertMonoIncr(Iterable<Long> n) { + long trailing = -1; + for (long i : n) { + if (i < trailing) { + return false; + } + trailing = i; + } + return true; + } + + private void assertTokenize(String input, List<String> indexed, List<String> orig) { + assertTokenize(input, Language.ENGLISH, StemMode.NONE, false, indexed, orig); + } + + /** + * <p>Compare the results of running an input string through the tokenizer with an "index" truth, and an optional + * "orig" truth.</p> + * + * @param input The text to process, passed to tokenizer. + * @param language The language tag, passed to tokenizer. + * @param stemMode If stemMode != NONE, test will silently succeed if tokenizer does not do stemming. + * @param accentDrop Passed to the tokenizer. + * @param indexed Compared to the "TokenString" result from the tokenizer. + * @param orig Compared to the "Orig" result from the tokenizer. + */ + private void assertTokenize(String input, Language language, StemMode stemMode, boolean accentDrop, + List<String> indexed, List<String> orig) { + int i = 0; + int j = 0; + for (Token token : tokenizer.tokenize(input, language, stemMode, accentDrop)) { + // System.err.println("got token orig '"+token.getOrig()+"'"); + // System.err.println("got token stem '"+token.getTokenString(stemMode)+"'"); + if (token.getNumComponents() > 0) { + for (int comp = 0; comp < token.getNumComponents(); comp++) { + Token t = token.getComponent(comp); + if (t.getType().isIndexable()) { + assertThat("comp index: " + i, toLowerCase(t.getTokenString()), is(indexed.get(i++))); + } + } + } else { + if (token.getType().isIndexable()) { + assertThat("exp index: " + i, toLowerCase(token.getTokenString()), is(indexed.get(i++))); + } + } + if (orig != null) { + assertThat("orig index: " + j, token.getOrig(), is(orig.get(j++))); + } + } + assertThat("indexed length", i, is(indexed.size())); + if (orig != null) { + assertThat("orig length", j, is(orig.size())); + } + } + +} diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java index 27cfc12da5e..72dd6f8ce58 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java @@ -65,8 +65,7 @@ public class TokenizationTestCase { assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false, Arrays.asList("on"), null); - assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false, - Arrays.asList("наименован"), null); + } @Test diff --git a/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java index e1cac896525..f9912f6b7a2 100644 --- a/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java @@ -50,9 +50,6 @@ public class SimpleDetectorTestCase { // a string from http://www.columbia.edu/kermit/utf8.html that says "I can eat glass (and it doesn't hurt me)". assertLanguage(Language.KOREAN, "\ub098\ub294 \uc720\ub9ac\ub97c \uba39\uc744 \uc218 \uc788\uc5b4\uc694. " + "\uadf8\ub798\ub3c4 \uc544\ud504\uc9c0 \uc54a\uc544\uc694"); - - // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F - assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии"); } @Test |