diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-11-26 00:10:17 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-11-26 23:55:10 +0100 |
commit | a8665da65c39d9e4a56c74c2d8e6a7bd61c7c313 (patch) | |
tree | 7ef8738fca139dfdab1464c5edfd3d7423427b9b /linguistics | |
parent | b36cb57248dfc02bae9dfe7b2cca0ddd551881c6 (diff) |
Split out opennlp-linguistics
Diffstat (limited to 'linguistics')
14 files changed, 0 insertions, 783 deletions
diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 87ecc35ba2c..bfbf1beeaea 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -58,10 +58,6 @@ <scope>provided</scope> <classifier>no_aop</classifier> </dependency> - <dependency> - <groupId>org.apache.opennlp</groupId> - <artifactId>opennlp-tools</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java b/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java deleted file mode 100644 index 27c23d8d3e6..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.ngram.NGramCharModel; -import opennlp.tools.util.normalizer.CharSequenceNormalizer; - -import java.util.HashSet; -import java.util.Set; - -/** - * Avoids using the unnecessarily slow {@link NGramCharModel}. - * - * @author jonmv - */ -public class DefaultLanguageDetectorContextGenerator extends opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator { - - public DefaultLanguageDetectorContextGenerator(int minLength, int maxLength, CharSequenceNormalizer... normalizers) { - super(minLength, maxLength, normalizers); - } - - @Override - public String[] getContext(CharSequence document) { - int[] normalized = normalizer.normalize(document).codePoints().map(Character::toLowerCase).toArray(); - Set<String> grams = new HashSet<>(); - for (int i = 0; i < normalized.length; i++) - for (int j = minLength; j <= maxLength && i + j < normalized.length; j++) - grams.add(new String(normalized, i, j)); - - return grams.toArray(new String[grams.size()]); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java deleted file mode 100644 index 0cf4634c6c3..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.langdetect.LanguageDetectorContextGenerator; - -/** - * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 - * - * @author jonmv - */ -@SuppressWarnings("unused") // Loaded by black magic: specified in properties in the loaded model. -public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { - - @Override - public LanguageDetectorContextGenerator getContextGenerator() { - return new DefaultLanguageDetectorContextGenerator(1, 3, - VespaCharSequenceNormalizer.getInstance()); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java deleted file mode 100644 index d7a7d3a4744..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detection; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.detect.Hint; -import com.yahoo.language.simple.SimpleDetector; -import opennlp.tools.langdetect.LanguageDetectorConfig; -import opennlp.tools.langdetect.LanguageDetectorME; -import opennlp.tools.langdetect.LanguageDetectorModel; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Detects text language using patched OpenNLP, with fallback to {@link SimpleDetector} for undetected CJK input. - * - * @author jonmv - */ -class OpenNlpDetector implements Detector { - - private static final Object monitor = new Object(); - private static LanguageDetectorModel model; - - private final SimpleDetector simple = new SimpleDetector(); - private final Map<String, Language> languagesByISO3 = new HashMap<>(); - private final LanguageDetectorME detector; - private final LanguageDetectorConfig config; - - OpenNlpDetector() { - detector = new LanguageDetectorME(loadModel()); - config = new LanguageDetectorConfig(); - config.setMinDiff(0.02); - config.setChunkSize(32); - config.setMaxLength(256); - for (Locale locale : Locale.getAvailableLocales()) { - Language language = Language.fromLocale(locale); - if (language != null) - languagesByISO3.put(locale.getISO3Language(), language); - } - } - - private static LanguageDetectorModel loadModel() { - synchronized (monitor) { - if (model == null) { - try { - model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin")); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - return model; - } - - @Override - public Detection detect(byte[] input, int offset, int length, Hint hint) { - Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length)); - return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false); - } - - @Override - public Detection detect(ByteBuffer input, Hint hint) { - if (input.hasArray()) - return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint); - - byte[] buffer = new byte[input.remaining()]; - input.get(buffer); - return detect(buffer, 0, buffer.length, hint); - } - - @Override - public Detection detect(String input, Hint hint) { - return new Detection(detectLanguage(input), UTF_8.name(), false); - } - - private Language detectLanguage(String input) { - var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0]; - var result = prediction.getConfidence() > 0.02 ? languagesByISO3.get(prediction.getLang()) : null; - return result != null ? result : simple.guessLanguage(input.substring(0, Math.min(input.length(), 256))); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java deleted file mode 100644 index 1d96d8a0cdf..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.language.Linguistics; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.process.Tokenizer; -import com.yahoo.language.simple.SimpleLinguistics; - -/** - * Returns a linguistics implementation based on OpenNlp. - * - * @author bratseth - * @author jonmv - */ -public class OpenNlpLinguistics extends SimpleLinguistics { - - private final Detector detector; - - @Inject - public OpenNlpLinguistics() { - this.detector = new OpenNlpDetector(); - } - - @Override - public Tokenizer getTokenizer() { - return new OpenNlpTokenizer(getNormalizer(), getTransformer()); - } - - @Override - public Detector getDetector() { return detector; } - - @Override - public boolean equals(Linguistics other) { return (other instanceof OpenNlpLinguistics); } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java deleted file mode 100644 index 8080dc92729..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.LinguisticsCase; -import com.yahoo.language.process.Normalizer; -import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.language.process.StemMode; -import com.yahoo.language.process.Token; -import com.yahoo.language.process.Tokenizer; -import com.yahoo.language.process.Transformer; -import com.yahoo.language.simple.SimpleNormalizer; -import com.yahoo.language.simple.SimpleTokenizer; -import com.yahoo.language.simple.SimpleTransformer; -import opennlp.tools.stemmer.Stemmer; -import opennlp.tools.stemmer.snowball.SnowballStemmer; - -import java.util.List; - -/** - * Tokenizer using OpenNlp - * - * @author matskin - * @author bratseth - */ -public class OpenNlpTokenizer implements Tokenizer { - - private final static int SPACE_CODE = 32; - private final Normalizer normalizer; - private final Transformer transformer; - private final SimpleTokenizer simpleTokenizer; - private final SpecialTokenRegistry specialTokenRegistry; - - public OpenNlpTokenizer() { - this(new SimpleNormalizer(), new SimpleTransformer()); - } - - public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) { - this(normalizer, transformer, new SpecialTokenRegistry(List.of())); - } - - public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) { - this.normalizer = normalizer; - this.transformer = transformer; - this.specialTokenRegistry = specialTokenRegistry; - this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); - } - - @Override - public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { - Stemmer stemmer = stemmerFor(language, stemMode); - if (stemmer == null) - return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); - else - return simpleTokenizer.tokenize(input, token -> processToken(token, language, stemMode, removeAccents, stemmer)); - } - - private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, - Stemmer stemmer) { - token = normalizer.normalize(token); - token = LinguisticsCase.toLowerCase(token); - if (removeAccents) - token = transformer.accentDrop(token, language); - if (stemMode != StemMode.NONE) - token = stemmer.stem(token).toString(); - return token; - } - - private Stemmer stemmerFor(Language language, StemMode stemMode) { - if (language == null || language == Language.ENGLISH || stemMode == StemMode.NONE) return null; - SnowballStemmer.ALGORITHM algorithm = algorithmFor(language); - if (algorithm == null) return null; - return new SnowballStemmer(algorithm); - } - - private SnowballStemmer.ALGORITHM algorithmFor(Language language) { - switch (language) { - case DANISH: return SnowballStemmer.ALGORITHM.DANISH; - case DUTCH: return SnowballStemmer.ALGORITHM.DUTCH; - case FINNISH: return SnowballStemmer.ALGORITHM.FINNISH; - case FRENCH: return SnowballStemmer.ALGORITHM.FRENCH; - case GERMAN: return SnowballStemmer.ALGORITHM.GERMAN; - case HUNGARIAN: return SnowballStemmer.ALGORITHM.HUNGARIAN; - case IRISH: return SnowballStemmer.ALGORITHM.IRISH; - case ITALIAN: return SnowballStemmer.ALGORITHM.ITALIAN; - case NORWEGIAN_BOKMAL: return SnowballStemmer.ALGORITHM.NORWEGIAN; - case NORWEGIAN_NYNORSK: return SnowballStemmer.ALGORITHM.NORWEGIAN; - case PORTUGUESE: return SnowballStemmer.ALGORITHM.PORTUGUESE; - case ROMANIAN: return SnowballStemmer.ALGORITHM.ROMANIAN; - case RUSSIAN: return SnowballStemmer.ALGORITHM.RUSSIAN; - case SPANISH: return SnowballStemmer.ALGORITHM.SPANISH; - case SWEDISH: return SnowballStemmer.ALGORITHM.SWEDISH; - case TURKISH: return SnowballStemmer.ALGORITHM.TURKISH; - case ENGLISH: return SnowballStemmer.ALGORITHM.ENGLISH; - default: return null; - } - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java deleted file mode 100644 index 883319e2f8b..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.util.normalizer.CharSequenceNormalizer; - -import java.util.regex.Pattern; - -/** - * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. - * - * @author jonmv - */ -public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { - - private static final Pattern URL_REGEX = - Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); - private static final Pattern MAIL_REGEX = - Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); - - private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); - - public static UrlCharSequenceNormalizer getInstance() { - return INSTANCE; - } - - public CharSequence normalize(CharSequence text) { - String modified = URL_REGEX.matcher(text).replaceAll(" "); - return MAIL_REGEX.matcher(modified).replaceAll(" "); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java deleted file mode 100644 index df8f3fad520..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/VespaCharSequenceNormalizer.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import opennlp.tools.util.normalizer.CharSequenceNormalizer; - -import java.util.function.IntConsumer; -import java.util.stream.IntStream; - -/** - * Simple normalizer - * - * @author arnej - */ -public class VespaCharSequenceNormalizer implements CharSequenceNormalizer { - - private static final VespaCharSequenceNormalizer INSTANCE = new VespaCharSequenceNormalizer(); - - public static VespaCharSequenceNormalizer getInstance() { - return INSTANCE; - } - - // filter replacing sequences of non-letters with a single space - static class OnlyLetters implements IntStream.IntMapMultiConsumer { - boolean addSpace = false; - public void accept(int codepoint, IntConsumer target) { - if (WordCharDetector.isWordChar(codepoint)) { - if (addSpace) { - target.accept(' '); - addSpace = false; - } - target.accept(Character.toLowerCase(codepoint)); - } else { - addSpace = true; - } - } - } - - public CharSequence normalize(CharSequence text) { - if (text.isEmpty()) { - return text; - } - var r = text - .codePoints() - .mapMulti(new OnlyLetters()) - .collect(StringBuilder::new, - StringBuilder::appendCodePoint, - StringBuilder::append); - return r; - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java deleted file mode 100644 index d7e3f88ae8d..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/WordCharDetector.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -class WordCharDetector { - public static boolean isWordChar(int codepoint) { - int unicodeGeneralCategory = Character.getType(codepoint); - switch (unicodeGeneralCategory) { - case Character.LOWERCASE_LETTER: - case Character.OTHER_LETTER: - case Character.TITLECASE_LETTER: - case Character.UPPERCASE_LETTER: - case Character.MODIFIER_LETTER: - return true; -/* - * these are the other categories, currently considered non-word-chars: - * - case Character.CONNECTOR_PUNCTUATION: - case Character.CONTROL: - case Character.CURRENCY_SYMBOL: - case Character.DASH_PUNCTUATION: - case Character.ENCLOSING_MARK: - case Character.END_PUNCTUATION: - case Character.FINAL_QUOTE_PUNCTUATION: - case Character.FORMAT: - case Character.INITIAL_QUOTE_PUNCTUATION: - case Character.MATH_SYMBOL: - case Character.MODIFIER_SYMBOL: - case Character.NON_SPACING_MARK: - case Character.OTHER_PUNCTUATION: - case Character.OTHER_SYMBOL: - case Character.PRIVATE_USE: - case Character.START_PUNCTUATION: - case Character.SURROGATE: - case Character.UNASSIGNED: - case Character.DECIMAL_DIGIT_NUMBER: - case Character.LETTER_NUMBER: - case Character.OTHER_NUMBER: - case Character.COMBINING_SPACING_MARK: - case Character.LINE_SEPARATOR: - case Character.SPACE_SEPARATOR: - case Character.PARAGRAPH_SEPARATOR: - * - */ - default: - return false; - } - } -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java b/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java deleted file mode 100644 index 9606578b3ac..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.language.opennlp; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin Binary files differdeleted file mode 100644 index c3cde217050..00000000000 --- a/linguistics/src/main/resources/models/langdetect-183.bin +++ /dev/null diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java deleted file mode 100644 index 746ed10da1c..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detector; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author jonmv - */ -public class OpenNlpDetectorTestCase { - - @Test - public void testDetection() { - Detector detector = new OpenNlpDetector(); - - assertLanguage(Language.UNKNOWN, - "", - detector); - - assertLanguage(Language.UNKNOWN, - "Hello!", - detector); - - // from https://en.wikipedia.org/wiki/Yahoo - assertLanguage(Language.ENGLISH, - "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.", - detector); - - // from https://de.wikipedia.org/wiki/Yahoo - assertLanguage(Language.GERMAN, - "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.", - detector); - - // from https://fr.wikipedia.org/wiki/Yahoo - assertLanguage(Language.FRENCH, - "À l'origine, Yahoo! était uniquement un annuaire Web.", - detector); - - // Test fallback to SimpleDetector - assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input - "我能吞下玻璃而不伤身体。", - detector); - - // from https://zh.wikipedia.org/wiki/Yahoo - assertLanguage(Language.CHINESE_TRADITIONAL, - "Yahoo! Next是一个展示雅虎新技术、新产品的场所,目前在测试阶段。", - detector); - - // from https://th.wikipedia.org/wiki/Yahoo - assertLanguage(Language.THAI, - "เดือนกรกฎาคม 2012 Yahoo! ก็ได้ประธานเจ้าหน้าที่บริหารคนใหม่ \"มาริสสา เมเยอร์\" อดีตผู้บริหารจาก Google มาทำหน้าที่พลิกฟื้นบริษัท", - detector); - - // from https://ar.wikipedia.org/wiki/Yahoo - assertLanguage(Language.ARABIC, - "وفقًا لمزودي تحليلات الويب دائما كأليكسا وسميلارويب،وصل موقع ياهولأكثر من 7 مليارات مشاهدة شهريًا - حيث احتل المرتبة السادسة بين أكثر مواقع الويب زيارة على مستوى العالم في عام 2016.", - detector); - - // from https://ko.wikipedia.org/wiki/Yahoo - assertLanguage(Language.KOREAN, - "야후!의 전신인 디렉터리 사이트는 1994년 1월에 스탠퍼드 대학교 출신의 제리 양과 데이비드 파일로가 만들었으며, 회사는 1995년 3월 2일에 설립되었다.", - detector); - - // from https://ja.wikipedia.org/wiki/Yahoo - assertLanguage(Language.JAPANESE, - "日本では、ヤフー株式会社がYahoo!(後にベライゾンがアルタバに売却)とソフトバンクの合弁会社として1996年に設立した。", - detector); - - // from https://ru.wikipedia.org/wiki/Yahoo - assertLanguage(Language.RUSSIAN, - "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.", - detector); - - // from https://he.wikipedia.org/wiki/Yahoo - assertLanguage(Language.HEBREW, - "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום", - detector); - } - - private void assertLanguage(Language language, String input, Detector detector) { - assertEquals(language, detector.detect(input, null).getLanguage()); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java deleted file mode 100644 index cd2a0f73895..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.process.StemMode; -import com.yahoo.language.process.Token; -import com.yahoo.language.process.TokenType; -import com.yahoo.language.process.Tokenizer; -import org.junit.Test; - - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -import static com.yahoo.language.LinguisticsCase.toLowerCase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test of tokenization, with stemming and accent removal - * - * @author matskin - */ -public class OpenNlpTokenizationTestCase { - - private final Tokenizer tokenizer = new OpenNlpTokenizer(); - - @Test - public void testTokenizer() { - assertTokenize("This is a test, 123", - List.of("this", "is", "a", "test", "123"), - List.of("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); - } - - @Test - public void testUnderScoreTokenization() { - assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, List.of("ugcapi", "1"), null); - } - - @Test - public void testPhrasesWithPunctuation() { - assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE, - false, - List.of("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), - null); - assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, List.of("phy", "101"), null); - assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, List.of("101", "3"), null); - } - - @Test - public void testDoubleWidthTokenization() { - // "sony" - assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.NONE, false, - List.of("sony"), null); - assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.SHORTEST, false, - List.of("sony"), null); - // "SONY" - assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.NONE, false, - List.of("sony"), null); - assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.SHORTEST, false, - List.of("sony"), null); - // "on" - assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.NONE, false, - List.of("on"), null); - assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.SHORTEST, false, - List.of("on"), null); - // "ON" - assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.NONE, false, - List.of("on"), null); - assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false, - List.of("on"), null); - assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false, - List.of("наименован"), null); - } - - @Test - public void testLargeTextTokenization() { - String input = "teststring ".repeat(100000); - int numTokens = 0; - List<Long> pos = new ArrayList<>(); - for (Token t : tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false)) { - numTokens++; - if ((numTokens % 100) == 0) { - pos.add(t.getOffset()); - } - } - - assertEquals("Check that all tokens have been tokenized", numTokens, 200000); - assertTrue("Increasing token pos", assertMonoIncr(pos)); - } - - @Test - public void testLargeTokenGuard() { - String input = "ab".repeat(128 * 256); - Iterator<Token> it = tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false).iterator(); - assertTrue(it.hasNext()); - assertNotNull(it.next().getTokenString()); - assertFalse(it.hasNext()); - } - - @Test - public void testTokenIterator() { - Iterator<Token> it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator(); - assertFalse(it.hasNext()); - try { - it.next(); - fail(); - } catch (NoSuchElementException e) { - // success - } - - it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator(); - assertFalse(it.hasNext()); - - it = tokenizer.tokenize("one two three", Language.ENGLISH, StemMode.NONE, false).iterator(); - assertNotNull(it.next()); - assertNotNull(it.next()); - assertNotNull(it.next()); - assertNotNull(it.next()); - assertNotNull(it.next()); - assertFalse(it.hasNext()); - } - - @Test - public void testGetOffsetLength() { - String input = "Deka-Chef Weber r\u00e4umt Kommunikationsfehler ein"; - long[] expOffset = { 0, 4, 5, 9, 10, 15, 16, 21, 22, 42, 43 }; - int[] len = { 4, 1, 4, 1, 5, 1, 5, 1, 20, 1, 3 }; - - int idx = 0; - for (Token token : tokenizer.tokenize(input, Language.GERMAN, StemMode.SHORTEST, false)) { - assertEquals("Token offset for token #" + idx, expOffset[idx], token.getOffset()); - assertEquals("Token len for token #" + idx, len[idx], token.getOrig().length()); - idx++; - } - } - - @Test - public void testRecursiveDecompose() { - for (Token t : tokenizer.tokenize("\u00a510%", Language.ENGLISH, StemMode.SHORTEST, false)) { - recurseDecompose(t); - } - } - - @Test - public void testIndexability() { - String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652"; - for (StemMode stemMode : new StemMode[] { StemMode.NONE, - StemMode.SHORTEST }) { - for (Language l : List.of(Language.INDONESIAN, Language.ENGLISH, Language.ARABIC)) { - for (boolean accentDrop : new boolean[] { true, false }) { - for (Token token : tokenizer.tokenize(input, l, stemMode, accentDrop)) { - if (token.getTokenString().length() == 0) { - assertFalse(token.isIndexable()); - } - } - } - } - } - } - - @Test - public void testTokenTypes() { - testTokenTypes(Language.ENGLISH); - testTokenTypes(Language.SPANISH); - } - - public void testTokenTypes(Language language) { - assertEquals(TokenType.ALPHABETIC, tokenize("word", language).iterator().next().getType()); - assertEquals(TokenType.NUMERIC, tokenize("123", language).iterator().next().getType()); - assertEquals(TokenType.SPACE, tokenize(" ", language).iterator().next().getType()); - assertEquals(TokenType.PUNCTUATION, tokenize(".", language).iterator().next().getType()); - assertEquals(TokenType.ALPHABETIC, tokenize("123word", language).iterator().next().getType()); - - var tokens = tokenize("123 123word word123", language).iterator(); - assertEquals(TokenType.NUMERIC, tokens.next().getType()); - assertEquals(TokenType.SPACE, tokens.next().getType()); - assertEquals(TokenType.ALPHABETIC, tokens.next().getType()); - assertEquals(TokenType.SPACE, tokens.next().getType()); - assertEquals(TokenType.ALPHABETIC, tokens.next().getType()); - } - - private Iterable<Token> tokenize(String input, Language language) { - return tokenizer.tokenize(input, language, StemMode.SHORTEST, true); - } - - private void recurseDecompose(Token t) { - assertTrue(t.getOffset() >= 0); - assertTrue(t.getOrig().length() >= 0); - - int numComp = t.getNumComponents(); - for (int i = 0; i < numComp; i++) { - Token comp = t.getComponent(i); - recurseDecompose(comp); - } - } - - private boolean assertMonoIncr(Iterable<Long> n) { - long trailing = -1; - for (long i : n) { - if (i < trailing) { - return false; - } - trailing = i; - } - return true; - } - - private void assertTokenize(String input, List<String> indexed, List<String> orig) { - assertTokenize(input, Language.ENGLISH, StemMode.NONE, false, indexed, orig); - } - - /** - * Compare the results of running an input string through the tokenizer with an "index" truth, and an optional - * "orig" truth. - * - * @param input the text to process, passed to tokenizer - * @param language the language tag, passed to tokenizer - * @param stemMode if stemMode != NONE, test will silently succeed if tokenizer does not do stemming - * @param accentDrop passed to the tokenizer - * @param indexed compared to the "TokenString" result from the tokenizer - * @param orig compared to the "Orig" result from the tokenizer - */ - private void assertTokenize(String input, Language language, StemMode stemMode, boolean accentDrop, - List<String> indexed, List<String> orig) { - int i = 0; - int j = 0; - for (Token token : tokenizer.tokenize(input, language, stemMode, accentDrop)) { - // System.err.println("got token orig '"+token.getOrig()+"'"); - // System.err.println("got token stem '"+token.getTokenString(stemMode)+"'"); - if (token.getNumComponents() > 0) { - for (int comp = 0; comp < token.getNumComponents(); comp++) { - Token t = token.getComponent(comp); - if (t.getType().isIndexable()) { - assertEquals("comp index: " + i, indexed.get(i++), toLowerCase(t.getTokenString())); - } - } - } else { - if (token.getType().isIndexable()) { - assertEquals("exp index: " + i, indexed.get(i++), toLowerCase(token.getTokenString())); - } - } - if (orig != null) { - assertEquals("orig index: " + j, token.getOrig(), orig.get(j++)); - } - } - assertEquals("indexed length", indexed.size(), i); - if (orig != null) { - assertEquals("orig length", orig.size(), j); - } - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java deleted file mode 100644 index a8c637bc6ec..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author jonmv - */ -public class UrlCharSequenceNormalizerTest { - - @Test - public void testNormalization() { - String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo"; - assertEquals(" foo _bax a@b § ", - UrlCharSequenceNormalizer.getInstance().normalize(text)); - } - -} |