1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.opennlp;
import com.yahoo.language.Language;
import com.yahoo.language.LinguisticsCase;
import com.yahoo.language.process.Normalizer;
import com.yahoo.language.process.SpecialTokenRegistry;
import com.yahoo.language.process.StemMode;
import com.yahoo.language.process.Token;
import com.yahoo.language.process.Tokenizer;
import com.yahoo.language.process.Transformer;
import com.yahoo.language.simple.SimpleNormalizer;
import com.yahoo.language.simple.SimpleTokenizer;
import com.yahoo.language.simple.SimpleTransformer;
import opennlp.tools.stemmer.Stemmer;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
import java.util.List;
/**
* Tokenizer using OpenNlp
*
* @author matskin
* @author bratseth
*/
public class OpenNlpTokenizer implements Tokenizer {
private final static int SPACE_CODE = 32;
private final Normalizer normalizer;
private final Transformer transformer;
private final SimpleTokenizer simpleTokenizer;
private final SpecialTokenRegistry specialTokenRegistry;
public OpenNlpTokenizer() {
this(new SimpleNormalizer(), new SimpleTransformer());
}
public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
}
public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
this.normalizer = normalizer;
this.transformer = transformer;
this.specialTokenRegistry = specialTokenRegistry;
this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry);
}
@Override
public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
Stemmer stemmer = stemmerFor(language, stemMode);
if (stemmer == null)
return simpleTokenizer.tokenize(input, language, stemMode, removeAccents);
else
return simpleTokenizer.tokenize(input, token -> processToken(token, language, stemMode, removeAccents, stemmer));
}
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
Stemmer stemmer) {
token = normalizer.normalize(token);
token = LinguisticsCase.toLowerCase(token);
if (removeAccents)
token = transformer.accentDrop(token, language);
if (stemMode != StemMode.NONE)
token = stemmer.stem(token).toString();
return token;
}
private Stemmer stemmerFor(Language language, StemMode stemMode) {
if (language == null || language == Language.ENGLISH || stemMode == StemMode.NONE) return null;
SnowballStemmer.ALGORITHM algorithm = algorithmFor(language);
if (algorithm == null) return null;
return new SnowballStemmer(algorithm);
}
private SnowballStemmer.ALGORITHM algorithmFor(Language language) {
switch (language) {
case DANISH: return SnowballStemmer.ALGORITHM.DANISH;
case DUTCH: return SnowballStemmer.ALGORITHM.DUTCH;
case FINNISH: return SnowballStemmer.ALGORITHM.FINNISH;
case FRENCH: return SnowballStemmer.ALGORITHM.FRENCH;
case GERMAN: return SnowballStemmer.ALGORITHM.GERMAN;
case HUNGARIAN: return SnowballStemmer.ALGORITHM.HUNGARIAN;
case IRISH: return SnowballStemmer.ALGORITHM.IRISH;
case ITALIAN: return SnowballStemmer.ALGORITHM.ITALIAN;
case NORWEGIAN_BOKMAL: return SnowballStemmer.ALGORITHM.NORWEGIAN;
case NORWEGIAN_NYNORSK: return SnowballStemmer.ALGORITHM.NORWEGIAN;
case PORTUGUESE: return SnowballStemmer.ALGORITHM.PORTUGUESE;
case ROMANIAN: return SnowballStemmer.ALGORITHM.ROMANIAN;
case RUSSIAN: return SnowballStemmer.ALGORITHM.RUSSIAN;
case SPANISH: return SnowballStemmer.ALGORITHM.SPANISH;
case SWEDISH: return SnowballStemmer.ALGORITHM.SWEDISH;
case TURKISH: return SnowballStemmer.ALGORITHM.TURKISH;
case ENGLISH: return SnowballStemmer.ALGORITHM.ENGLISH;
default: return null;
}
}
}
|
