1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.opennlp;
import com.yahoo.language.Language;
import com.yahoo.language.LinguisticsCase;
import com.yahoo.language.process.*;
import com.yahoo.language.simple.*;
import opennlp.tools.stemmer.Stemmer;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.logging.Logger;
import java.util.logging.Level;
public class OpenNlpTokenizer implements Tokenizer {
private final static int SPACE_CODE = 32;
private static final Logger log = Logger.getLogger(OpenNlpTokenizer.class.getName());
private final Normalizer normalizer;
private final Transformer transformer;
private final SimpleTokenizer simpleTokenizer;
public OpenNlpTokenizer() {
this(new SimpleNormalizer(), new SimpleTransformer());
}
public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
this.normalizer = normalizer;
this.transformer = transformer;
simpleTokenizer = new SimpleTokenizer(normalizer, transformer);
}
@Override
public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
if (input.isEmpty()) return Collections.emptyList();
Stemmer stemmer = getStemmerForLanguage(language, stemMode);
if (stemmer == null) {
return simpleTokenizer.tokenize(input, language, stemMode, removeAccents);
}
List<Token> tokens = new ArrayList<>();
int nextCode = input.codePointAt(0);
TokenType prevType = SimpleTokenType.valueOf(nextCode);
for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
TokenType nextType = SimpleTokenType.valueOf(nextCode);
if (!prevType.isIndexable() || !nextType.isIndexable()) {
String original = input.substring(prev, next);
String token = processToken(original, language, stemMode, removeAccents, stemmer);
tokens.add(new SimpleToken(original).setOffset(prev)
.setType(prevType)
.setTokenString(token));
prev = next;
prevType = nextType;
}
next += Character.charCount(nextCode);
}
return tokens;
}
private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) {
log.log(Level.FINEST, () -> "getStemmerForLanguage '"+language+"' mode: "+stemMode);
if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) {
return null;
}
SnowballStemmer.ALGORITHM alg;
switch (language) {
case DANISH:
alg = SnowballStemmer.ALGORITHM.DANISH;
break;
case DUTCH:
alg = SnowballStemmer.ALGORITHM.DUTCH;
break;
case FINNISH:
alg = SnowballStemmer.ALGORITHM.FINNISH;
break;
case FRENCH:
alg = SnowballStemmer.ALGORITHM.FRENCH;
break;
case GERMAN:
alg = SnowballStemmer.ALGORITHM.GERMAN;
break;
case HUNGARIAN:
alg = SnowballStemmer.ALGORITHM.HUNGARIAN;
break;
case IRISH:
alg = SnowballStemmer.ALGORITHM.IRISH;
break;
case ITALIAN:
alg = SnowballStemmer.ALGORITHM.ITALIAN;
break;
case NORWEGIAN_BOKMAL:
case NORWEGIAN_NYNORSK:
alg = SnowballStemmer.ALGORITHM.NORWEGIAN;
break;
case PORTUGUESE:
alg = SnowballStemmer.ALGORITHM.PORTUGUESE;
break;
case ROMANIAN:
alg = SnowballStemmer.ALGORITHM.ROMANIAN;
break;
case RUSSIAN:
alg = SnowballStemmer.ALGORITHM.RUSSIAN;
break;
case SPANISH:
alg = SnowballStemmer.ALGORITHM.SPANISH;
break;
case SWEDISH:
alg = SnowballStemmer.ALGORITHM.SWEDISH;
break;
case TURKISH:
alg = SnowballStemmer.ALGORITHM.TURKISH;
break;
case ENGLISH:
alg = SnowballStemmer.ALGORITHM.ENGLISH;
break;
default:
return null;
}
return new SnowballStemmer(alg);
}
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
Stemmer stemmer) {
log.log(Level.FINEST, () -> "processToken '"+token+"'");
token = normalizer.normalize(token);
token = LinguisticsCase.toLowerCase(token);
if (removeAccents)
token = transformer.accentDrop(token, language);
if (stemMode != StemMode.NONE) {
String oldToken = token;
token = doStemming(token, stemmer);
log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'");
}
log.log(Level.FINEST, () -> "processed token is: "+token);
return token;
}
private String doStemming(String token, Stemmer stemmer) {
return stemmer.stem(token).toString();
}
}
|
