1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.simple;
import com.yahoo.language.Language;
import com.yahoo.language.LinguisticsCase;
import com.yahoo.language.process.*;
import com.yahoo.language.simple.kstem.KStemmer;
import opennlp.tools.stemmer.Stemmer;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* <p>A tokenizer which splits on whitespace, normalizes and transforms using the given implementations
* and stems using the kstem algorithm.</p>
*
* <p>This is not multithread safe.</p>
*
* @author Mathias Mølster Lidal
* @author bratseth
*/
public class SimpleTokenizer implements Tokenizer {
private final static int SPACE_CODE = 32;
private final Normalizer normalizer;
private final Transformer transformer;
public SimpleTokenizer() {
this(new SimpleNormalizer(), new SimpleTransformer());
}
public SimpleTokenizer(Normalizer normalizer) {
this(normalizer, new SimpleTransformer());
}
public SimpleTokenizer(Normalizer normalizer, Transformer transformer) {
this.normalizer = normalizer;
this.transformer = transformer;
}
@Override
public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
if (input.isEmpty()) return Collections.emptyList();
opennlp.tools.stemmer.Stemmer stemmer = getStemmerForLanguage(language, stemMode);
List<Token> tokens = new ArrayList<>();
int nextCode = input.codePointAt(0);
TokenType prevType = SimpleTokenType.valueOf(nextCode);
for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
TokenType nextType = SimpleTokenType.valueOf(nextCode);
if (!prevType.isIndexable() || !nextType.isIndexable()) {
String original = input.substring(prev, next);
String token = processToken(original, language, stemMode, removeAccents, stemmer);
tokens.add(new SimpleToken(original).setOffset(prev)
.setType(prevType)
.setTokenString(token));
prev = next;
prevType = nextType;
}
next += Character.charCount(nextCode);
}
return tokens;
}
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) {
token = normalizer.normalize(token);
token = LinguisticsCase.toLowerCase(token);
if (removeAccents)
token = transformer.accentDrop(token, language);
if (stemMode != StemMode.NONE && token != null)
token = stemmer.stem(token).toString();
return token;
}
private static Stemmer getStemmerForLanguage(Language language, StemMode stemMode) {
SnowballStemmer.ALGORITHM alg;
switch (language) {
case DANISH:
alg = SnowballStemmer.ALGORITHM.DANISH;
break;
case DUTCH:
alg = SnowballStemmer.ALGORITHM.DUTCH;
break;
case FINNISH:
alg = SnowballStemmer.ALGORITHM.FINNISH;
break;
case FRENCH:
alg = SnowballStemmer.ALGORITHM.FRENCH;
break;
case GERMAN:
alg = SnowballStemmer.ALGORITHM.GERMAN;
break;
case HUNGARIAN:
alg = SnowballStemmer.ALGORITHM.HUNGARIAN;
break;
case IRISH:
alg = SnowballStemmer.ALGORITHM.IRISH;
break;
case ITALIAN:
alg = SnowballStemmer.ALGORITHM.ITALIAN;
break;
case NORWEGIAN_BOKMAL:
case NORWEGIAN_NYNORSK:
alg = SnowballStemmer.ALGORITHM.NORWEGIAN;
break;
case PORTUGUESE:
alg = SnowballStemmer.ALGORITHM.PORTUGUESE;
break;
case ROMANIAN:
alg = SnowballStemmer.ALGORITHM.ROMANIAN;
break;
case RUSSIAN:
alg = SnowballStemmer.ALGORITHM.RUSSIAN;
break;
case SPANISH:
alg = SnowballStemmer.ALGORITHM.SPANISH;
break;
case SWEDISH:
alg = SnowballStemmer.ALGORITHM.SWEDISH;
break;
case TURKISH:
alg = SnowballStemmer.ALGORITHM.TURKISH;
break;
default:
return charSequence -> charSequence == null ? null : new KStemmer().stem(charSequence.toString());
}
return new SnowballStemmer(alg);
}
}
|
