1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.simple;
import com.yahoo.language.Language;
import com.yahoo.language.LinguisticsCase;
import com.yahoo.language.process.Normalizer;
import com.yahoo.language.process.SpecialTokenRegistry;
import com.yahoo.language.process.StemMode;
import com.yahoo.language.process.Token;
import com.yahoo.language.process.TokenType;
import com.yahoo.language.process.Tokenizer;
import com.yahoo.language.process.Transformer;
import com.yahoo.language.simple.kstem.KStemmer;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
/**
* <p>A tokenizer which splits on whitespace, normalizes and transforms using the given implementations
* and stems using the kstem algorithm.</p>
*
* <p>This is not multithread safe.</p>
*
* @author Mathias Mølster Lidal
* @author bratseth
*/
public class SimpleTokenizer implements Tokenizer {
private final static int SPACE_CODE = 32;
private final Normalizer normalizer;
private final Transformer transformer;
private final KStemmer stemmer = new KStemmer();
private final SpecialTokenRegistry specialTokenRegistry;
public SimpleTokenizer() {
this(new SimpleNormalizer(), new SimpleTransformer());
}
public SimpleTokenizer(Normalizer normalizer) {
this(normalizer, new SimpleTransformer());
}
public SimpleTokenizer(Normalizer normalizer, Transformer transformer) {
this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
}
public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
this.normalizer = normalizer;
this.transformer = transformer;
this.specialTokenRegistry = specialTokenRegistry;
}
/** Tokenize the input, applying the transform of this to each token string. */
@Override
public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
return tokenize(input,
token -> processToken(token, language, stemMode, removeAccents));
}
/** Tokenize the input, and apply the given transform to each token string. */
public Iterable<Token> tokenize(String input, Function<String, String> tokenProcessor) {
if (input.isEmpty()) return List.of();
List<Token> tokens = new ArrayList<>();
int nextCode = input.codePointAt(0);
TokenType prevType = SimpleTokenType.valueOf(nextCode);
TokenType tokenType = prevType;
for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
TokenType nextType = SimpleTokenType.valueOf(nextCode);
if (!prevType.isIndexable() || !nextType.isIndexable()) {
String original = input.substring(prev, next);
tokens.add(new SimpleToken(original).setOffset(prev)
.setType(tokenType)
.setTokenString(tokenProcessor.apply(original)));
prev = next;
prevType = nextType;
tokenType = prevType;
}
else {
tokenType = determineType(tokenType, nextType);
}
next += Character.charCount(nextCode);
}
return tokens;
}
private TokenType determineType(TokenType tokenType, TokenType characterType) {
if (characterType == TokenType.ALPHABETIC) return TokenType.ALPHABETIC;
return tokenType;
}
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) {
token = normalizer.normalize(token);
token = LinguisticsCase.toLowerCase(token);
if (removeAccents)
token = transformer.accentDrop(token, language);
if (stemMode != StemMode.NONE)
token = stemmer.stem(token);
return token;
}
}
|
