blob: 3cb82572976e3efa35555494c8b811b04d196cdf (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.language.Language;
import com.yahoo.language.Linguistics;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
* @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
*/
public abstract class AbstractTokenizerTestCase {
private boolean accentDrop = false;
private Language language = Language.ENGLISH;
private Linguistics linguistics;
private StemMode stemMode = StemMode.NONE;
public void assertTokenStrings(String input, List<String> expectedTokenStrings) {
List<String> actual = new ArrayList<>();
for (Token token : tokenize(input)) {
findTokenStrings(token, actual);
}
assertEquals(expectedTokenStrings, actual);
}
public List<String> findTokenStrings(Token token, List<String> out) {
int numComponents = token.getNumComponents();
if (token.isSpecialToken() || numComponents == 0) {
out.add(token.getTokenString());
} else {
for (int i = 0; i < numComponents; ++i) {
findTokenStrings(token.getComponent(i), out);
}
}
return out;
}
public Iterable<Token> tokenize(String input) {
return linguistics.getTokenizer().tokenize(input, language, stemMode, accentDrop);
}
public AbstractTokenizerTestCase setAccentDrop(boolean accentDrop) {
this.accentDrop = accentDrop;
return this;
}
public AbstractTokenizerTestCase setLanguage(Language language) {
this.language = language;
return this;
}
public AbstractTokenizerTestCase setLinguistics(Linguistics linguistics) {
this.linguistics = linguistics;
return this;
}
public AbstractTokenizerTestCase setStemMode(StemMode stemMode) {
this.stemMode = stemMode;
return this;
}
}
|