blob: 6906c48f2ae016b3c5e8c7a6d65226d12f85ed19 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.query.parser;
import com.yahoo.language.Language;
import com.yahoo.language.process.Segmenter;
import java.util.List;
/**
* @author bratseth
*/
public class TestSegmenter implements Segmenter {
/**
* <p>Splits "cd" and "fg" and every other single letter into separate tokens.</p>
* <p/>
* <p><b>Special case</b> for testing overlapping tokens:
* Any occurence of the string "bcd" will <b>not</b> split into the tokens
* "bc" and "d", but will instead split into "bc" and "cd".</p>
*/
@Override
public List<String> segment(String string, Language language) {
List<String> tokens = new java.util.ArrayList<>();
// Tokenize
for (int i = 0; i < string.length(); i++) {
String token = startsByTestToken(string, i);
if (token != null) {
tokens.add(token);
i = i + token.length() - 1;
} else {
tokens.add(string.substring(i, i + 1));
}
}
// Special case
for (int i = 0; i < tokens.size(); i++) {
String token = tokens.get(i);
if (token.equals("bc") && tokens.size() > i + 1 && tokens.get(i + 1).equals("d")) {
tokens.set(i + 1, "cd");
}
}
return tokens;
}
private static final String[] testTokens = new String[] { "bc", "fg", "first", "second", "third" };
private static String startsByTestToken(String string, int index) {
for (String testToken : testTokens) {
if (string.startsWith(testToken, index)) {
return testToken;
}
}
return null;
}
}
|