blob: 7c1148dac5047ae1f4b571bdb7170f1bc20b0741 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.language.Language;
import java.util.ArrayList;
import java.util.List;
/**
* @author Simon Thoresen Hult
*/
public class SegmenterImpl implements Segmenter {
private final Tokenizer tokenizer;
public SegmenterImpl(Tokenizer tokenizer) {
this.tokenizer = tokenizer;
}
@Override
public List<String> segment(String input, Language language, LinguisticsContext context) {
List<String> segments = new ArrayList<>();
for (Token token : tokenizer.tokenize(input, language, StemMode.NONE, false, context)) {
findSegments(token, segments);
}
if (segments.isEmpty()) {
segments.add(input); // no segments, return original string
}
return segments;
}
private void findSegments(Token token, List<String> out) {
int len;
if (token.isSpecialToken() || (len = token.getNumComponents()) == 0) {
if (token.isIndexable()) {
String orig = token.getOrig();
if (! orig.isEmpty()) {
out.add(orig);
}
}
} else {
for (int i = 0; i < len; ++i) {
findSegments(token.getComponent(i), out);
}
}
}
}
|
