blob: a1df10f481b79c524fa332f2d051482697ebc0b1 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.language.Language;
import java.util.List;
/**
* Interface providing segmentation, i.e. splitting of CJK character blocks into separate tokens. This is primarily a
* convenience feature for users who don't need full tokenization (or who use a separate tokenizer and only need CJK
* processing).
*
* @author Mathias Mølster Lidal
*/
public interface Segmenter {
/**
* Split input-string into tokens, and returned a list of tokens in unprocessed form (i.e. lowercased, normalized
* and stemmed if applicable, see @link{StemMode} for list of stemming options). It is assumed that the input only
* contains word-characters, any punctuation and spacing tokens will be removed.
*
* This default implementation does nothing and returns no segments.
*
* @param input the text to segment.
* @param language language of input text.
* @return the list of segments.
* @throws ProcessingException if an exception is encountered during processing
* @deprecated use the method with a context
*/
@Deprecated // TODO: Remove on Vespa 8
default List<String> segment(String input, Language language) {
return List.of();
}
/**
* Split input-string into tokens, and returned a list of tokens in unprocessed form (i.e. lowercased, normalized
* and stemmed if applicable, see @link{StemMode} for list of stemming options). It is assumed that the input only
* contains word-characters, any punctuation and spacing tokens will be removed.
*
* This default implementation calls the method without a context.
*
* @param input the text to segment.
* @param language language of input text.
* @return the list of segments.
* @throws ProcessingException if an exception is encountered during processing
*/
default List<String> segment(String input, Language language, LinguisticsContext context) {
return segment(input, language);
}
}
|
