blob: 11c00f2163b96ee2fbdb6077a5829ba8ae65c26e (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.language.Language;
import java.util.List;
/**
* Interface providing segmentation, i.e. splitting of CJK character blocks into separate tokens. This is primarily a
* convenience feature for users who don't need full tokenization (or who use a separate tokenizer and only need CJK
* processing).
*
* @author Mathias Mølster Lidal
*/
public interface Segmenter {
/**
* Split input-string into tokens, and returned a list of tokens in unprocessed form (i.e. lowercased, normalized
* and stemmed if applicable, see @link{StemMode} for list of stemming options). It is assumed that the input only
* contains word-characters, any punctuation and spacing tokens will be removed.
*
* @param input the text to segment.
* @param language language of input text.
* @return the list of segments.
* @throws ProcessingException if an exception is encountered during processing
*/
List<String> segment(String input, Language language);
}
|