linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java


1
2
3
4
5
6
7
8
9
10
11
12
13

// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.sentencepiece;

/**
 * SentencePiece token types
 *
 * @author bratseth
 */
enum TokenType {

    text, control, userDefined, unknown, unused

}