diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java')
| -rw-r--r-- | linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index ac78b5c78a6..782030a8e4d 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; /** @@ -9,5 +7,7 @@ package com.yahoo.language.sentencepiece; * @author bratseth */ enum TokenType { + text, control, userDefined, unknown, unused + } |