Separate component from linguistics

1 files changed, 13 insertions, 0 deletions

diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java
new file mode 100644
index 00000000000..782030a8e4d
--- /dev/null
+++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/TokenType.java
@@ -0,0 +1,13 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.sentencepiece;
+
+/**
+ * SentencePiece token types
+ *
+ * @author bratseth
+ */
+enum TokenType {
+
+    text, control, userDefined, unknown, unused
+
+}