diff options
5 files changed, 2 insertions, 9 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java index 1e2d5c87f66..74f300057dc 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; import com.yahoo.io.IOUtils; diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java index 0446b13f0ee..1659e3c0fa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; /** diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java index 53e8513a611..b6659ebeaa3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java @@ -1,5 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.language.sentencepiece; import com.google.common.annotations.Beta; diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index ac78b5c78a6..782030a8e4d 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; /** @@ -9,5 +7,7 @@ package com.yahoo.language.sentencepiece; * @author bratseth */ enum TokenType { + text, control, userDefined, unknown, unused + } diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java index ca54d94f88a..f3287a49517 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; import java.util.HashMap; |