diff options
| author | Jon Bratseth <bratseth@gmail.com> | 2021-09-17 00:04:41 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-17 00:04:41 +0200 |
| commit | e9b65e88869ce0ce773330de0b1128554257f359 (patch) | |
| tree | e7f7dfdadf67a6d9de7dea6e5bc0839e4a5f58e9 /linguistics | |
| parent | ef7854b61b38c6b1c1bd671b0c8e6d76554ce354 (diff) | |
Cleanup
Diffstat (limited to 'linguistics')
5 files changed, 2 insertions, 9 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java index 1e2d5c87f66..74f300057dc 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; import com.yahoo.io.IOUtils; diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java index 0446b13f0ee..1659e3c0fa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; /** diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java index 53e8513a611..b6659ebeaa3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java @@ -1,5 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.language.sentencepiece; import com.google.common.annotations.Beta; diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java index ac78b5c78a6..782030a8e4d 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; /** @@ -9,5 +7,7 @@ package com.yahoo.language.sentencepiece; * @author bratseth */ enum TokenType { + text, control, userDefined, unknown, unused + } diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java index ca54d94f88a..f3287a49517 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - package com.yahoo.language.sentencepiece; import java.util.HashMap; |