diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-11-03 18:09:12 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2022-11-03 18:09:12 +0100 |
commit | 9e5a6fe3caf8ed4d7810202d843662ba8cac8bc0 (patch) | |
tree | 2418ace521d5dee02b56629004a27b21c2c67660 /linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java | |
parent | bb132428fa56e52317fad756e8ca498a0f32db30 (diff) |
Accept LinguisticContextbratseth/linguistics-context-rebased
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java | 19 |
1 files changed, 0 insertions, 19 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java index c9e432dd459..975a32c8852 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java @@ -22,28 +22,9 @@ public interface Tokenizer { * @param removeAccents if true accents and similar are removed from the returned tokens * @return the tokens of the input String. * @throws ProcessingException If the underlying library throws an Exception. - * @deprecated use tokenize with a context instead */ - @Deprecated // TODO: Remove on Vespa 8 default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { return List.of(); } - /** - * Returns the tokens produced from an input string under the rules of the given Language and additional options. - * This dsefault implementation delegates to the tokenize method without context. - * - * @param input the string to tokenize. May be arbitrarily large. - * @param language the language of the input string. - * @param stemMode the stem mode applied on the returned tokens - * @param removeAccents if true accents and similar are removed from the returned tokens - * @param context the context of this processing - * @return the tokens of the input String. - * @throws ProcessingException If the underlying library throws an Exception. - */ - default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents, - LinguisticsContext context) { - return tokenize(input, language, stemMode, removeAccents); - } - } |