diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java | 19 |
1 files changed, 0 insertions, 19 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java index c9e432dd459..975a32c8852 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java @@ -22,28 +22,9 @@ public interface Tokenizer { * @param removeAccents if true accents and similar are removed from the returned tokens * @return the tokens of the input String. * @throws ProcessingException If the underlying library throws an Exception. - * @deprecated use tokenize with a context instead */ - @Deprecated // TODO: Remove on Vespa 8 default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { return List.of(); } - /** - * Returns the tokens produced from an input string under the rules of the given Language and additional options. - * This dsefault implementation delegates to the tokenize method without context. - * - * @param input the string to tokenize. May be arbitrarily large. - * @param language the language of the input string. - * @param stemMode the stem mode applied on the returned tokens - * @param removeAccents if true accents and similar are removed from the returned tokens - * @param context the context of this processing - * @return the tokens of the input String. - * @throws ProcessingException If the underlying library throws an Exception. - */ - default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents, - LinguisticsContext context) { - return tokenize(input, language, stemMode, removeAccents); - } - } |