aboutsummaryrefslogtreecommitdiffstats
path: root/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java19
1 files changed, 0 insertions, 19 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
index c9e432dd459..975a32c8852 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
@@ -22,28 +22,9 @@ public interface Tokenizer {
* @param removeAccents if true accents and similar are removed from the returned tokens
* @return the tokens of the input String.
* @throws ProcessingException If the underlying library throws an Exception.
- * @deprecated use tokenize with a context instead
*/
- @Deprecated // TODO: Remove on Vespa 8
default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
return List.of();
}
- /**
- * Returns the tokens produced from an input string under the rules of the given Language and additional options.
- * This dsefault implementation delegates to the tokenize method without context.
- *
- * @param input the string to tokenize. May be arbitrarily large.
- * @param language the language of the input string.
- * @param stemMode the stem mode applied on the returned tokens
- * @param removeAccents if true accents and similar are removed from the returned tokens
- * @param context the context of this processing
- * @return the tokens of the input String.
- * @throws ProcessingException If the underlying library throws an Exception.
- */
- default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents,
- LinguisticsContext context) {
- return tokenize(input, language, stemMode, removeAccents);
- }
-
}