aboutsummaryrefslogtreecommitdiffstats
path: root/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2022-11-03 18:09:12 +0100
committerJon Bratseth <bratseth@gmail.com>2022-11-03 18:09:12 +0100
commit9e5a6fe3caf8ed4d7810202d843662ba8cac8bc0 (patch)
tree2418ace521d5dee02b56629004a27b21c2c67660 /linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
parentbb132428fa56e52317fad756e8ca498a0f32db30 (diff)
Accept LinguisticContextbratseth/linguistics-context-rebased
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java19
1 files changed, 0 insertions, 19 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
index c9e432dd459..975a32c8852 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java
@@ -22,28 +22,9 @@ public interface Tokenizer {
* @param removeAccents if true accents and similar are removed from the returned tokens
* @return the tokens of the input String.
* @throws ProcessingException If the underlying library throws an Exception.
- * @deprecated use tokenize with a context instead
*/
- @Deprecated // TODO: Remove on Vespa 8
default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
return List.of();
}
- /**
- * Returns the tokens produced from an input string under the rules of the given Language and additional options.
- * This dsefault implementation delegates to the tokenize method without context.
- *
- * @param input the string to tokenize. May be arbitrarily large.
- * @param language the language of the input string.
- * @param stemMode the stem mode applied on the returned tokens
- * @param removeAccents if true accents and similar are removed from the returned tokens
- * @param context the context of this processing
- * @return the tokens of the input String.
- * @throws ProcessingException If the underlying library throws an Exception.
- */
- default Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents,
- LinguisticsContext context) {
- return tokenize(input, language, stemMode, removeAccents);
- }
-
}