diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java')
| -rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java index b65c3ba663c..60071c3147c 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java +++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java @@ -12,8 +12,8 @@ import java.util.Map; import java.util.stream.Collectors; /** - * A registry which is responsible for knowing the current - * set of special tokens.Usage of this registry is multithread safe. + * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics + * and be tokenized into a single token. * * @author bratseth */ |