diff options
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java index e36d90b3206..27cfc12da5e 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java @@ -64,6 +64,9 @@ public class TokenizationTestCase { Arrays.asList("on"), null); assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false, Arrays.asList("on"), null); + + assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false, + Arrays.asList("наименован"), null); } @Test |