diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-08-16 23:31:19 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2022-08-16 23:31:19 +0200 |
commit | b24fcd4853b21971f404851147b51bf069864345 (patch) | |
tree | d5938be7dcf4f41864367a3da6d4ab914beb2907 /linguistics/src/test/java/com/yahoo/language/process | |
parent | ce04cf67da4f954f996efce328efb01a3ce0e85f (diff) |
Determine token types considering all characters
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java | 23 |
1 files changed, 16 insertions, 7 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index 5054f5a9bff..fa8419e200f 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -171,21 +171,30 @@ public class GramSplitterTestCase { public void testChineseComma() { String text = "我喜欢红色、蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); - for (; grams.hasNext(); ) { - System.out.println(grams.next().extractFrom(text)); - } + assertEquals("我喜", grams.next().extractFrom(text)); + assertEquals("喜欢", grams.next().extractFrom(text)); + assertEquals("欢红", grams.next().extractFrom(text)); + assertEquals("红色", grams.next().extractFrom(text)); + assertEquals("蓝色", grams.next().extractFrom(text)); + assertEquals("色和", grams.next().extractFrom(text)); + assertEquals("和紫", grams.next().extractFrom(text)); + assertEquals("紫色", grams.next().extractFrom(text)); } @Test public void testEnglishComma() { String text = "我喜欢红色,蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); - for (; grams.hasNext(); ) { - System.out.println(grams.next().extractFrom(text)); - } + assertEquals("我喜", grams.next().extractFrom(text)); + assertEquals("喜欢", grams.next().extractFrom(text)); + assertEquals("欢红", grams.next().extractFrom(text)); + assertEquals("红色", grams.next().extractFrom(text)); + assertEquals("蓝色", grams.next().extractFrom(text)); + assertEquals("色和", grams.next().extractFrom(text)); + assertEquals("和紫", grams.next().extractFrom(text)); + assertEquals("紫色", grams.next().extractFrom(text)); } - private void assertGramSplits(String input, int gramSize, String ... expected) { assertEquals(Arrays.asList(expected), gramSplitter.split(input, gramSize).toExtractedList()); } |