summaryrefslogtreecommitdiffstats
path: root/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java')
-rw-r--r--linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java19
1 files changed, 19 insertions, 0 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
index 8fa23626193..a37ef6bfde5 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
@@ -167,6 +167,25 @@ public class GramSplitterTestCase {
assertFalse(grams.hasNext());
}
+ @Test
+ public void testChineseComma() {
+ String text = "我喜欢红色、蓝色和紫色";
+ Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2);
+ for (; grams.hasNext(); ) {
+ System.out.println(grams.next().extractFrom(text));
+ }
+ }
+
+ @Test
+ public void testEnglishComma() {
+ String text = "我喜欢红色,蓝色和紫色";
+ Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2);
+ for (; grams.hasNext(); ) {
+ System.out.println(grams.next().extractFrom(text));
+ }
+ }
+
+
private void assertGramSplits(String input, int gramSize, String ... expected) {
assertEquals(Arrays.asList(expected), gramSplitter.split(input, gramSize).toExtractedList());
}