diff options
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index fa8419e200f..6cefcfbf67a 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -1,13 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.process; +import com.yahoo.language.process.GramSplitter.Gram; +import com.yahoo.language.process.GramSplitter.GramSplitterIterator; import com.yahoo.language.simple.SimpleLinguistics; import org.junit.Test; import java.util.Arrays; import java.util.Iterator; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author bratseth @@ -168,6 +172,15 @@ public class GramSplitterTestCase { } @Test + public void testLongString() { + String input = "hey ho come 色 let's go, and then we go again!\n色色色".repeat(10_000); + for (GramSplitterIterator grams = new GramSplitter(new CharacterClasses()).split(input, 3); grams.hasNext(); ) { + Gram gram = grams.next(); + gram.extractFrom(input); + } + } + + @Test public void testChineseComma() { String text = "我喜欢红色、蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); |