summaryrefslogtreecommitdiffstats
path: root/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java')
-rw-r--r--linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java15
1 files changed, 14 insertions, 1 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
index fa8419e200f..6cefcfbf67a 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
@@ -1,13 +1,17 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
+import com.yahoo.language.process.GramSplitter.Gram;
+import com.yahoo.language.process.GramSplitter.GramSplitterIterator;
import com.yahoo.language.simple.SimpleLinguistics;
import org.junit.Test;
import java.util.Arrays;
import java.util.Iterator;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
/**
* @author bratseth
@@ -168,6 +172,15 @@ public class GramSplitterTestCase {
}
@Test
+ public void testLongString() {
+ String input = "hey ho come 色 let's go, and then we go again!\n色色色".repeat(10_000);
+ for (GramSplitterIterator grams = new GramSplitter(new CharacterClasses()).split(input, 3); grams.hasNext(); ) {
+ Gram gram = grams.next();
+ gram.extractFrom(input);
+ }
+ }
+
+ @Test
public void testChineseComma() {
String text = "我喜欢红色、蓝色和紫色";
Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2);