aboutsummaryrefslogtreecommitdiffstats
path: root/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
diff options
context:
space:
mode:
authorLester Solbakken <lester.solbakken@gmail.com>2024-04-12 13:04:01 +0200
committerLester Solbakken <lester.solbakken@gmail.com>2024-04-12 13:04:01 +0200
commitb4d6c0856e0313e8925505265121ce743f079ada (patch)
treefd8a28fb707d7dae4ac9c4f19d48aad082cf01ff /linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
parente11438c6335038f6d99ea50eef086511eb204d43 (diff)
parentdd996cbeb9ddf7c22089b6115966f03227b35a48 (diff)
Merge branch 'master' into lesters/add-local-llms
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java')
-rw-r--r--linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java13
1 files changed, 6 insertions, 7 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
index cf5a26c1f04..2cfb6c33b93 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
@@ -7,7 +7,6 @@ import org.junit.Test;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
@@ -31,23 +30,23 @@ public class TokenizationTestCase {
@Test
public void testTokenizer() {
assertTokenize("This is a test, 123",
- Arrays.asList("this", "is", "a", "test", "123"),
- Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123"));
+ List.of("this", "is", "a", "test", "123"),
+ List.of("This", " ", "is", " ", "a", " ", "test", ",", " ", "123"));
}
@Test
public void testUnderScoreTokenization() {
- assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null);
+ assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, List.of("ugcapi", "1"), null);
}
@Test
public void testPhrasesWithPunctuation() {
assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE,
false,
- Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"),
+ List.of("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"),
null);
- assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null);
- assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null);
+ assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, List.of("phy", "101"), null);
+ assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, List.of("101", "3"), null);
}
@Test