diff options
author | Lester Solbakken <lester.solbakken@gmail.com> | 2024-04-12 13:04:01 +0200 |
---|---|---|
committer | Lester Solbakken <lester.solbakken@gmail.com> | 2024-04-12 13:04:01 +0200 |
commit | b4d6c0856e0313e8925505265121ce743f079ada (patch) | |
tree | fd8a28fb707d7dae4ac9c4f19d48aad082cf01ff /linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java | |
parent | e11438c6335038f6d99ea50eef086511eb204d43 (diff) | |
parent | dd996cbeb9ddf7c22089b6115966f03227b35a48 (diff) |
Merge branch 'master' into lesters/add-local-llms
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java | 13 |
1 files changed, 6 insertions, 7 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java index cf5a26c1f04..2cfb6c33b93 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java @@ -7,7 +7,6 @@ import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -31,23 +30,23 @@ public class TokenizationTestCase { @Test public void testTokenizer() { assertTokenize("This is a test, 123", - Arrays.asList("this", "is", "a", "test", "123"), - Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); + List.of("this", "is", "a", "test", "123"), + List.of("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); } @Test public void testUnderScoreTokenization() { - assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null); + assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, List.of("ugcapi", "1"), null); } @Test public void testPhrasesWithPunctuation() { assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE, false, - Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), + List.of("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), null); - assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null); - assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null); + assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, List.of("phy", "101"), null); + assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, List.of("101", "3"), null); } @Test |