diff options
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index b8fb8a2fdbb..5b77324a6fc 100644 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java @@ -52,6 +52,12 @@ public class SentencePieceTest { } @Test + public void testSparseTensorEncoding() { + var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); + tester.assertEncoded("hello", "tensor(token{})", "{lo:1.0,'▁hel':0.0}"); + } + + @Test public void testNoCollapse() { var tester = new SentencePieceTester(new SentencePieceEncoder.Builder() .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) |