diff options
Diffstat (limited to 'linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java')
-rw-r--r-- | linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java index 8a7af01a8a3..daa31f8773b 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java @@ -3,11 +3,14 @@ package com.yahoo.language.sentencepiece; import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; import com.yahoo.language.tools.EmbedderTester; import org.junit.Test; import java.io.File; +import static org.junit.Assert.assertEquals; + /** * @author bratseth */ @@ -53,6 +56,15 @@ public class SentencePieceTest { } @Test + public void testSkipControl() { + var embedder = new SentencePieceEmbedder.Builder("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").build(); + var context = new Embedder.Context("test"); + var tokens = embedder.embed("<s>hello</s>, world!", context); + assertEquals("<s>hello</s>, world!", embedder.decode(tokens, context, false)); + assertEquals("hello, world!", embedder.decode(tokens, context, true)); + } + + @Test public void testNoCollapse() { var builder = new SentencePieceEmbedder.Builder() .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) |