summaryrefslogtreecommitdiffstats
path: root/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java')
-rw-r--r--linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java12
1 files changed, 12 insertions, 0 deletions
diff --git a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java
index 8a7af01a8a3..daa31f8773b 100644
--- a/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java
+++ b/linguistics-components/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java
@@ -3,11 +3,14 @@
package com.yahoo.language.sentencepiece;
import com.yahoo.language.Language;
+import com.yahoo.language.process.Embedder;
import com.yahoo.language.tools.EmbedderTester;
import org.junit.Test;
import java.io.File;
+import static org.junit.Assert.assertEquals;
+
/**
* @author bratseth
*/
@@ -53,6 +56,15 @@ public class SentencePieceTest {
}
@Test
+ public void testSkipControl() {
+ var embedder = new SentencePieceEmbedder.Builder("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").build();
+ var context = new Embedder.Context("test");
+ var tokens = embedder.embed("<s>hello</s>, world!", context);
+ assertEquals("<s>hello</s>, world!", embedder.decode(tokens, context, false));
+ assertEquals("hello, world!", embedder.decode(tokens, context, true));
+ }
+
+ @Test
public void testNoCollapse() {
var builder = new SentencePieceEmbedder.Builder()
.addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath())