diff options
| author | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-05-11 15:41:00 +0200 |
|---|---|---|
| committer | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-05-11 16:41:54 +0200 |
| commit | fe63824738fc1892221311e7ddd777efcb209f5b (patch) | |
| tree | dc7d3ce16c4e56ab7cbbc941f2cb9f162d6dacb2 /linguistics-components/src/test/java | |
| parent | ae700d12753e1a81de4def087d2f64607f0361df (diff) | |
Disable special tokens by default
Diffstat (limited to 'linguistics-components/src/test/java')
| -rw-r--r-- | linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java b/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java index f9fa0ef2afe..c79ecbfbfbe 100644 --- a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java +++ b/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java @@ -71,6 +71,7 @@ class HuggingFaceTokenizerTest { private static HuggingFaceTokenizer createTokenizer(Path tmp, String model) throws IOException { return new HuggingFaceTokenizer.Builder() + .addSpecialTokens(false) .addDefaultModel(decompressModelFile(tmp, Paths.get("src/test/models/huggingface/%s.json.gz".formatted(model)))) .build(); } |