diff options
Diffstat (limited to 'model-integration/src/main/java/ai')
| -rw-r--r-- | model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java index 699dafdd94e..4ed1ae8dc0f 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java @@ -41,6 +41,7 @@ public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { outputName = config.transformerOutput(); normalize = config.normalize(); tokenizer = new HuggingFaceTokenizer.Builder() + .addSpecialTokens(true) .addDefaultModel(Paths.get(config.tokenizerPath().toString())) .build(); var onnxOpts = new OnnxEvaluatorOptions(); |