aboutsummaryrefslogtreecommitdiffstats
path: root/configdefinitions/src/vespa/hugging-face-tokenizer.def
blob: a82e4b4a1f4b2558f3d5852acba3d97aaaacaa84 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

namespace=language.huggingface.config

# The language a model is for, one of the language tags in com.yahoo.language.Language.
# Use "unknown" for models to be used with any language.
model[].language string
# The path to the model relative to the application package root
model[].path model

# Include special tokens in output
addSpecialTokens bool default=true

# Used for truncation/padding. Use -1 for model default.
maxLength int default=-1

# Truncation strategy. Use NOTSET for model default.
truncation enum { ON, OFF, NOTSET } default=NOTSET

# Padding strategy. Use NOTSET for model default.
padding enum { ON, OFF, NOTSET } default=NOTSET