diff options
| author | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-06-08 15:48:28 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-06-08 15:48:28 +0200 |
| commit | adf6ed430f5d86def123f03cc6a967d035255f14 (patch) | |
| tree | 6f2a920a73760691f4c9290469003a7905b9dab2 /config-model | |
| parent | eae78b80d5d94e4cff9eb112637be20eb998712d (diff) | |
| parent | 7ae3e26016c32175aaf437ac09fca48457eb338e (diff) | |
Merge pull request #27349 from vespa-engine/bjorncs/disable-padding
Bjorncs/disable padding
Diffstat (limited to 'config-model')
| -rw-r--r-- | config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java | 3 | ||||
| -rw-r--r-- | config-model/src/main/resources/schema/common.rnc | 3 |
2 files changed, 5 insertions, 1 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java index 966dbe8260a..e0572f8391e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java @@ -23,6 +23,7 @@ public class HuggingFaceTokenizer extends TypedComponent implements HuggingFaceT private final Boolean specialTokens; private final Integer maxLength; private final Boolean truncation; + private final Boolean padding; public HuggingFaceTokenizer(Element xml, DeployState state) { super("com.yahoo.language.huggingface.HuggingFaceTokenizer", LINGUISTICS_BUNDLE_NAME, xml); @@ -33,6 +34,7 @@ public class HuggingFaceTokenizer extends TypedComponent implements HuggingFaceT specialTokens = getOptionalChildValue(xml, "special-tokens").map(Boolean::parseBoolean).orElse(null); maxLength = getOptionalChildValue(xml, "max-length").map(Integer::parseInt).orElse(null); truncation = getOptionalChildValue(xml, "truncation").map(Boolean::parseBoolean).orElse(null); + padding = getOptionalChildValue(xml, "padding").map(Boolean::parseBoolean).orElse(null); } @Override @@ -43,5 +45,6 @@ public class HuggingFaceTokenizer extends TypedComponent implements HuggingFaceT if (specialTokens != null) builder.addSpecialTokens(specialTokens); if (maxLength != null) builder.maxLength(maxLength); if (truncation != null) builder.truncation(truncation); + if (padding != null) builder.padding(padding); } } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 061e54740f1..e130bed0297 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -102,7 +102,8 @@ HuggingFaceTokenizer = element model { attribute language { xsd:string }? & ModelReference }+ & element special-tokens { xsd:boolean }? & element max-length { xsd:integer }? & - element truncation { xsd:boolean }? + element truncation { xsd:boolean }? & + element padding { xsd:boolean }? BertBaseEmbedder = attribute type { "bert-embedder" } & |