BERT -> WordPiece, make subword prefix configurable

1 files changed, 14 insertions, 0 deletions

diff --git a/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def
new file mode 100644
index 00000000000..08592250eb5
--- /dev/null
+++ b/linguistics-components/src/main/resources/configdefinitions/language.wordpiece.word-piece.def
@@ -0,0 +1,14 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# Configures com.yahoo.language.wordpiece.WordPieceEmbedder
+
+namespace=language.wordpiece
+
+# The prefix to prepend to subword tokens
+subwordPrefix string default="##"
+
+# The language a model is for, one of the language tags in com.yahoo.language.Language.
+# Use "unknown" for a model to be used for any language (i.e by default).
+model[].language string
+# The path to the model relative to the application package root
+model[].path path