BERT -> WordPiece, make subword prefix configurable

1 files changed, 122 insertions, 114 deletions

diff --git a/linguistics-components/abi-spec.json b/linguistics-components/abi-spec.json
index 6dba8b602bd..39666fd93a3 100644
--- a/linguistics-components/abi-spec.json
+++ b/linguistics-components/abi-spec.json
@@ -1,5 +1,22 @@
 {
-  "com.yahoo.language.bert.BertConfig$Builder": {
+  "com.yahoo.language.sentencepiece.Scoring": {
+    "superClass": "java.lang.Enum",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods": [
+      "public static com.yahoo.language.sentencepiece.Scoring[] values()",
+      "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)"
+    ],
+    "fields": [
+      "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore",
+      "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments"
+    ]
+  },
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigInstance$Builder"
@@ -9,23 +26,25 @@
     ],
     "methods": [
       "public void <init>()",
-      "public void <init>(com.yahoo.language.bert.BertConfig)",
-      "public com.yahoo.language.bert.BertConfig$Builder model(com.yahoo.language.bert.BertConfig$Model$Builder)",
-      "public com.yahoo.language.bert.BertConfig$Builder model(java.util.function.Consumer)",
-      "public com.yahoo.language.bert.BertConfig$Builder model(java.util.List)",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.function.Consumer)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)",
       "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)",
       "public final java.lang.String getDefMd5()",
       "public final java.lang.String getDefName()",
       "public final java.lang.String getDefNamespace()",
       "public final boolean getApplyOnRestart()",
       "public final void setApplyOnRestart(boolean)",
-      "public com.yahoo.language.bert.BertConfig build()"
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig build()"
     ],
     "fields": [
       "public java.util.List model"
     ]
   },
-  "com.yahoo.language.bert.BertConfig$Model$Builder": {
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigBuilder"
@@ -35,14 +54,14 @@
     ],
     "methods": [
       "public void <init>()",
-      "public void <init>(com.yahoo.language.bert.BertConfig$Model)",
-      "public com.yahoo.language.bert.BertConfig$Model$Builder language(java.lang.String)",
-      "public com.yahoo.language.bert.BertConfig$Model$Builder path(com.yahoo.config.FileReference)",
-      "public com.yahoo.language.bert.BertConfig$Model build()"
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()"
     ],
     "fields": []
   },
-  "com.yahoo.language.bert.BertConfig$Model": {
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": {
     "superClass": "com.yahoo.config.InnerNode",
     "interfaces": [],
     "attributes": [
@@ -50,13 +69,13 @@
       "final"
     ],
     "methods": [
-      "public void <init>(com.yahoo.language.bert.BertConfig$Model$Builder)",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)",
       "public java.lang.String language()",
       "public java.nio.file.Path path()"
     ],
     "fields": []
   },
-  "com.yahoo.language.bert.BertConfig$Producer": {
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigInstance$Producer"
@@ -67,11 +86,44 @@
       "abstract"
     ],
     "methods": [
-      "public abstract void getConfig(com.yahoo.language.bert.BertConfig$Builder)"
+      "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)"
     ],
     "fields": []
   },
-  "com.yahoo.language.bert.BertConfig": {
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": {
+    "superClass": "java.lang.Enum",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods": [
+      "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()",
+      "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)"
+    ],
+    "fields": [
+      "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore",
+      "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments"
+    ]
+  },
+  "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": {
+    "superClass": "com.yahoo.config.EnumNode",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)"
+    ],
+    "fields": [
+      "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore",
+      "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments"
+    ]
+  },
+  "com.yahoo.language.sentencepiece.SentencePieceConfig": {
     "superClass": "com.yahoo.config.ConfigInstance",
     "interfaces": [],
     "attributes": [
@@ -83,9 +135,11 @@
       "public static java.lang.String getDefName()",
       "public static java.lang.String getDefNamespace()",
       "public static java.lang.String getDefVersion()",
-      "public void <init>(com.yahoo.language.bert.BertConfig$Builder)",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)",
+      "public boolean collapseUnknowns()",
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()",
       "public java.util.List model()",
-      "public com.yahoo.language.bert.BertConfig$Model model(int)"
+      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)"
     ],
     "fields": [
       "public static final java.lang.String CONFIG_DEF_MD5",
@@ -95,56 +149,47 @@
       "public static final java.lang.String[] CONFIG_DEF_SCHEMA"
     ]
   },
-  "com.yahoo.language.bert.BertEmbedder$Builder": {
+  "com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [],
     "attributes": [
-      "public"
+      "public",
+      "final"
     ],
     "methods": [
       "public void <init>()",
+      "public void <init>(java.lang.String)",
       "public void addModel(com.yahoo.language.Language, java.nio.file.Path)",
-      "public com.yahoo.language.bert.BertEmbedder$Builder addDefaultModel(java.nio.file.Path)",
+      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder addDefaultModel(java.nio.file.Path)",
       "public java.util.Map getModels()",
-      "public com.yahoo.language.bert.BertEmbedder build()"
+      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder setCollapseUnknowns(boolean)",
+      "public boolean getCollapseUnknowns()",
+      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)",
+      "public com.yahoo.language.sentencepiece.Scoring getScoring()",
+      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder build()"
     ],
     "fields": []
   },
-  "com.yahoo.language.bert.BertEmbedder": {
+  "com.yahoo.language.sentencepiece.SentencePieceEmbedder": {
     "superClass": "java.lang.Object",
     "interfaces": [
-      "com.yahoo.language.process.Embedder",
-      "com.yahoo.language.process.Segmenter"
+      "com.yahoo.language.process.Segmenter",
+      "com.yahoo.language.process.Embedder"
     ],
     "attributes": [
       "public"
     ],
     "methods": [
-      "public void <init>(com.yahoo.language.bert.BertConfig)",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)",
+      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder)",
       "public java.util.List segment(java.lang.String, com.yahoo.language.Language)",
       "public java.util.List embed(java.lang.String, com.yahoo.language.process.Embedder$Context)",
-      "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.process.Embedder$Context, com.yahoo.tensor.TensorType)"
+      "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.process.Embedder$Context, com.yahoo.tensor.TensorType)",
+      "public java.lang.String normalize(java.lang.String)"
     ],
     "fields": []
   },
-  "com.yahoo.language.sentencepiece.Scoring": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.language.sentencepiece.Scoring[] values()",
-      "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore",
-      "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments"
-    ]
-  },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": {
+  "com.yahoo.language.wordpiece.WordPieceConfig$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigInstance$Builder"
@@ -154,25 +199,24 @@
     ],
     "methods": [
       "public void <init>()",
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.function.Consumer)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)",
+      "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Builder subwordPrefix(java.lang.String)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(java.util.function.Consumer)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Builder model(java.util.List)",
       "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)",
       "public final java.lang.String getDefMd5()",
       "public final java.lang.String getDefName()",
       "public final java.lang.String getDefNamespace()",
       "public final boolean getApplyOnRestart()",
       "public final void setApplyOnRestart(boolean)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig build()"
+      "public com.yahoo.language.wordpiece.WordPieceConfig build()"
     ],
     "fields": [
       "public java.util.List model"
     ]
   },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": {
+  "com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigBuilder"
@@ -182,14 +226,14 @@
     ],
     "methods": [
       "public void <init>()",
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()"
+      "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Model)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder language(java.lang.String)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder path(com.yahoo.config.FileReference)",
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Model build()"
     ],
     "fields": []
   },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": {
+  "com.yahoo.language.wordpiece.WordPieceConfig$Model": {
     "superClass": "com.yahoo.config.InnerNode",
     "interfaces": [],
     "attributes": [
@@ -197,13 +241,13 @@
       "final"
     ],
     "methods": [
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)",
+      "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Model$Builder)",
       "public java.lang.String language()",
       "public java.nio.file.Path path()"
     ],
     "fields": []
   },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": {
+  "com.yahoo.language.wordpiece.WordPieceConfig$Producer": {
     "superClass": "java.lang.Object",
     "interfaces": [
       "com.yahoo.config.ConfigInstance$Producer"
@@ -214,44 +258,11 @@
       "abstract"
     ],
     "methods": [
-      "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)"
+      "public abstract void getConfig(com.yahoo.language.wordpiece.WordPieceConfig$Builder)"
     ],
     "fields": []
   },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()",
-      "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore",
-      "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments"
-    ]
-  },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": {
-    "superClass": "com.yahoo.config.EnumNode",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public void <init>()",
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)"
-    ],
-    "fields": [
-      "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore",
-      "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments"
-    ]
-  },
-  "com.yahoo.language.sentencepiece.SentencePieceConfig": {
+  "com.yahoo.language.wordpiece.WordPieceConfig": {
     "superClass": "com.yahoo.config.ConfigInstance",
     "interfaces": [],
     "attributes": [
@@ -263,11 +274,10 @@
       "public static java.lang.String getDefName()",
       "public static java.lang.String getDefNamespace()",
       "public static java.lang.String getDefVersion()",
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)",
-      "public boolean collapseUnknowns()",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()",
+      "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Builder)",
+      "public java.lang.String subwordPrefix()",
       "public java.util.List model()",
-      "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)"
+      "public com.yahoo.language.wordpiece.WordPieceConfig$Model model(int)"
     ],
     "fields": [
       "public static final java.lang.String CONFIG_DEF_MD5",
@@ -277,41 +287,39 @@
       "public static final java.lang.String[] CONFIG_DEF_SCHEMA"
     ]
   },
-  "com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder": {
+  "com.yahoo.language.wordpiece.WordPieceEmbedder$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [],
     "attributes": [
-      "public"
+      "public",
+      "final"
     ],
     "methods": [
       "public void <init>()",
+      "public void <init>(java.lang.String)",
+      "public com.yahoo.language.wordpiece.WordPieceEmbedder$Builder setSubwordPrefix(java.lang.String)",
+      "public java.lang.String getSubwordPrefix()",
       "public void addModel(com.yahoo.language.Language, java.nio.file.Path)",
-      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder addDefaultModel(java.nio.file.Path)",
+      "public com.yahoo.language.wordpiece.WordPieceEmbedder$Builder addDefaultModel(java.nio.file.Path)",
       "public java.util.Map getModels()",
-      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder setCollapseUnknowns(boolean)",
-      "public boolean getCollapseUnknowns()",
-      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)",
-      "public com.yahoo.language.sentencepiece.Scoring getScoring()",
-      "public com.yahoo.language.sentencepiece.SentencePieceEmbedder build()"
+      "public com.yahoo.language.wordpiece.WordPieceEmbedder build()"
     ],
     "fields": []
   },
-  "com.yahoo.language.sentencepiece.SentencePieceEmbedder": {
+  "com.yahoo.language.wordpiece.WordPieceEmbedder": {
     "superClass": "java.lang.Object",
     "interfaces": [
-      "com.yahoo.language.process.Segmenter",
-      "com.yahoo.language.process.Embedder"
+      "com.yahoo.language.process.Embedder",
+      "com.yahoo.language.process.Segmenter"
     ],
     "attributes": [
       "public"
     ],
     "methods": [
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)",
-      "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEmbedder$Builder)",
+      "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig)",
       "public java.util.List segment(java.lang.String, com.yahoo.language.Language)",
       "public java.util.List embed(java.lang.String, com.yahoo.language.process.Embedder$Context)",
-      "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.process.Embedder$Context, com.yahoo.tensor.TensorType)",
-      "public java.lang.String normalize(java.lang.String)"
+      "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.process.Embedder$Context, com.yahoo.tensor.TensorType)"
     ],
     "fields": []
   }