diff options
author | connell gough <connell@portrait-analytics.com> | 2023-04-11 09:52:39 -0700 |
---|---|---|
committer | connell gough <connell@portrait-analytics.com> | 2023-04-13 10:19:36 -0700 |
commit | 24626895a39d4abefe8c7ce19f4b52738a74d1c2 (patch) | |
tree | af9cfcbeef23f3529d051feb349bc30602997afa | |
parent | 5f157a6eedc43e4ddb19c99aeea721a93d740f89 (diff) |
Remove separator input and fix spelling error
-rw-r--r-- | model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java | 5 |
1 files changed, 2 insertions, 3 deletions
diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index a8c4d935cae..dcce64ad1ee 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -35,7 +35,6 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { private final int maxTokens; private final int startSequenceToken; private final int endSequenceToken; - private final int separatorToken; private final String inputIdsName; private final String attentionMaskName; private final String tokenTypeIdsName; @@ -100,7 +99,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { if (!type.dimensions().get(0).isIndexed()) { throw new IllegalArgumentException("Error in embedding to type '" + type + "': dimension should be indexed."); } - List<Integer> tokens = embedWithSeperatorTokens(text, context, maxTokens); + List<Integer> tokens = embedWithSeparatorTokens(text, context, maxTokens); return embedTokens(tokens, type); } @@ -140,7 +139,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { return builder.build(); } - private List<Integer> embedWithSeperatorTokens(String text, Context context, int maxLength) { + private List<Integer> embedWithSeparatorTokens(String text, Context context, int maxLength) { List<Integer> tokens = new ArrayList<>(); tokens.add(startSequenceToken); tokens.addAll(embed(text, context)); |