diff options
author | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-05-08 15:15:52 +0200 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-05-08 15:15:52 +0200 |
commit | 76a294520e94352d4256f3d579c6d5aa44522995 (patch) | |
tree | d259ec33be51d6445ca9ba0a61fe7171c01a0166 /model-integration | |
parent | 52c04d5a633f4571300f75e3024c5198d484a267 (diff) |
Handle nulls
Diffstat (limited to 'model-integration')
-rw-r--r-- | model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java index f1c0244bfb3..274c29a57b2 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java @@ -14,9 +14,12 @@ public record Encoding( List<Long> specialTokenMask, List<CharSpan> charTokenSpans, List<Encoding> overflowing) { public record CharSpan(int start, int end) { + public static final CharSpan NONE = new CharSpan(-1, -1); static CharSpan from(ai.djl.huggingface.tokenizers.jni.CharSpan s) { + if (s == null) return NONE; return new CharSpan(s.getStart(), s.getEnd()); } + public boolean isNone() { return this.equals(NONE); } } public Encoding { @@ -43,6 +46,7 @@ public record Encoding( } private static List<Long> toList(long[] array) { + if (array == null) return List.of(); var list = new ArrayList<Long>(array.length); for (long e : array) list.add(e); return list; |