diff options
author | Arnstein Ressem <aressem@gmail.com> | 2023-05-12 10:05:12 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-12 10:05:12 +0200 |
commit | 2bb74878879b3acb1919fd658b8f2c476d8129d6 (patch) | |
tree | a10aa176e159797be725968c936521082239b935 /linguistics-components | |
parent | 4fcbaf3415490b0563b50d492ce87ce06e8b966a (diff) |
Revert "Bjorncs/huggingface tokenizer"
Diffstat (limited to 'linguistics-components')
8 files changed, 2 insertions, 303 deletions
diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml index 5031ad73556..ad4cbd6ce22 100644 --- a/linguistics-components/pom.xml +++ b/linguistics-components/pom.xml @@ -19,42 +19,12 @@ <artifactId>protobuf-java</artifactId> </dependency> <dependency> - <groupId>ai.djl.huggingface</groupId> - <artifactId>tokenizers</artifactId> - <version>0.22.1</version> - <exclusions> - <exclusion> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </exclusion> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> + <groupId>junit</groupId> + <artifactId>junit</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java deleted file mode 100644 index 107900ff73c..00000000000 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/Encoding.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.language.huggingface; - -import com.yahoo.api.annotations.Beta; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author bjorncs - */ -@Beta -public record Encoding( - List<Long> ids, List<Long> typeIds, List<String> tokens, List<Long> wordIds, List<Long> attentionMask, - List<Long> specialTokenMask, List<CharSpan> charTokenSpans, List<Encoding> overflowing) { - - public record CharSpan(int start, int end) { - public static final CharSpan NONE = new CharSpan(-1, -1); - static CharSpan from(ai.djl.huggingface.tokenizers.jni.CharSpan s) { - if (s == null) return NONE; - return new CharSpan(s.getStart(), s.getEnd()); - } - public boolean isNone() { return this.equals(NONE); } - } - - public Encoding { - ids = List.copyOf(ids); - typeIds = List.copyOf(typeIds); - tokens = List.copyOf(tokens); - wordIds = List.copyOf(wordIds); - attentionMask = List.copyOf(attentionMask); - specialTokenMask = List.copyOf(specialTokenMask); - charTokenSpans = List.copyOf(charTokenSpans); - overflowing = List.copyOf(overflowing); - } - - static Encoding from(ai.djl.huggingface.tokenizers.Encoding e) { - return new Encoding( - toList(e.getIds()), - toList(e.getTypeIds()), - List.of(e.getTokens()), - toList(e.getWordIds()), - toList(e.getAttentionMask()), - toList(e.getSpecialTokenMask()), - Arrays.stream(e.getCharTokenSpans()).map(CharSpan::from).toList(), - Arrays.stream(e.getOverflowing()).map(Encoding::from).toList()); - } - - private static List<Long> toList(long[] array) { - if (array == null) return List.of(); - var list = new ArrayList<Long>(array.length); - for (long e : array) list.add(e); - return list; - } -} diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java deleted file mode 100644 index b92e0678970..00000000000 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.language.huggingface; - -import com.yahoo.api.annotations.Beta; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.language.Language; -import com.yahoo.language.process.Embedder; -import com.yahoo.language.process.Segmenter; -import com.yahoo.language.tools.Embed; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * {@link Embedder}/{@link Segmenter} using Deep Java Library's HuggingFace Tokenizer. - * - * @author bjorncs - */ -@Beta -public class HuggingFaceTokenizer extends AbstractComponent implements Embedder, Segmenter, AutoCloseable { - - private final Map<Language, ai.djl.huggingface.tokenizers.HuggingFaceTokenizer> models = new EnumMap<>(Language.class); - - @Inject public HuggingFaceTokenizer(HuggingFaceTokenizerConfig cfg) { this(new Builder(cfg)); } - - private HuggingFaceTokenizer(Builder b) { - var original = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(HuggingFaceTokenizer.class.getClassLoader()); - try { - b.models.forEach((language, path) -> { - models.put(language, - uncheck(() -> ai.djl.huggingface.tokenizers.HuggingFaceTokenizer.builder() - .optTokenizerPath(path) - .optAddSpecialTokens(b.addSpecialTokens != null ? b.addSpecialTokens : true) - .build())); - }); - } finally { - Thread.currentThread().setContextClassLoader(original); - } - } - - @Override - public List<Integer> embed(String text, Context ctx) { - var encoding = resolve(ctx.getLanguage()).encode(text); - return Arrays.stream(encoding.getIds()).mapToInt(Math::toIntExact).boxed().toList(); - } - - @Override - public Tensor embed(String text, Context ctx, TensorType type) { - return Embed.asTensor(text, this, ctx, type); - } - - @Override - public List<String> segment(String input, Language language) { - return List.of(resolve(language).encode(input).getTokens()); - } - - @Override - public String decode(List<Integer> tokens, Context ctx) { - return resolve(ctx.getLanguage()).decode(toArray(tokens)); - } - - public Encoding encode(String text) { return encode(text, Language.UNKNOWN); } - public Encoding encode(String text, Language language) { return Encoding.from(resolve(language).encode(text)); } - public String decode(List<Long> tokens) { return decode(tokens, Language.UNKNOWN); } - public String decode(List<Long> tokens, Language language) { return resolve(language).decode(toArray(tokens)); } - - @Override public void close() { models.forEach((__, model) -> model.close()); } - - private ai.djl.huggingface.tokenizers.HuggingFaceTokenizer resolve(Language language) { - // Disregard language if there is default model - if (models.size() == 1 && models.containsKey(Language.UNKNOWN)) return models.get(Language.UNKNOWN); - if (models.containsKey(language)) return models.get(language); - throw new IllegalArgumentException("No model for language " + language); - } - - private static long[] toArray(Collection<? extends Number> c) { return c.stream().mapToLong(Number::longValue).toArray(); } - - public static final class Builder { - private final Map<Language, Path> models = new EnumMap<>(Language.class); - private Boolean addSpecialTokens; - - public Builder() {} - public Builder(HuggingFaceTokenizerConfig cfg) { - for (var model : cfg.model()) - addModel(Language.fromLanguageTag(model.language()), model.path()); - addSpecialTokens(cfg.addSpecialTokens()); - } - - public Builder addModel(Language lang, Path path) { models.put(lang, path); return this; } - public Builder addDefaultModel(Path path) { return addModel(Language.UNKNOWN, path); } - public Builder addSpecialTokens(boolean enabled) { addSpecialTokens = enabled; return this; } - public HuggingFaceTokenizer build() { return new HuggingFaceTokenizer(this); } - } - -}
\ No newline at end of file diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java deleted file mode 100644 index 7cec01ffed6..00000000000 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.language.huggingface; - -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/linguistics-components/src/main/resources/configdefinitions/language.huggingface.hugging-face-tokenizer.def b/linguistics-components/src/main/resources/configdefinitions/language.huggingface.hugging-face-tokenizer.def deleted file mode 100644 index a3e54ea38da..00000000000 --- a/linguistics-components/src/main/resources/configdefinitions/language.huggingface.hugging-face-tokenizer.def +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace=language.huggingface - -# The language a model is for, one of the language tags in com.yahoo.language.Language. -# Use "unknown" for models to be used with any language. -model[].language string -# The path to the model relative to the application package root -model[].path path - -addSpecialTokens bool default=true
\ No newline at end of file diff --git a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java b/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java deleted file mode 100644 index c79ecbfbfbe..00000000000 --- a/linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.language.huggingface; - -import com.yahoo.language.tools.EmbedderTester; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.zip.GZIPInputStream; - -/** - * @author bjorncs - */ -class HuggingFaceTokenizerTest { - - @TempDir Path tmp; - - @Test - void bert_tokenizer() throws IOException { - try (var tokenizer = createTokenizer(tmp, "bert-base-uncased")) { - var tester = new EmbedderTester(tokenizer); - tester.assertSegmented("what was the impact of the manhattan project", - "what", "was", "the", "impact", "of", "the", "manhattan", "project"); - tester.assertSegmented("overcommunication", "over", "##com", "##mun", "##ication"); - tester.assertEmbedded("what was the impact of the manhattan project", - "tensor(x[8])", - 2054, 2001, 1996, 4254, 1997, 1996, 7128, 2622); - tester.assertDecoded("what was the impact of the manhattan project"); - } - } - - @Test - void tokenizes_using_paraphrase_multilingual_mpnet_base_v2() throws IOException { - try (var tokenizer = createTokenizer(tmp, "paraphrase-multilingual-mpnet-base-v2")) { - var tester = new EmbedderTester(tokenizer); - tester.assertSegmented("h", "▁h"); - tester.assertSegmented("he", "▁he"); - tester.assertSegmented("hel", "▁hel"); - tester.assertSegmented("hello", "▁hell", "o"); - tester.assertSegmented("hei", "▁hei"); - tester.assertSegmented("hei you", "▁hei", "▁you"); - tester.assertSegmented("hei you", "▁hei", "▁you"); - tester.assertSegmented("this is another sentence", "▁this", "▁is", "▁another", "▁sentence"); - tester.assertSegmented("hello world!", "▁hell", "o", "▁world", "!"); - tester.assertSegmented("Hello, world!", "▁Hello", ",", "▁world", "!"); - tester.assertSegmented("HELLO, world!", "▁H", "ELLO", ",", "▁world", "!"); - tester.assertSegmented("KHJKJHHKJHHSH", "▁KH", "JK", "J", "H", "HK", "J", "HH", "SH"); - tester.assertSegmented("KHJKJHHKJHHSH hello", "▁KH", "JK", "J", "H", "HK", "J", "HH", "SH", "▁hell", "o"); - tester.assertSegmented(" hello ", "▁hell", "o"); - tester.assertSegmented(")(/&#()/\"\")", "▁", ")(", "/", "&#", "(", ")", "/", "\"", "\")"); - tester.assertSegmented(")(/&#(small)/\"in quotes\")", "▁", ")(", "/", "&#", "(", "s", "mall", ")", "/", "\"", "in", "▁quote", "s", "\")"); - tester.assertSegmented("x.400AS", "▁x", ".", "400", "AS"); - tester.assertSegmented("A normal sentence. Yes one more.", "▁A", "▁normal", "▁sentence", ".", "▁Yes", "▁one", "▁more", "."); - - tester.assertEmbedded("hello, world!", "tensor(d[10])", 33600, 31, 4, 8999, 38); - tester.assertEmbedded("Hello, world!", "tensor(d[10])", 35378, 4, 8999, 38); - tester.assertEmbedded("hello, world!", "tensor(d[2])", 33600, 31, 4, 8999, 38); - - tester.assertDecoded("this is a sentence"); - tester.assertDecoded("hello, world!"); - tester.assertDecoded(")(/&#(small)/ \"in quotes\")"); - } - } - - private static HuggingFaceTokenizer createTokenizer(Path tmp, String model) throws IOException { - return new HuggingFaceTokenizer.Builder() - .addSpecialTokens(false) - .addDefaultModel(decompressModelFile(tmp, Paths.get("src/test/models/huggingface/%s.json.gz".formatted(model)))) - .build(); - } - - private static Path decompressModelFile(Path tmp, Path source) throws IOException { - Path destination = tmp.resolve(source.getFileName().toString().replace(".gz", "")); - try (InputStream in = new GZIPInputStream(Files.newInputStream(source)); - OutputStream out = Files.newOutputStream(destination, StandardOpenOption.CREATE)) { - in.transferTo(out); - } - return destination; - } - -}
\ No newline at end of file diff --git a/linguistics-components/src/test/models/huggingface/bert-base-uncased.json.gz b/linguistics-components/src/test/models/huggingface/bert-base-uncased.json.gz Binary files differdeleted file mode 100644 index 7d0541849f7..00000000000 --- a/linguistics-components/src/test/models/huggingface/bert-base-uncased.json.gz +++ /dev/null diff --git a/linguistics-components/src/test/models/huggingface/paraphrase-multilingual-mpnet-base-v2.json.gz b/linguistics-components/src/test/models/huggingface/paraphrase-multilingual-mpnet-base-v2.json.gz Binary files differdeleted file mode 100644 index 7b61a27198c..00000000000 --- a/linguistics-components/src/test/models/huggingface/paraphrase-multilingual-mpnet-base-v2.json.gz +++ /dev/null |