diff options
Diffstat (limited to 'linguistics')
17 files changed, 0 insertions, 1206 deletions
diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index dbf4842ea1a..cfbf2abda1a 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -731,192 +731,5 @@ "public abstract java.lang.String accentDrop(java.lang.String, com.yahoo.language.Language)" ], "fields": [] - }, - "com.yahoo.language.sentencepiece.Scoring": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.Scoring[] values()", - "public static com.yahoo.language.sentencepiece.Scoring valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.Scoring highestScore", - "public static final enum com.yahoo.language.sentencepiece.Scoring fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder collapseUnknowns(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder scoring(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Builder model(java.util.List)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig build()" - ], - "fields": [ - "public java.util.List model" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder language(java.lang.String)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder path(com.yahoo.config.FileReference)", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Model": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Model$Builder)", - "public java.lang.String language()", - "public java.nio.file.Path path()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum[] values()", - "public static com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final enum com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum)" - ], - "fields": [ - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum highestScore", - "public static final com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum fewestSegments" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", - "public boolean collapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", - "public java.util.List model()", - "public com.yahoo.language.sentencepiece.SentencePieceConfig$Model model(int)" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void addModel(com.yahoo.language.Language, java.nio.file.Path)", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder addDefaultModel(java.nio.file.Path)", - "public java.util.Map getModels()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setCollapseUnknowns(boolean)", - "public boolean getCollapseUnknowns()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder setScoring(com.yahoo.language.sentencepiece.Scoring)", - "public com.yahoo.language.sentencepiece.Scoring getScoring()", - "public com.yahoo.language.sentencepiece.SentencePieceEncoder build()" - ], - "fields": [] - }, - "com.yahoo.language.sentencepiece.SentencePieceEncoder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.language.process.Segmenter", - "com.yahoo.language.process.Encoder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig)", - "public void <init>(com.yahoo.language.sentencepiece.SentencePieceEncoder$Builder)", - "public java.util.List segment(java.lang.String, com.yahoo.language.Language)", - "public java.util.List encode(java.lang.String, com.yahoo.language.Language)", - "public com.yahoo.tensor.Tensor encode(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)", - "public java.lang.String normalize(java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 221d7181616..0e5f9e15b85 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -15,10 +15,6 @@ <version>7-SNAPSHOT</version> <dependencies> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java deleted file mode 100644 index 74f300057dc..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Model.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -import com.yahoo.io.IOUtils; -import com.yahoo.language.Language; -import sentencepiece.SentencepieceModel; - -import java.io.IOException; -import java.nio.file.Path; - -/** - * A SentencePiece model - * - * @author bratseth - */ -final class Model { - - final Path source; - final Language language; - final float minScore; - final float maxScore; - final Trie tokens = new Trie(); - - Model(Language language, Path path) { - try { - this.source = path; - this.language = language; - var sp = SentencepieceModel.ModelProto.parseFrom(IOUtils.readFileBytes(path.toFile())); - float minScore = Float.MAX_VALUE; - float maxScore = Float.MIN_VALUE; - for (int i = 0; i < sp.getPiecesCount(); i++) { - var piece = sp.getPieces(i); - tokens.add(toTokenType(piece.getType()), i, piece.getPiece(), piece.getScore()); - minScore = Math.min(piece.getScore(), minScore); - maxScore = Math.max(piece.getScore(), maxScore); - } - this.minScore = minScore; - this.maxScore = maxScore; - } catch (IOException e) { - throw new IllegalArgumentException("Could not read a SentencePiece model from " + path, e); - } - } - - private static TokenType toTokenType(SentencepieceModel.ModelProto.SentencePiece.Type type) { - switch (type) { - case USER_DEFINED : return TokenType.userDefined; - case UNKNOWN : return TokenType.unknown; - case NORMAL : return TokenType.text; - case CONTROL : return TokenType.control; - case UNUSED : return TokenType.unused; - default : throw new IllegalArgumentException("Unknkown token type " + type); - } - } - - @Override - public String toString() { - return "SentencePiece model for " + language + ": '" + source + "'"; - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java deleted file mode 100644 index 2141505374c..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/ResultBuilder.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -/** - * Builds a result from a sentencepiece tokenization by being called for each segment in reverse - * - * @param <RESULTTYPE> the type of result this produces - * @author bratseth - */ -abstract class ResultBuilder<RESULTTYPE> { - - private final RESULTTYPE result; - - ResultBuilder(RESULTTYPE result) { - this.result = result; - } - - /** Called for each segment, starting from the last and working backwards */ - abstract void add(int start, int end, SentencePieceAlgorithm.SegmentEnd[] segmentEnds); - - RESULTTYPE result() {return result;} - - void build(String input, SentencePieceAlgorithm.SegmentEnd[] segmentEnds, boolean collapseUnknowns) { - if (collapseUnknowns) { - int segmentEnd = input.length(); - int collapsedSegmentEnd = segmentEnd; - while (segmentEnd > 0) { - if (segmentEnds[segmentEnd].type != TokenType.unknown ) { - if (collapsedSegmentEnd != segmentEnd) { // We have deferred an unknown collapsed segment - add(segmentEnd, collapsedSegmentEnd, segmentEnds); - } - add(segmentEnds[segmentEnd].segmentStart, segmentEnd, segmentEnds); - collapsedSegmentEnd = segmentEnds[segmentEnd].segmentStart; - } - segmentEnd = segmentEnds[segmentEnd].segmentStart; - } - } - else { - int segmentEnd = input.length(); - while (segmentEnd > 0) { - add(segmentEnds[segmentEnd].segmentStart, segmentEnd, segmentEnds); - segmentEnd = segmentEnds[segmentEnd].segmentStart; - } - } - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java deleted file mode 100644 index 6c8560abee7..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Scoring.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -/** - * The scoring strategy to use for picking segments - * - * @author bratseth - */ -public enum Scoring { - - /** Find the segmentation that has the highest score */ - highestScore, - - /** Find the segmentation that has the fewest segments, resolve ties by score sum */ - fewestSegments - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java deleted file mode 100644 index 1659e3c0fa7..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceAlgorithm.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -/** - * SentencePiece algorithm implementation - * - * @author bratseth - */ -class SentencePieceAlgorithm { - - // TODO: Support characters beyond BMP - - static final char spaceSymbol = '▁'; - - private final boolean collapseUnknowns; - private final Scoring scoring; - - SentencePieceAlgorithm(boolean collapseUnknowns, Scoring scoring) { - this.collapseUnknowns = collapseUnknowns; - this.scoring = scoring; - } - - public <RESULTTYPE> void segment(String input, ResultBuilder<RESULTTYPE> resultBuilder, Model model) { - SegmentEnd[] segmentEnds = new SegmentEnd[input.length() + 1]; - segmentEnds[0] = new SegmentEnd(TokenType.unknown, 0, 0, 0, 0); - int start = 0; - while (start < input.length()) { // segment from this position to the end of the text - Trie.Node node = model.tokens.root; - int characterPosition = start; - while (node != null && characterPosition < input.length()) { // traverse the trie one character at the time from this position - node = node.children.get(input.charAt(characterPosition++)); - int length = characterPosition - start; - if (node != null && node.isToken() && node.type != TokenType.unused) { - float score = node.type == TokenType.userDefined ? (length * model.maxScore - 0.1f) : node.score; - addSegment(TokenType.text, node.id, start, characterPosition, score, segmentEnds); - } - else if (length == 1) { // add an 'unknown' length 1 token to make the next position reachable - addSegment(TokenType.unknown, 0, start, start + 1, model.minScore - 10.0f, segmentEnds); - } - } - start++; - } - resultBuilder.build(input, segmentEnds, collapseUnknowns); - } - - private void addSegment(TokenType type, int id, int start, int end, float score, SegmentEnd[] segmentEnds) { - if (segmentEnds[end] == null || - segmentEnds[start].scoreWith(score) > segmentEnds[end].score()) { - segmentEnds[end] = new SegmentEnd(type, id, - segmentEnds[start].pathScoreSum + score, - segmentEnds[start].pathSegmentCount + 1, - start); - } - } - - final class SegmentEnd { - - final TokenType type; - final int id; - final float pathScoreSum; - final int pathSegmentCount; - final int segmentStart; - - SegmentEnd(TokenType type, int id, float pathScoreSum, int pathSegmentCount, int segmentStart) { - this.type = type; - this.id = id; - this.pathScoreSum = pathScoreSum; - this.pathSegmentCount = pathSegmentCount; - this.segmentStart = segmentStart; - } - - public float score() { - switch (scoring) { - case fewestSegments: return 1f / pathSegmentCount * 10_000_000 + pathScoreSum; - case highestScore: return pathScoreSum; - default : throw new IllegalArgumentException("Unknown scoring " + scoring); - } - } - - public float scoreWith(float additionalSegmentScore) { - switch (scoring) { - case fewestSegments: return 1f / (pathSegmentCount + 1) * 10_000_000 + (pathScoreSum + additionalSegmentScore ); - case highestScore: return pathScoreSum + additionalSegmentScore; - default : throw new IllegalArgumentException("Unknown scoring " + scoring); - } - } - - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java deleted file mode 100644 index b6659ebeaa3..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -import com.google.common.annotations.Beta; -import com.google.inject.Inject; -import com.yahoo.language.Language; -import com.yahoo.language.process.Encoder; -import com.yahoo.language.process.Segmenter; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorAddress; -import com.yahoo.tensor.TensorType; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Integration with https://github.com/google/sentencepiece - * through http://docs.djl.ai/extensions/sentencepiece/index.html - * - * SentencePiece is a language-agnostic tokenizer for neural nets. - * - * @author bratseth - */ -@Beta -public class SentencePieceEncoder implements Segmenter, Encoder { - - private final Map<Language, Model> models; - - private final SentencePieceAlgorithm algorithm; - - @Inject - public SentencePieceEncoder(SentencePieceConfig config) { - this(new Builder(config)); - } - - public SentencePieceEncoder(Builder builder) { - algorithm = new SentencePieceAlgorithm(builder.collapseUnknowns, builder.getScoring()); - - models = builder.getModels().entrySet() - .stream() - .map(e -> new Model(e.getKey(), e.getValue())) - .collect(Collectors.toUnmodifiableMap(m -> m.language, m -> m)); - if (models.isEmpty()) - throw new IllegalArgumentException("SentencePieceEncoder requires at least one model configured"); - } - - /** - * Segments the given text into token segments using the SentencePiece algorithm - * - * @param rawInput the text to segment. Any sequence of BMP (Unicode-16 the True Unicode) is supported. - * @param language the model to use, or Language.UNKNOWN to use the default model if any - * @return the list of zero or more tokens resulting from segmenting the input text - */ - @Override - public List<String> segment(String rawInput, Language language) { - String input = normalize(rawInput); - var resultBuilder = new ResultBuilder<List<String>>(new ArrayList<>()) { - public void add(int segmentStart, int segmentEnd, SentencePieceAlgorithm.SegmentEnd[] segmentEnds) { - result().add(input.substring(segmentStart, segmentEnd)); - } - }; - segment(input, language, resultBuilder); - Collections.reverse(resultBuilder.result()); - return resultBuilder.result(); - } - - /** - * Segments the given text into token segments using the SentencePiece algorithm and returns the segment ids. - * - * @param rawInput the text to segment. Any sequence of BMP (Unicode-16 the True Unicode) is supported. - * @param language the model to use, or Language.UNKNOWN to use the default model if any - * @return the list of zero or more token ids resulting from segmenting the input text - */ - @Override - public List<Integer> encode(String rawInput, Language language) { - var resultBuilder = new ResultBuilder<List<Integer>>(new ArrayList<>()) { - public void add(int segmentStart, int segmentEnd, SentencePieceAlgorithm.SegmentEnd[] segmentEnds) { - result().add(segmentEnds[segmentEnd].id); - } - }; - segment(normalize(rawInput), language, resultBuilder); - Collections.reverse(resultBuilder.result()); - return resultBuilder.result(); - } - - /** - * <p>Encodes directly to a tensor.</p> - * - * <p>If the tensor type is indexed 1-d (bound or unbound) this will return a tensor containing the token ids in the order - * they were encountered in the text. If the dimension is bound and too large it will be zero padded, if too small - * it will be truncated.</p> - * - * <p>If the tensor type is1-d sparse this will return a tensor containing the token strings as keys and the token - * position as value.</p> - * - * <p>If the tensor is any other type IllegalArgumentException is thrown.</p> - */ - @Override - public Tensor encode(String rawInput, Language language, TensorType type) { - if (type.dimensions().size() == 1 && type.dimensions().get(0).isIndexed()) { - // Build to a list first since we can't reverse a tensor builder - List<Integer> values = encode(rawInput, language); - - long maxSize = values.size(); - if (type.dimensions().get(0).size().isPresent()) - maxSize = Math.min(maxSize, type.dimensions().get(0).size().get()); - - Tensor.Builder builder = Tensor.Builder.of(type); - for (int i = 0; i < maxSize; i++) - builder.cell(values.get(i), i); - return builder.build(); - } - else if (type.dimensions().size() == 1 && type.dimensions().get(0).isMapped()) { - // Build to a list first since we can't reverse a tensor builder - List<String> values = segment(rawInput, language); - - Tensor.Builder builder = Tensor.Builder.of(type); - for (int i = 0; i < values.size(); i++) - builder.cell(TensorAddress.ofLabels(values.get(i)), i); - return builder.build(); - } - else { - throw new IllegalArgumentException("Don't know how to encode with SentencePiece into " + type); - } - } - - private <RESULTTYPE> void segment(String input, Language language, - ResultBuilder<RESULTTYPE> resultBuilder) { - Model model = resolveFrom(language); - algorithm.segment(input, resultBuilder, model); - } - - private Model resolveFrom(Language language) { - // Disregard language if there is default model - if (models.size() == 1 && models.containsKey(Language.UNKNOWN)) return models.get(Language.UNKNOWN); - if (models.containsKey(language)) return models.get(language); - throw new IllegalArgumentException("No SentencePiece model for language " + language + " is configured"); - } - - public String normalize(String s) { - StringBuilder b = new StringBuilder(s.length() + 1); - boolean queuedSpace = true; // Always start by one space - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - if (s.charAt(i) == ' ') { - queuedSpace = true; - } - else { - if (queuedSpace) { - b.append(SentencePieceAlgorithm.spaceSymbol); - queuedSpace = false; - } - b.append(c); - } - } - return b.toString(); - } - - public static class Builder { - - private final Map<Language, Path> models = new HashMap<>(); - private boolean collapseUnknowns = true; - private Scoring scoring = Scoring.fewestSegments; - - public Builder() { - } - - private Builder(SentencePieceConfig config) { - collapseUnknowns = config.collapseUnknowns(); - scoring = config.scoring() == SentencePieceConfig.Scoring.fewestSegments ? Scoring.fewestSegments - : Scoring.highestScore; - for (SentencePieceConfig.Model model : config.model()) { - addModel(Language.fromLanguageTag(model.language()), model.path()); - } - } - - public void addModel(Language language, Path model) { - models.put(language, model); - } - - /** - * Adds the model that will be used if the language is unknown, OR only one model is specified. - * The same as addModel(Language.UNKNOWN, model). - */ - public Builder addDefaultModel(Path model) { - addModel(Language.UNKNOWN, model); - return this; - } - public Map<Language, Path> getModels() { return models; } - - /** - * Sets whether consecutive unknown character should be collapsed into one large unknown token (default) - * or be returned as single character tokens. - */ - public Builder setCollapseUnknowns(boolean collapseUnknowns) { - this.collapseUnknowns = collapseUnknowns; - return this; - } - public boolean getCollapseUnknowns() { return collapseUnknowns; } - - /** Sets the scoring strategy to use when picking a segmentation. Default: fewestSegments. */ - public Builder setScoring(Scoring scoring) { - this.scoring = scoring; - return this; - } - public Scoring getScoring() { return scoring; } - - public SentencePieceEncoder build() { - if (models.isEmpty()) throw new IllegalStateException("At least one model must be supplied"); - return new SentencePieceEncoder(this); - } - - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java deleted file mode 100644 index 782030a8e4d..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/TokenType.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -/** - * SentencePiece token types - * - * @author bratseth - */ -enum TokenType { - - text, control, userDefined, unknown, unused - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java deleted file mode 100644 index 8e7c2db2ed3..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.sentencepiece; - -import java.util.HashMap; -import java.util.Map; - -/** - * A simple trie for sentencepiece token lookups. - * - * @author bratseth - */ -class Trie { - - final Node root = new Node(); - - void add(TokenType type, int id, String word, float score) { - Node current = root; - for (char l : word.toCharArray()) - current = current.children.computeIfAbsent(l, c -> new Node()); - current.type = type; - current.id = id; - current.score = score; - } - - static class Node { - - Integer id; - TokenType type; - Float score; - final Map<Character, Node> children = new HashMap<>(); - - boolean isToken() { return type != null; } - - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java deleted file mode 100644 index 4a8673705ec..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi -package com.yahoo.language.sentencepiece; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/linguistics/src/main/protobuf/sentencepiece_model.proto b/linguistics/src/main/protobuf/sentencepiece_model.proto deleted file mode 100644 index 39626aede53..00000000000 --- a/linguistics/src/main/protobuf/sentencepiece_model.proto +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.! - -syntax = "proto2"; - -// TODO(taku): Needs to use LITE RUNTIME in OSS release. -option optimize_for = LITE_RUNTIME; - -package sentencepiece; - -// TrainerSpec encodes a various parameters for SentencePiece training. -message TrainerSpec { - /////////////////////////////////////////////////////////////////// - // General parameters - // - // Input corpus files. - // Trainer accepts the following two formats: - // A) Monolingual: plain text, one sentence per line. - // B) Bilingual: TSV, source sentence <tab> target sentence - // When bilingual data is passed, shared vocabulary model is built. - // Note that the input file must be raw corpus, not a preprocessed corpus. - // Trainer only loads the first `input_sentence_size` sentences specified - // with this parameter. - repeated string input = 1; - - // Input corpus format: - // "text": one-sentence-per-line text format (default) - // "tsv": sentence <tab> freq - optional string input_format = 7; - - // Output model file prefix. - // <model_prefix>.model and <model_prefix>.vocab are generated. - optional string model_prefix = 2; - - // Model type. only have UNIGRAM now. - enum ModelType { - UNIGRAM = 1; // Unigram language model with dynamic algorithm - BPE = 2; // Byte Pair Encoding - WORD = 3; // Delimitered by whitespace. - CHAR = 4; // tokenizes into character sequence - } - optional ModelType model_type = 3 [default = UNIGRAM]; - - // Vocabulary size. 8k is the default size. - optional int32 vocab_size = 4 [default = 8000]; - - // List of the languages this model can accept. - // Since the model is language-agnostic, this field is used as a reference. - repeated string accept_language = 5; - - // Size of self-test samples, which are encoded in the model file. - optional int32 self_test_sample_size = 6 [default = 0]; - - /////////////////////////////////////////////////////////////////// - // Training parameters. - // - // Uses characters which cover the corpus with the ratio of `chars_coverage`. - // This parameter determines the set of basic Alphabet of sentence piece. - // 1.0 - `chars_coverage` characters are treated as UNK. - // See also required_chars field. - optional float character_coverage = 10 [default = 0.9995]; - - // Maximum size of sentences the trainer loads from `input` parameter. - // Trainer simply loads the `input` files in sequence. - // It is better to shuffle the input corpus randomly. - optional uint64 input_sentence_size = 11 [default = 0]; - optional bool shuffle_input_sentence = 19 [default = true]; - - // Maximum size of sentences to make seed sentence pieces. - // Extended suffix array is constructed to extract frequent - // sub-strings from the corpus. This uses 20N working space, - // where N is the size of corpus. - optional int32 mining_sentence_size = 12 [deprecated = true]; - - // Maximum size of sentences to train sentence pieces. - optional int32 training_sentence_size = 13 [deprecated = true]; - - // The size of seed sentencepieces. - // `seed_sentencepiece_size` must be larger than `vocab_size`. - optional int32 seed_sentencepiece_size = 14 [default = 1000000]; - - // In every EM sub-iterations, keeps top - // `shrinking_factor` * `current sentencepieces size` with respect to - // the loss of the sentence piece. This value should be smaller than 1.0. - optional float shrinking_factor = 15 [default = 0.75]; - - // The maximum sentence length in byte. The sentences with the length - // larger than `max_sentence_length` is simply ignored. - // Longer input tends to bring the following risks: - // * Overflow during EM training (unigram language model only) - // * Performance drop because of O(n log n) cost in BPE. - optional int32 max_sentence_length = 18 [default = 4192]; - - // Number of threads in the training. - optional int32 num_threads = 16 [default = 16]; - - // Number of EM sub iterations. - optional int32 num_sub_iterations = 17 [default = 2]; - - /////////////////////////////////////////////////////////////////// - // SentencePiece parameters which control the shapes of sentence piece. - // - // Maximum length of sentencepiece. - optional int32 max_sentencepiece_length = 20 [default = 16]; - - // Uses Unicode script to split sentence pieces. - // When `split_by_unicode_script` is true, we do not allow sentence piece to - // include multiple Unicode scripts, e.g. "F1" is not a valid piece. - // Exception: CJ characters (Hiragana/Katakana/Han) are all handled - // as one script type, since Japanese word can consist of multiple scripts. - // This exception is always applied regardless of the accept-language - // parameter. - optional bool split_by_unicode_script = 21 [default = true]; - - // When `split_by_number` is true, put a boundary between number and - // non-number transition. If we want to treat "F1" is one token, set this flag - // to be false. - optional bool split_by_number = 23 [default = true]; - - // Use a white space to split sentence pieces. - // When `split_by_whitespace` is false, we may have the piece containing - // a white space in the middle. e.g., "in_the". - optional bool split_by_whitespace = 22 [default = true]; - - // Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello => - // hello_. When `treat_whitespace_as_suffix` is true, - // NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end - // of sentence. - optional bool treat_whitespace_as_suffix = 24 [default = false]; - - // Allows pieces that only contain whitespaces instead of appearing only as - // prefix or suffix of other pieces. - optional bool allow_whitespace_only_pieces = 26 [default = false]; - - // Split all digits (0-9) into separate pieces. - optional bool split_digits = 25 [default = false]; - - /////////////////////////////////////////////////////////////////// - // Vocabulary management - // - // Defines control symbols used as an indicator to - // change the behavior of the decoder. <s> and </s> are pre-defined. - // We can use this field to encode various meta information, - // including language indicator in multilingual model. - // These symbols are not visible to users, but visible to - // the decoder. Note that when the input sentence contains control symbols, - // they are not treated as one token, but segmented into normal pieces. - // Control symbols must be inserted independently from the segmentation. - repeated string control_symbols = 30; - - // Defines user defined symbols. - // These symbols are added with extremely high score - // so they are always treated as one unique symbol in any context. - // Typical usage of user_defined_symbols is placeholder for named entities. - repeated string user_defined_symbols = 31; - - // Defines required characters. Each UTF8 character in this string is included - // in the character set regardless of character_coverage value. Unlike - // user_defined_symbols, these characters have scores based on the frequency - // on input sentences, and the model can form subwords using characters - // in this field. - optional string required_chars = 36; - - // Decomposes unknown pieces into UTF-8 bytes. - optional bool byte_fallback = 35 [default = false]; - - // When creating the vocabulary file, defines whether or not to additionally - // output the score for each piece. - optional bool vocabulary_output_piece_score = 32 [default = true]; - - // `vocab_size` is treated as hard limit. Crash if - // the model can not produce the vocab of size `vocab_size`, - // When `hard_vocab_limit` is false, vocab_size is treated - // as soft limit. Note that when model_type=char, - // always assumes hard_vocab_limit = false. - optional bool hard_vocab_limit = 33 [default = true]; - - // use all symbols for vocab extraction. This flag is valid - // if model type is either CHAR or WORD - optional bool use_all_vocab = 34 [default = false]; - - /////////////////////////////////////////////////////////////////// - // Reserved special meta tokens. - // * -1 is not used. - // * unk_id must not be -1. - // Id must starts with 0 and be contigous. - optional int32 unk_id = 40 [default = 0]; // <unk> - optional int32 bos_id = 41 [default = 1]; // <s> - optional int32 eos_id = 42 [default = 2]; // </s> - optional int32 pad_id = 43 [default = -1]; // <pad> (padding) - optional string unk_piece = 45 [default = "<unk>"]; - optional string bos_piece = 46 [default = "<s>"]; - optional string eos_piece = 47 [default = "</s>"]; - optional string pad_piece = 48 [default = "<pad>"]; - - // Encodes <unk> into U+2047 (DOUBLE QUESTION MARK), - // since this character can be useful both for user and - // developer. We can easily figure out that <unk> is emitted. - optional string unk_surface = 44 [default = " \xE2\x81\x87 "]; - - // Increase bit depth to allow unigram model training on large - // (>10M sentences) corpora. A Side-effect of enabling this flag - // is increased memory usage. - optional bool train_extremely_large_corpus = 49 [default = false]; - - // Customized extensions: the range of field numbers - // are open to third-party extensions. - extensions 200 to max; -} - -// NormalizerSpec encodes a various parameters for string normalizaiton -message NormalizerSpec { - // name of normalization rule. - optional string name = 1; - - // Pre-compiled normalization rule created by - // Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method. - // Usually this field is set by Builder::GetNormalizerSpec() method. - optional bytes precompiled_charsmap = 2; - - // Adds dummy whitespace at the beginning of text in order to - // treat "world" in "world" and "hello world" in the same way. - optional bool add_dummy_prefix = 3 [default = true]; - - // Removes leading, trailing, and duplicate internal whitespace. - optional bool remove_extra_whitespaces = 4 [default = true]; - - // Replaces whitespace with meta symbol. - // This field must be true to train sentence piece model. - optional bool escape_whitespaces = 5 [default = true]; - - // Custom normalization rule file in TSV format. - // https://github.com/google/sentencepiece/blob/master/doc/normalization.md - // This field is only used in SentencePieceTrainer::Train() method, which - // compiles the rule into the binary rule stored in `precompiled_charsmap`. - optional string normalization_rule_tsv = 6; - - // Customized extensions: the range of field numbers - // are open to third-party extensions. - extensions 200 to max; -} - -// Proto to store samples for self-testing. -message SelfTestData { - message Sample { - optional string input = 1; - optional string expected = 2; - } - repeated Sample samples = 1; - - // Customized extensions: the range of field numbers - // are open to third-party extensions. - extensions 200 to max; -} - -// ModelProto stores model parameters. -// SentencePieceProcessor is supposed to be self-contained. -// All settings/parameters which may change the behavior must be encoded -// in ModelProto. -message ModelProto { - message SentencePiece { - enum Type { - NORMAL = 1; // normal symbol - UNKNOWN = 2; // unknown symbol. only <unk> for now. - CONTROL = 3; // control symbols. </s>, <s>, <2ja> etc. - USER_DEFINED = 4; // user defined symbols. - // Typical usage of USER_DEFINED symbol - // is placeholder. - BYTE = 6; // byte symbols. Used when `byte_fallback` is true. - UNUSED = 5; // this piece is not used. - } - optional string piece = 1; // piece must not be empty. - optional float score = 2; - optional Type type = 3 [default = NORMAL]; - - // Customized extensions: the range of field numbers - // are open to third-party extensions. - extensions 200 to max; - } - - // Sentence pieces with scores. - repeated SentencePiece pieces = 1; - - // Spec used to generate this model file. - optional TrainerSpec trainer_spec = 2; - - // Spec for text normalization. - optional NormalizerSpec normalizer_spec = 3; - - // Stores sample input and its expected segmentation to verify the model. - optional SelfTestData self_test_data = 4; - - // Spec for text de-normalization. - optional NormalizerSpec denormalizer_spec = 5; - - // Customized extensions: the range of field numbers - // are open to third-party extensions. - extensions 200 to max; -}
\ No newline at end of file diff --git a/linguistics/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def b/linguistics/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def deleted file mode 100644 index 0b8e5103aa3..00000000000 --- a/linguistics/src/main/resources/configdefinitions/language.sentencepiece.sentence-piece.def +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# Configures com.yahoo.language.sentencepiece.SentencePieceEncoder - -namespace=language.sentencepiece - -# Whether consecutive unknown character should be collapsed into one large unknown token (default -# or be returned as single character tokens. -collapseUnknowns bool default=true - -# The scoring strategy to use when picking a segmentation. -scoring enum { highestScore, fewestSegments } default=fewestSegments - -# The language a model is for, one of the language tags in com.yahoo.language.Language. -# Use "unknown" for models to be used with any language. -model[].language string -# The path to the model relative to the application package root -model[].path path diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java b/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java deleted file mode 100644 index edbbe21ec53..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceConfigurationTest.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.language.sentencepiece; - -import com.yahoo.config.FileReference; -import com.yahoo.language.Language; -import org.junit.Test; - -/** - * @author bratseth - */ -public class SentencePieceConfigurationTest { - - @Test - public void testEnglishTokenization() { - var b = new SentencePieceConfig.Builder(); - addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - var tester = new SentencePieceTester(new SentencePieceEncoder(b.build())); - tester.assertSegmented("this is another sentence", "▁this", "▁is", "▁another", "▁sentence"); - tester.assertSegmented("KHJKJHHKJHHSH hello", "▁", "KHJKJHHKJHHSH", "▁hel", "lo"); - } - - @Test - public void testNoCollapse() { - var b = new SentencePieceConfig.Builder(); - addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - b.collapseUnknowns(false); - var tester = new SentencePieceTester(new SentencePieceEncoder(b.build())); - tester.assertSegmented("KHJ hello", "▁", "K", "H", "J", "▁hel", "lo"); - } - - @Test - public void testHighestScore() { - var b = new SentencePieceConfig.Builder(); - addModel("unknown", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - b.scoring(SentencePieceConfig.Scoring.highestScore); - var tester = new SentencePieceTester(new SentencePieceEncoder(b.build())); - tester.assertSegmented("hello", "▁h", "el", "lo"); - } - - @Test - public void testMultiLanguageTokenization() { - var b = new SentencePieceConfig.Builder(); - addModel("ja", "src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model", b); - addModel("en", "src/test/models/sentencepiece/en.wiki.bpe.vs10000.model", b); - var tester = new SentencePieceTester(new SentencePieceEncoder(b.build())); - tester.assertSegmented(Language.JAPANESE, "いくつかの通常のテキスト", "▁", "いく", "つか", "の", "通常", "の", "テ", "キ", "スト"); - tester.assertSegmented(Language.ENGLISH, "hello", "▁hel", "lo"); - tester.assertSegmented(Language.JAPANESE, "hello", "▁h", "ell", "o"); - } - - private void addModel(String language, String file, SentencePieceConfig.Builder b) { - var mb = new SentencePieceConfig.Model.Builder(); - mb.language(language); - mb.path(new FileReference(file)); - b.model(mb); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java b/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java deleted file mode 100644 index d60d7386d4b..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTest.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.language.sentencepiece; - -import com.yahoo.language.Language; -import org.junit.Test; - -import java.io.File; - -/** - * @author bratseth - */ -public class SentencePieceTest { - - @Test - public void testEnglishTokenization() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertSegmented("h", "▁h"); - tester.assertSegmented("he", "▁he"); - tester.assertSegmented("hel", "▁hel"); - tester.assertSegmented("hello", "▁hel", "lo"); - tester.assertSegmented("hei", "▁he", "i"); - tester.assertSegmented("hei you", "▁he", "i", "▁you"); - tester.assertSegmented("hei you", "▁he", "i", "▁you"); - tester.assertSegmented("this is another sentence", "▁this", "▁is", "▁another", "▁sentence"); - tester.assertSegmented("hello world!", "▁hel", "lo", "▁world", "!"); - tester.assertSegmented("Hello, world!", "▁", "H", "ello", ",", "▁world", "!"); - tester.assertSegmented("HELLO, world!", "▁", "HELLO", ",", "▁world", "!"); - tester.assertSegmented("KHJKJHHKJHHSH", "▁", "KHJKJHHKJHHSH"); - tester.assertSegmented("KHJKJHHKJHHSH hello", "▁", "KHJKJHHKJHHSH", "▁hel", "lo"); - tester.assertSegmented(" hello ", "▁hel", "lo"); - tester.assertSegmented(")(/&#()/\"\")", "▁)", "(", "/", "&", "#", "(", ")", "/", "\"", "\")"); - tester.assertSegmented(")(/&#(small)/\"in quotes\")", "▁)", "(", "/", "&", "#", "(", "sm", "all", ")", "/", "\"", "in", "▁qu", "otes", "\")"); - tester.assertSegmented("x.400AS", "▁x", ".", "4", "00", "AS"); - tester.assertSegmented("A normal sentence. Yes one more.", "▁", "A", "▁normal", "▁sentence", ".", "▁", "Y", "es", "▁one", "▁more", "."); - } - - @Test - public void testIntegerListEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEncoded("hello, world!", 908, 1418, 9934, 501, 9960); - tester.assertEncoded("Hello, world!", 9912, 0, 6595, 9934, 501, 9960); - } - - @Test - public void testDenseTensorEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEncoded("hello, world!", "tensor(d[10])", "[908,1418,9934,501,9960,0,0,0,0,0]"); - tester.assertEncoded("Hello, world!", "tensor(d[10])", "[9912,0,6595,9934,501,9960,0,0,0,0]"); - tester.assertEncoded("hello, world!", "tensor(d[2])", "[908,1418]"); - } - - @Test - public void testSparseTensorEncoding() { - var tester = new SentencePieceTester(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - tester.assertEncoded("hello", "tensor(token{})", "{lo:1.0,'▁hel':0.0}"); - } - - @Test - public void testNoCollapse() { - var tester = new SentencePieceTester(new SentencePieceEncoder.Builder() - .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) - .setCollapseUnknowns(false)); - tester.assertSegmented("KHJ hello", "▁", "K", "H", "J", "▁hel", "lo"); - } - - @Test - public void testHighestScore() { - var tester = new SentencePieceTester(new SentencePieceEncoder.Builder() - .addDefaultModel(new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()) - .setScoring(Scoring.highestScore)); - tester.assertSegmented("h", "▁h"); - tester.assertSegmented("he", "▁he"); - tester.assertSegmented("hel", "▁h", "el"); - tester.assertSegmented("hello", "▁h", "el", "lo"); - } - - @Test - public void testMultiLanguageTokenization() { - SentencePieceEncoder.Builder builder = new SentencePieceEncoder.Builder(); - builder.addModel(Language.JAPANESE, new File("src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model").toPath()); - builder.addModel(Language.ENGLISH, new File("src/test/models/sentencepiece/en.wiki.bpe.vs10000.model").toPath()); - var tester = new SentencePieceTester(builder); - tester.assertSegmented(Language.JAPANESE, "いくつかの通常のテキスト", "▁", "いく", "つか", "の", "通常", "の", "テ", "キ", "スト"); - tester.assertSegmented(Language.ENGLISH, "hello", "▁hel", "lo"); - tester.assertSegmented(Language.JAPANESE, "hello", "▁h", "ell", "o"); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java b/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java deleted file mode 100644 index 1ba7c9b472d..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/sentencepiece/SentencePieceTester.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// - -package com.yahoo.language.sentencepiece; - -import com.yahoo.language.Language; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.nio.file.Path; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -class SentencePieceTester { - - private final SentencePieceEncoder encoder; - - public SentencePieceTester(Path model) { - this(new SentencePieceEncoder.Builder().addDefaultModel(model)); - } - - public SentencePieceTester(SentencePieceEncoder.Builder builder) { - this(builder.build()); - } - - public SentencePieceTester(SentencePieceEncoder encoder) { - this.encoder = encoder; - } - - public void assertEncoded(String input, Integer... expectedCodes) { - assertArrayEquals(expectedCodes, encoder.encode(input, Language.UNKNOWN).toArray()); - } - - public void assertEncoded(String input, String tensorType, String tensor) { - TensorType type = TensorType.fromSpec(tensorType); - Tensor expected = Tensor.from(type, tensor); - assertEquals(expected, encoder.encode(input, Language.UNKNOWN, type)); - } - - public void assertSegmented(String input, String... expectedSegments) { - assertSegmented(Language.UNKNOWN, input, expectedSegments); - } - - public void assertSegmented(Language language, String input, String... expectedSegments) { - assertArrayEquals(expectedSegments, encoder.segment(input, language).toArray()); - } - -} diff --git a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model b/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model Binary files differdeleted file mode 100644 index 89f93ef3517..00000000000 --- a/linguistics/src/test/models/sentencepiece/en.wiki.bpe.vs10000.model +++ /dev/null diff --git a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model b/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model Binary files differdeleted file mode 100644 index 41c0688d9df..00000000000 --- a/linguistics/src/test/models/sentencepiece/ja.wiki.bpe.vs5000.model +++ /dev/null |