diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-09-16 22:46:17 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-16 22:46:17 +0200 |
commit | 583101eec4032dda2310df146962e8471d70f188 (patch) | |
tree | e8f9f53f29d7c2bf82665d3256a7f333c696d1b8 /linguistics/src/main/java/com/yahoo/language/process/Encoder.java | |
parent | daab62042f34575d545dcd0b6fd100e232848c85 (diff) |
Encoder interface
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Encoder.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Encoder.java | 39 |
1 files changed, 39 insertions, 0 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Encoder.java b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java new file mode 100644 index 00000000000..91de16f669b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.language.Language; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.List; + +/** + * An encoder converts a text string to a tensor or list of tokens + * + * @author bratseth + */ +public interface Encoder { + + /** + * Encodes text into tokens in a list of ids. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @return the text encoded to a list of segment ids + * @throws IllegalArgumentException if the language is not supported by this encoder + */ + List<Integer> encode(String text, Language language); + + /** + * Encodes text into tokens in a tensor. + * The information contained in the encoding may depend on the tensor type. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @param tensorType the type of the ttensor to be returned + * @return the tex encoded into a tensor of the supplied type + * @throws IllegalArgumentException if the language or tensor type is not supported by this encoder + */ + Tensor encode(String text, Language language, TensorType tensorType); + +} |