diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/Encoder.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Encoder.java | 39 |
1 files changed, 39 insertions, 0 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Encoder.java b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java new file mode 100644 index 00000000000..91de16f669b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.language.Language; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.List; + +/** + * An encoder converts a text string to a tensor or list of tokens + * + * @author bratseth + */ +public interface Encoder { + + /** + * Encodes text into tokens in a list of ids. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @return the text encoded to a list of segment ids + * @throws IllegalArgumentException if the language is not supported by this encoder + */ + List<Integer> encode(String text, Language language); + + /** + * Encodes text into tokens in a tensor. + * The information contained in the encoding may depend on the tensor type. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @param tensorType the type of the ttensor to be returned + * @return the tex encoded into a tensor of the supplied type + * @throws IllegalArgumentException if the language or tensor type is not supported by this encoder + */ + Tensor encode(String text, Language language, TensorType tensorType); + +} |