diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-09-16 22:46:17 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-16 22:46:17 +0200 |
commit | 583101eec4032dda2310df146962e8471d70f188 (patch) | |
tree | e8f9f53f29d7c2bf82665d3256a7f333c696d1b8 /linguistics/src/main/java/com/yahoo/language/process | |
parent | daab62042f34575d545dcd0b6fd100e232848c85 (diff) |
Encoder interface
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Encoder.java | 39 | ||||
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/Stemmer.java | 4 |
2 files changed, 41 insertions, 2 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Encoder.java b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java new file mode 100644 index 00000000000..91de16f669b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/process/Encoder.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.language.Language; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.List; + +/** + * An encoder converts a text string to a tensor or list of tokens + * + * @author bratseth + */ +public interface Encoder { + + /** + * Encodes text into tokens in a list of ids. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @return the text encoded to a list of segment ids + * @throws IllegalArgumentException if the language is not supported by this encoder + */ + List<Integer> encode(String text, Language language); + + /** + * Encodes text into tokens in a tensor. + * The information contained in the encoding may depend on the tensor type. + * + * @param text the text to encode + * @param language the language of the text, or UNKNOWN to use language independent encoding + * @param tensorType the type of the ttensor to be returned + * @return the tex encoded into a tensor of the supplied type + * @throws IllegalArgumentException if the language or tensor type is not supported by this encoder + */ + Tensor encode(String text, Language language, TensorType tensorType); + +} diff --git a/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java b/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java index da8a73407ff..a2d0d0a84c9 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java @@ -6,9 +6,9 @@ import com.yahoo.language.Language; import java.util.List; /** - * <p>Interface providing stemming of single words.</p> + * Interface providing stemming of single words. * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public interface Stemmer { |