aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@vespa.ai>2024-05-14 20:29:05 -0600
committerJon Bratseth <bratseth@vespa.ai>2024-05-14 20:29:05 -0600
commitfae1697e7cddae0c4232862f18ad0f6f12201852 (patch)
tree5df21cf840be09cadf9d21544d2e605231239340
parent63c765e1e33e02cd28f15f1a7bfad01f5f63fd43 (diff)
Improve javadoc
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java2
-rw-r--r--linguistics/src/main/java/com/yahoo/language/process/Segmenter.java19
2 files changed, 11 insertions, 10 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
index 7d258f71ebd..5e9c3dcc6ea 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
@@ -335,7 +335,7 @@ public abstract class AbstractParser implements CustomParser {
/**
* Segments a token
*
- * @param indexName the index name which preceeded this token, or null if none
+ * @param indexName the index name which preceded this token, or null if none
* @param token the token to segment
* @param quoted whether this segment is within quoted text
* @return the resulting item
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java b/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java
index 7e7ee44bf74..5240737ae45 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java
@@ -6,22 +6,23 @@ import com.yahoo.language.Language;
import java.util.List;
/**
- * Interface providing segmentation, i.e. splitting of CJK character blocks into separate tokens. This is primarily a
- * convenience feature for users who don't need full tokenization (or who use a separate tokenizer and only need CJK
- * processing).
+ * A segmnenter splits a string into separate segments (such as words) without applying any further
+ * processing (such as stemming) on each segment.
+ *
+ * This is useful when token processing should be done separately from segmentation, such as in
+ * linguistic processing of queries, where token processing depends on field settings in a specific
+ * schema, while segmentation only depends on language and happens before schema-specific processing.
*
* @author Mathias Mølster Lidal
*/
public interface Segmenter {
/**
- * Split input-string into tokens, and returned a list of tokens in unprocessed form (i.e. lowercased, normalized
- * and stemmed if applicable, see @link{StemMode} for list of stemming options). It is assumed that the input only
- * contains word-characters, any punctuation and spacing tokens will be removed.
+ * Returns a list of segments produced from a string.
*
- * @param input the text to segment.
- * @param language language of input text.
- * @return the list of segments.
+ * @param input the text to segment
+ * @param language the language of the input text
+ * @return the resulting list of segments
* @throws ProcessingException if an exception is encountered during processing
*/
List<String> segment(String input, Language language);