diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-07-31 14:04:02 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-07-31 14:04:02 +0200 |
commit | 00040fc85d3cb5a4752e4d916926f21195d16520 (patch) | |
tree | 265f0725d183998ced1389002d624a0ad016a04e /lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java | |
parent | 113e57691652232871a122b0c0a748a9692b60c6 (diff) |
Fallback to default implementations
Diffstat (limited to 'lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java')
-rw-r--r-- | lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java | 35 |
1 files changed, 0 insertions, 35 deletions
diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java index b5c5ba47ab6..37d2f6abdd7 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java @@ -22,11 +22,7 @@ import java.util.logging.Logger; public class LuceneLinguistics extends SimpleLinguistics { private static final Logger log = Logger.getLogger(LuceneLinguistics.class.getName()); - private final Normalizer normalizer; - private final Transformer transformer; private final Tokenizer tokenizer; - private final Stemmer stemmer; - private final Segmenter segmenter; private final LuceneAnalysisConfig config; @Inject @@ -34,42 +30,11 @@ public class LuceneLinguistics extends SimpleLinguistics { log.info("Creating LuceneLinguistics with: " + config); this.config = config; this.tokenizer = new LuceneTokenizer(config, analyzers); - // NOOP stemmer - this.stemmer = (word, stemMode, language) -> { - ArrayList<StemList> stemLists = new ArrayList<>(); - StemList stems = new StemList(); - stems.add(word); - stemLists.add(stems); - return stemLists; - }; - // Segmenter that just wraps a tokenizer - this.segmenter = (string, language) -> { - ArrayList<String> segments = new ArrayList<>(); - Iterable<Token> tokens = tokenizer.tokenize(string, language, StemMode.NONE, false); - tokens.forEach(token -> segments.add(token.getTokenString())); - return segments; - }; - // NOOP normalizer - this.normalizer = (string) -> string; - // NOOP transformer - this.transformer = (string, language) -> string; } @Override - public Stemmer getStemmer() { return stemmer; } - - @Override public Tokenizer getTokenizer() { return tokenizer; } - @Override - public Normalizer getNormalizer() { return normalizer; } - - @Override - public Transformer getTransformer() { return transformer; } - - @Override - public Segmenter getSegmenter() { return segmenter; } - public LuceneAnalysisConfig getConfig() { return config; } |