diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-11-10 21:47:17 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-11-10 21:47:17 +0100 |
commit | 0d79f0fbcdf7d4af2b698ce735fccf0ea7eadcdb (patch) | |
tree | 3918e43fc13a602f876f4fd3d57c7f22a847f85a /indexinglanguage | |
parent | 90965807bd8a6134fe92f7058b3b0a3287050c2a (diff) |
Cleanup
Diffstat (limited to 'indexinglanguage')
2 files changed, 1 insertions, 5 deletions
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index 74afd30d7ef..52cd8a8ff54 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -128,19 +128,16 @@ public class LinguisticsAnnotator { Span where = parent.span((int)token.getOffset(), token.getOrig().length()); String lowercasedOrig = toLowerCase(token.getOrig()); - String termOrIfNullOrig = lowercasedOrig; String term = token.getTokenString(); if (term != null) { addAnnotation(where, term, token.getOrig(), termOccurrences); - termOrIfNullOrig = term; if ( ! term.equals(lowercasedOrig)) addAnnotation(where, token.getOrig(), token.getOrig(), termOccurrences); } for (int i = 0; i < token.getNumStems(); i++) { String stem = token.getStem(i); - if (! (lowercasedOrig.equals(stem) || termOrIfNullOrig.equals(stem))) { + if (! (stem.equals(lowercasedOrig) || stem.equals(term))) addAnnotation(where, stem, token.getOrig(), termOccurrences); - } } } else { String term = token.getTokenString(); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index c52b877ba3b..a4dbe1fe826 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -272,7 +272,6 @@ public class LinguisticsAnnotatorTestCase { private Token replace(Token token, Map<String, String> replacementTerms) { var simpleToken = (SimpleToken)token; - System.out.println("Token: " + token + ", getTokenString: " + token.getTokenString()); simpleToken.setTokenString(replacementTerms.getOrDefault(token.getTokenString(), token.getTokenString())); return simpleToken; } |