summaryrefslogtreecommitdiffstats
path: root/indexinglanguage
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@vespa.ai>2023-11-10 21:47:17 +0100
committerJon Bratseth <bratseth@vespa.ai>2023-11-10 21:47:17 +0100
commit0d79f0fbcdf7d4af2b698ce735fccf0ea7eadcdb (patch)
tree3918e43fc13a602f876f4fd3d57c7f22a847f85a /indexinglanguage
parent90965807bd8a6134fe92f7058b3b0a3287050c2a (diff)
Cleanup
Diffstat (limited to 'indexinglanguage')
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java5
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java1
2 files changed, 1 insertions, 5 deletions
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
index 74afd30d7ef..52cd8a8ff54 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
@@ -128,19 +128,16 @@ public class LinguisticsAnnotator {
Span where = parent.span((int)token.getOffset(), token.getOrig().length());
String lowercasedOrig = toLowerCase(token.getOrig());
- String termOrIfNullOrig = lowercasedOrig;
String term = token.getTokenString();
if (term != null) {
addAnnotation(where, term, token.getOrig(), termOccurrences);
- termOrIfNullOrig = term;
if ( ! term.equals(lowercasedOrig))
addAnnotation(where, token.getOrig(), token.getOrig(), termOccurrences);
}
for (int i = 0; i < token.getNumStems(); i++) {
String stem = token.getStem(i);
- if (! (lowercasedOrig.equals(stem) || termOrIfNullOrig.equals(stem))) {
+ if (! (stem.equals(lowercasedOrig) || stem.equals(term)))
addAnnotation(where, stem, token.getOrig(), termOccurrences);
- }
}
} else {
String term = token.getTokenString();
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
index c52b877ba3b..a4dbe1fe826 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
@@ -272,7 +272,6 @@ public class LinguisticsAnnotatorTestCase {
private Token replace(Token token, Map<String, String> replacementTerms) {
var simpleToken = (SimpleToken)token;
- System.out.println("Token: " + token + ", getTokenString: " + token.getTokenString());
simpleToken.setTokenString(replacementTerms.getOrDefault(token.getTokenString(), token.getTokenString()));
return simpleToken;
}