summaryrefslogtreecommitdiffstats
path: root/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java13
1 files changed, 8 insertions, 5 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index aa24e359b53..7df432f496d 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -67,18 +67,21 @@ public class SimpleTokenizer implements Tokenizer {
}
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) {
- log.log(Level.FINEST, "processToken '"+token+"'");
+ final String original = token;
+ log.log(Level.FINEST, () -> "processToken '"+original+"'");
token = normalizer.normalize(token);
token = LinguisticsCase.toLowerCase(token);
if (removeAccents)
token = transformer.accentDrop(token, language);
if (stemMode != StemMode.NONE) {
- String oldToken = token;
+ final String oldToken = token;
token = stemmer.stem(token);
- log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'");
+ final String newToken = token;
+ log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'");
}
- log.log(Level.FINEST, () -> "processed token is: "+token);
- return token;
+ final String result = token;
+ log.log(Level.FINEST, () -> "processed token is: "+result);
+ return result;
}
}