Slight algorithm simplification

1 files changed, 10 insertions, 11 deletions

diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java
index 9509c1d070d..24b2a03bf3c 100644
--- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java
+++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEncoder.java
@@ -96,23 +96,22 @@ public class SentencePieceEncoder implements Segmenter {
         while (start < input.length()) { // segment from this position to the end of the text
             Trie.Node node = model.tokens.root;
             int characterPosition = start;
-            boolean addedSingleCharacterSegment = false;
-            while (characterPosition < input.length()) { // traverse the trie one character at the time from this position
+            while (node != null && characterPosition < input.length()) { // traverse the trie one character at the time from this position
                 node = node.children.get(input.charAt(characterPosition));
                 characterPosition++;
-                if (node == null) break;
                 int length = characterPosition - start;
-                if (node.isToken()) {
-                    if (node.type == TokenType.unused) continue;
+                if (node != null) {
+                    if (node.isToken()) {
+                        if (node.type == TokenType.unused) continue;
 
-                    float score = node.type == TokenType.userDefined ? (length * model.maxScore - 0.1f) : node.score;
-                    addSegment(TokenType.text, node.id, start, characterPosition, score, segmentEnds);
+                        float score = node.type == TokenType.userDefined ? (length * model.maxScore - 0.1f) : node.score;
+                        addSegment(TokenType.text, node.id, start, characterPosition, score, segmentEnds);
+                    }
+                }
+                else if (length == 1) { // add an 'unknown' token of length 1 instead to make the next position reachable
+                    addSegment(TokenType.unknown, 0, start, start + 1, unknownScore, segmentEnds);
                 }
-                if (! addedSingleCharacterSegment && length == 1)
-                    addedSingleCharacterSegment = true;
             }
-            if ( ! addedSingleCharacterSegment) // add an unknown 1 character token to be able to start from the next character
-                addSegment(TokenType.unknown, 0, start, start + 1, unknownScore, segmentEnds);
             start++;
         }