add comment for intention in determineScript function

1 files changed, 1 insertions, 0 deletions

diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index fb876f5f066..75360e93729 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -104,6 +104,7 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     private TokenScript determineScript(TokenScript tokenScript, TokenScript characterScript) {
+        // if any character is LATIN, use that as token script; otherwise use script of first character seen.
         if (characterScript == TokenScript.LATIN) return TokenScript.LATIN;
         return tokenScript;
     }