From a1e50d27853b14c6a52028e60e32b8e14e28d1ad Mon Sep 17 00:00:00 2001
From: MariusArhaug <mariusarhaug@hotmail.com>
Date: Thu, 4 Apr 2024 13:49:27 +0200
Subject: add comment for intention in determineScript function

---
 linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java | 1 +
 1 file changed, 1 insertion(+)

(limited to 'linguistics')

diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index fb876f5f066..75360e93729 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -104,6 +104,7 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     private TokenScript determineScript(TokenScript tokenScript, TokenScript characterScript) {
+        // if any character is LATIN, use that as token script; otherwise use script of first character seen.
         if (characterScript == TokenScript.LATIN) return TokenScript.LATIN;
         return tokenScript;
     }
-- 
cgit v1.2.3