diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java')
| -rw-r--r-- | linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java index 5c321e4da9b..8a88ae8f005 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java @@ -31,8 +31,9 @@ public class SimpleTokenType { case Character.MATH_SYMBOL: case Character.CURRENCY_SYMBOL: case Character.MODIFIER_SYMBOL: - case Character.OTHER_SYMBOL: return TokenType.SYMBOL; + case Character.OTHER_SYMBOL: + return TokenType.INDEXABLE_SYMBOL; case Character.OTHER_NUMBER: // "SUPERSCRIPT TWO", |