Always treat each symbol as a separate token

4 files changed, 56 insertions, 20 deletions

diff --git a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java
index 5946a00b8bf..f6177262bf9 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java
@@ -15,7 +15,6 @@ public class CharacterClasses {
     public boolean isLetter(int c) {
         if (Character.isLetter(c)) return true;
         if (Character.isDigit(c) &&  ! isLatin(c)) return true; // Not considering these digits, so treat them as letters
-        if (Character.getType(c) == Character.OTHER_SYMBOL) return true; // emojis searchable
 
         // Some CJK punctuation defined as word characters
         if (c == '\u3008' || c == '\u3009' || c == '\u300a' || c == '\u300b' ||
@@ -30,6 +29,13 @@ public class CharacterClasses {
     }
 
     /**
+     * Returns true if the character is in the class "other symbol" - emojis etc.
+     */
+    public boolean isSymbol(int c) {
+        return Character.getType(c) == Character.OTHER_SYMBOL;
+    }
+
+    /**
      * Returns true for code points which should be considered digits - same as java.lang.Character.isDigit
      */
     public boolean isDigit(int c) {
diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java
index 83110c0021e..210d7ac94ff 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java
@@ -88,46 +88,54 @@ public class GramSplitter {
         }
 
         private Gram findNext() {
-            // Skip to next word character
-            while (i < input.length() && !characterClasses.isLetterOrDigit(input.codePointAt(i))) {
+            // Skip to next indexable character
+            while (i < input.length() && !isIndexable(input.codePointAt(i))) {
                 i = input.next(i);
                 isFirstAfterSeparator = true;
             }
-            if (i >= input.length()) return null;
-
-            UnicodeString gram = input.substring(i, n);
-            int nonWordChar = indexOfNonWordCodepoint(gram);
-            if (nonWordChar == 0) throw new RuntimeException("Programming error");
-
-            if (nonWordChar > 0)
-                gram = new UnicodeString(gram.toString().substring(0, nonWordChar));
+            if (i >= input.length()) return null; // no indexable characters
 
+            int tokenStart = i;
+            UnicodeString gram = input.substring(tokenStart, n);
+            int tokenEnd = tokenEnd(gram);
+            gram = new UnicodeString(gram.toString().substring(0, tokenEnd));
             if (gram.codePointCount() == n) { // normal case: got a full length gram
                 Gram g = new Gram(i, gram.codePointCount());
                 i = input.next(i);
                 isFirstAfterSeparator = false;
                 return g;
             }
-            else { // gram is too short due either to a non-word separator or end of string
-                if (isFirstAfterSeparator) { // make a gram anyway
+            else { // gram is too short due either to being a symbol, being followed by a non-word separator, or end of string
+                if (isFirstAfterSeparator || ( gram.codePointCount() == 1 && characterClasses.isSymbol(gram.codePointAt(0)))) { // make a gram anyway
                     Gram g = new Gram(i, gram.codePointCount());
                     i = input.next(i);
                     isFirstAfterSeparator = false;
                     return g;
                 } else { // skip to next
-                    i = input.skip(gram.codePointCount() + 1, i);
+                    i = input.skip(gram.codePointCount(), i);
                     isFirstAfterSeparator = true;
                     return findNext();
                 }
             }
         }
 
-        private int indexOfNonWordCodepoint(UnicodeString s) {
-            for (int i = 0; i < s.length(); i = s.next(i)) {
+        private boolean isIndexable(int codepoint) {
+            if (characterClasses.isLetterOrDigit(codepoint)) return true;
+            if (characterClasses.isSymbol(codepoint)) return true;
+            return false;
+        }
+
+        /** Given a string s starting by an indexable character, return the position where that token should end. */
+        private int tokenEnd(UnicodeString s) {
+            if (characterClasses.isSymbol(s.codePointAt(0)))
+                return s.next(0); // symbols have length 1
+
+            int i = 0;
+            for (; i < s.length(); i = s.next(i)) {
                 if ( ! characterClasses.isLetterOrDigit(s.codePointAt(i)))
                     return i;
             }
-            return -1;
+            return i;
         }
 
         @Override
diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
index 6cefcfbf67a..a219efce3cd 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java
@@ -49,6 +49,17 @@ public class GramSplitterTestCase {
     }
 
     @Test
+    public void testEmojis() {
+        String emoji1 = "\uD83D\uDD2A"; // 🔪
+        String emoji2 = "\uD83D\uDE00"; // 😀
+        assertGramSplit(emoji1, 2, "[" + emoji1+ "]");
+        assertGramSplit(emoji1 + emoji2, 2, "[" + emoji1 + ", " + emoji2 + "]");
+        assertGramSplit(emoji1 + "." + emoji2, 2, "[" + emoji1 + ", " + emoji2 + "]");
+        assertGramSplit("." + emoji1 + "." + emoji2 + ".", 2, "[" + emoji1 + ", " + emoji2 + "]");
+        assertGramSplit("foo" + emoji1 + "bar" + emoji2 + "baz", 2, "[fo, oo, " + emoji1 + ", ba, ar, " + emoji2 + ", ba, az]");
+    }
+
+    @Test
     public void testSpaceCornerCases() {
         // space corner cases
         assertGramSplit("e   en   e", 1, "[e, e, n, e]");
diff --git a/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java b/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java
index 1c2f7377bde..b4f080405bd 100644
--- a/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java
@@ -1,10 +1,18 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.language.simple;
 
+import com.yahoo.language.Language;
 import com.yahoo.language.process.AbstractTokenizerTestCase;
 import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
 import org.junit.Test;
 
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 /**
  * @author Steinar Knutsen
  * @author bratseth
@@ -36,9 +44,12 @@ public class SimpleTokenizerTestCase extends AbstractTokenizerTestCase {
     @Test
     public void testTokenizeEmojis() {
         TokenizerTester tester = new TokenizerTester().setStemMode(StemMode.ALL);
-        String emoji = "\uD83D\uDD2A"; // 🔪
-        tester.assertTokens(emoji, emoji);
-        tester.assertTokens(emoji + "foo", emoji, "foo");
+
+        String emoji1 = "\uD83D\uDD2A"; // 🔪
+        String emoji2 = "\uD83D\uDE00"; // 😀
+        tester.assertTokens(emoji1, emoji1);
+        tester.assertTokens(emoji1 + "foo", emoji1, "foo");
+        tester.assertTokens(emoji1 + emoji2, emoji1, emoji2);
     }
 
 }