From cc60531ac22a7e9601055174a02a6e67c428f800 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 22 May 2023 23:08:48 +0200 Subject: Always treat each symbol as a separate token --- .../yahoo/language/process/CharacterClasses.java | 8 ++++- .../com/yahoo/language/process/GramSplitter.java | 40 +++++++++++++--------- .../language/process/GramSplitterTestCase.java | 11 ++++++ .../language/simple/SimpleTokenizerTestCase.java | 17 +++++++-- 4 files changed, 56 insertions(+), 20 deletions(-) (limited to 'linguistics') diff --git a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java index 5946a00b8bf..f6177262bf9 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java +++ b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java @@ -15,7 +15,6 @@ public class CharacterClasses { public boolean isLetter(int c) { if (Character.isLetter(c)) return true; if (Character.isDigit(c) && ! isLatin(c)) return true; // Not considering these digits, so treat them as letters - if (Character.getType(c) == Character.OTHER_SYMBOL) return true; // emojis searchable // Some CJK punctuation defined as word characters if (c == '\u3008' || c == '\u3009' || c == '\u300a' || c == '\u300b' || @@ -29,6 +28,13 @@ public class CharacterClasses { type == java.lang.Character.ENCLOSING_MARK; } + /** + * Returns true if the character is in the class "other symbol" - emojis etc. + */ + public boolean isSymbol(int c) { + return Character.getType(c) == Character.OTHER_SYMBOL; + } + /** * Returns true for code points which should be considered digits - same as java.lang.Character.isDigit */ diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index 83110c0021e..210d7ac94ff 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -88,46 +88,54 @@ public class GramSplitter { } private Gram findNext() { - // Skip to next word character - while (i < input.length() && !characterClasses.isLetterOrDigit(input.codePointAt(i))) { + // Skip to next indexable character + while (i < input.length() && !isIndexable(input.codePointAt(i))) { i = input.next(i); isFirstAfterSeparator = true; } - if (i >= input.length()) return null; - - UnicodeString gram = input.substring(i, n); - int nonWordChar = indexOfNonWordCodepoint(gram); - if (nonWordChar == 0) throw new RuntimeException("Programming error"); - - if (nonWordChar > 0) - gram = new UnicodeString(gram.toString().substring(0, nonWordChar)); + if (i >= input.length()) return null; // no indexable characters + int tokenStart = i; + UnicodeString gram = input.substring(tokenStart, n); + int tokenEnd = tokenEnd(gram); + gram = new UnicodeString(gram.toString().substring(0, tokenEnd)); if (gram.codePointCount() == n) { // normal case: got a full length gram Gram g = new Gram(i, gram.codePointCount()); i = input.next(i); isFirstAfterSeparator = false; return g; } - else { // gram is too short due either to a non-word separator or end of string - if (isFirstAfterSeparator) { // make a gram anyway + else { // gram is too short due either to being a symbol, being followed by a non-word separator, or end of string + if (isFirstAfterSeparator || ( gram.codePointCount() == 1 && characterClasses.isSymbol(gram.codePointAt(0)))) { // make a gram anyway Gram g = new Gram(i, gram.codePointCount()); i = input.next(i); isFirstAfterSeparator = false; return g; } else { // skip to next - i = input.skip(gram.codePointCount() + 1, i); + i = input.skip(gram.codePointCount(), i); isFirstAfterSeparator = true; return findNext(); } } } - private int indexOfNonWordCodepoint(UnicodeString s) { - for (int i = 0; i < s.length(); i = s.next(i)) { + private boolean isIndexable(int codepoint) { + if (characterClasses.isLetterOrDigit(codepoint)) return true; + if (characterClasses.isSymbol(codepoint)) return true; + return false; + } + + /** Given a string s starting by an indexable character, return the position where that token should end. */ + private int tokenEnd(UnicodeString s) { + if (characterClasses.isSymbol(s.codePointAt(0))) + return s.next(0); // symbols have length 1 + + int i = 0; + for (; i < s.length(); i = s.next(i)) { if ( ! characterClasses.isLetterOrDigit(s.codePointAt(i))) return i; } - return -1; + return i; } @Override diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index 6cefcfbf67a..a219efce3cd 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -48,6 +48,17 @@ public class GramSplitterTestCase { assertGramSplit("en", 3, "[en]"); } + @Test + public void testEmojis() { + String emoji1 = "\uD83D\uDD2A"; // 🔪 + String emoji2 = "\uD83D\uDE00"; // 😀 + assertGramSplit(emoji1, 2, "[" + emoji1+ "]"); + assertGramSplit(emoji1 + emoji2, 2, "[" + emoji1 + ", " + emoji2 + "]"); + assertGramSplit(emoji1 + "." + emoji2, 2, "[" + emoji1 + ", " + emoji2 + "]"); + assertGramSplit("." + emoji1 + "." + emoji2 + ".", 2, "[" + emoji1 + ", " + emoji2 + "]"); + assertGramSplit("foo" + emoji1 + "bar" + emoji2 + "baz", 2, "[fo, oo, " + emoji1 + ", ba, ar, " + emoji2 + ", ba, az]"); + } + @Test public void testSpaceCornerCases() { // space corner cases diff --git a/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java b/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java index 1c2f7377bde..b4f080405bd 100644 --- a/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenizerTestCase.java @@ -1,10 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; +import com.yahoo.language.Language; import com.yahoo.language.process.AbstractTokenizerTestCase; import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; import org.junit.Test; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author Steinar Knutsen * @author bratseth @@ -36,9 +44,12 @@ public class SimpleTokenizerTestCase extends AbstractTokenizerTestCase { @Test public void testTokenizeEmojis() { TokenizerTester tester = new TokenizerTester().setStemMode(StemMode.ALL); - String emoji = "\uD83D\uDD2A"; // 🔪 - tester.assertTokens(emoji, emoji); - tester.assertTokens(emoji + "foo", emoji, "foo"); + + String emoji1 = "\uD83D\uDD2A"; // 🔪 + String emoji2 = "\uD83D\uDE00"; // 😀 + tester.assertTokens(emoji1, emoji1); + tester.assertTokens(emoji1 + "foo", emoji1, "foo"); + tester.assertTokens(emoji1 + emoji2, emoji1, emoji2); } } -- cgit v1.2.3