From cc60531ac22a7e9601055174a02a6e67c428f800 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 22 May 2023 23:08:48 +0200 Subject: Always treat each symbol as a separate token --- .../src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'container-search/src/main/java/com/yahoo/prelude/query') diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index c1d415b8e27..01bb606e9ee 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -107,7 +107,9 @@ public final class Tokenizer { if (i >= source.length()) break; int c = source.codePointAt(i); - if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { + if (characterClasses.isSymbol(c)) { // treat each symbol is a separate word + addToken(WORD, Character.toString(c), i, i + 1); + } else if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { i = consumeWordOrNumber(i, currentIndex); } else if (Character.isWhitespace(c)) { addToken(SPACE, " ", i, i + 1); -- cgit v1.2.3