Non-functional changes only

3 files changed, 25 insertions, 20 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index 61f09e2f7b7..5e243e52057 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -108,8 +108,7 @@ public final class Tokenizer {
             if (i >= source.length()) break;
 
             int c = source.codePointAt(i);
-            if (characterClasses.isLetterOrDigit(c)
-                    || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) {
+            if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) {
                 i = consumeWordOrNumber(i, currentIndex);
             } else if (Character.isWhitespace(c)) {
                 addToken(SPACE, " ", i, i + 1);
@@ -187,7 +186,6 @@ public final class Tokenizer {
         return true;
     }
 
-    @SuppressWarnings({"deprecation"})
     private Index determineCurrentIndex(Index defaultIndex, IndexFacts.Session indexFacts) {
         int backtrack = tokens.size();
         int tokencnt = 0;
@@ -328,7 +326,6 @@ public final class Tokenizer {
                     wantEndQuote = true;
                     actualStart = curPos+1;
                 } else if (wantEndQuote && looksLikeExactEnd(curPos+1)) {
-                    // System.err.println("seen quoted token from "+actualStart+" to "+curPos);
                     seenSome = true;
                     wantEndQuote = false;
                     isQuoted = true;
@@ -435,7 +432,7 @@ public final class Tokenizer {
         if (suffStar) {
             addToken(STAR, "*", starPos, starPos + 1);
         }
-        tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // XXX: Unsafe?
+        tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source)));
 
         // skip terminating quote
         if (isQuoted) {
@@ -451,17 +448,17 @@ public final class Tokenizer {
                 break;
             end++;
         }
-        tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); // XXX: Unsafe start?
-        if (end>=source.length())
+        tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source)));
+        if (end >= source.length())
             return end;
         else
-            return end+terminator.length(); // Don't create a token for the terminator
+            return end + terminator.length(); // Don't create a token for the terminator
     }
 
     private boolean terminatorStartsAt(int start,String terminator) {
-        int terminatorPosition=0;
-        while ((terminatorPosition+start)<source.length()) {
-            if (source.charAt(start+terminatorPosition)!=terminator.charAt(terminatorPosition))
+        int terminatorPosition = 0;
+        while ((terminatorPosition + start) < source.length()) {
+            if (source.charAt(start+terminatorPosition) != terminator.charAt(terminatorPosition))
                 return false;
             terminatorPosition++;
             if (terminatorPosition >= terminator.length())
@@ -481,8 +478,8 @@ public final class Tokenizer {
 
         while (tokenEnd < source.length()) {
             if (substringSpecialTokens) {
-                substringSpecialToken=getSpecialToken(tokenEnd);
-                if (substringSpecialToken!=null) break;
+                substringSpecialToken = getSpecialToken(tokenEnd);
+                if (substringSpecialToken != null) break;
             }
 
             int c = source.codePointAt(tokenEnd);
@@ -506,7 +503,7 @@ public final class Tokenizer {
                 // underscoresOnly = false;
                 quotesOnly = false;
             } else if (c == '\'') {
-                if (!acceptApostropheAsWordCharacter(currentIndex)) {
+                if ( ! acceptApostropheAsWordCharacter(currentIndex)) {
                     break;
                 }
                 // Otherwise consume apostrophes...
@@ -530,15 +527,15 @@ public final class Tokenizer {
             }
         }
 
-        if (substringSpecialToken==null)
+        if (substringSpecialToken == null)
             return --tokenEnd;
         // TODO: test the logic around tokenEnd with friends
-        addToken(substringSpecialToken.toToken(tokenEnd,source));
-        return --tokenEnd+substringSpecialToken.token().length();
+        addToken(substringSpecialToken.toToken(tokenEnd, source));
+        return --tokenEnd + substringSpecialToken.token().length();
     }
 
     private void addToken(Token.Kind kind, String word, int start, int end) {
-        addToken(new Token(kind, word, false, new Substring(start, end, source))); // XXX: Unsafe?
+        addToken(new Token(kind, word, false, new Substring(start, end, source)));
     }
 
     private void addToken(Token token) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 12d993e8d41..aa2e9dbcf75 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -46,7 +46,7 @@ public class TokenizerTestCase {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
         tokenizer.setSpecialTokens(createSpecialTokens());
-        List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1");
+        List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a");
 
         assertEquals(new Token(WORD, "drive"), tokens.get(0));
         assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -69,6 +69,11 @@ public class TokenizerTestCase {
         assertEquals(new Token(WORD, "ugcapi"), tokens.get(18));
         assertEquals(new Token(UNDERSCORE, "_"), tokens.get(19));
         assertEquals(new Token(NUMBER, "1"), tokens.get(20));
+        assertEquals(new Token(SPACE, " "), tokens.get(21));
+        assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(22));
+        assertEquals(new Token(SPACE, " "), tokens.get(23));
+        assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(24));
+        assertEquals(new Token(WORD, "a"), tokens.get(25));
     }
 
     @Test
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index 4e591cdfbd4..2e08814bf43 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -82,6 +82,7 @@ public class DocumentSelectorTestCase {
         assertParse("id.hash() > 0");
         assertParse("id.namespace.hash() > 0");
         assertParse("music.artist = \"*\"");
+        assertParse("music.artist = \"&\"");
         assertParse("music.artist.lowercase() = \"*\"");
         assertParse("music_.artist = \"*\"");
         assertParse("music_foo.artist = \"*\"");
@@ -339,6 +340,7 @@ public class DocumentSelectorTestCase {
         documents.add(createDocument("id:myspace:test:n=2345:mail2", 15, 1.0f, "bar", "baz"));
         documents.add(createDocument("id:myspace:test:g=mygroup:qux", 15, 1.0f, "quux", "corge"));
         documents.add(createDocument("id:myspace:test::missingint", null, 2.0f, null, "bar"));
+        documents.add(createDocument("id:myspace:test::ampersand", null, 2.0f, null, "&"));
 
         // Add some array/struct info to doc 1
         Struct sval = new Struct(documents.get(1).getDocument().getField("mystruct").getDataType());
@@ -483,6 +485,8 @@ public class DocumentSelectorTestCase {
         assertEquals(Result.FALSE, evaluate("test.hstring == test.content", documents.get(0)));
         assertEquals(Result.TRUE, evaluate("test.hstring == test.content", documents.get(2)));
         assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"&\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"&\"", documents.get(7)));
         // Case where field is not present (i.e. null) is defined for (in)equality comparisons, but
         // not for other relations.
         DocumentPut doc1234 = documents.get(6);
@@ -874,7 +878,6 @@ public class DocumentSelectorTestCase {
         } catch (ParseException e) {
             fail("The expression '" + expressionString + "' should assertEquals ok.");
         } catch (RuntimeException e) {
-            System.err.println("Error was : " + e);
             assertTrue(e.getMessage().length() >= expectedError.length());
             assertEquals(expectedError, e.getMessage().substring(0, expectedError.length()));
         }