Prepare for upgrading antlr by using recommended approach for lowercasing.

4 files changed, 97 insertions, 104 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java
new file mode 100644
index 00000000000..1784aa77966
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java
@@ -0,0 +1,77 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.search.yql;
+
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.CharStream;
+
+/**
+ * This class supports case-insensitive lexing by wrapping an existing
+ * {@link CharStream} and forcing the lexer to see only lowercase characters.
+ * Grammar literals should then be only lower case such as 'begin'. The text of the character
+ * stream is unaffected. Example: input 'BeGiN' would match lexer rule
+ * 'begin', but getText() would return 'BeGiN'.
+ * It is based on https://github.com/antlr/antlr4/blob/master/doc/resources/CaseChangingCharStream.java
+ */
+class CaseInsensitiveCharStream implements CharStream {
+
+    final CharStream stream;
+
+    /**
+     * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing
+     * all characters lower case.
+     * @param stream The stream to wrap.
+     */
+    CaseInsensitiveCharStream(CharStream stream) {
+        this.stream = stream;
+    }
+
+    @Override
+    public String getText(Interval interval) {
+        return stream.getText(interval);
+    }
+
+    @Override
+    public void consume() {
+        stream.consume();
+    }
+
+    @Override
+    public int LA(int i) {
+        int c = stream.LA(i);
+        if (c <= 0) {
+            return c;
+        }
+        return Character.toLowerCase(c);
+    }
+
+    @Override
+    public int mark() {
+        return stream.mark();
+    }
+
+    @Override
+    public void release(int marker) {
+        stream.release(marker);
+    }
+
+    @Override
+    public int index() {
+        return stream.index();
+    }
+
+    @Override
+    public void seek(int index) {
+        stream.seek(index);
+    }
+
+    @Override
+    public int size() {
+        return stream.size();
+    }
+
+    @Override
+    public String getSourceName() {
+        return stream.getSourceName();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
deleted file mode 100644
index 0f83fd40cc1..00000000000
--- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.yql;
-
-import org.antlr.v4.runtime.ANTLRFileStream;
-import org.antlr.v4.runtime.CharStream;
-
-import java.io.IOException;
-
-/**
- * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
- */
-
-class CaseInsensitiveFileStream extends ANTLRFileStream {
-
-    public CaseInsensitiveFileStream(String fileName) throws IOException {
-        super(fileName);
-    }
-
-    @Override
-    public int LA(int i) {
-        if (i == 0) {
-            return 0;
-        }
-        if (i < 0) {
-            i++; // e.g., translate LA(-1) to use offset 0
-        }
-
-        if ((p + i - 1) >= n) {
-            return CharStream.EOF;
-        }
-        return Character.toLowerCase(data[p + i - 1]);
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
deleted file mode 100644
index 8e14b544439..00000000000
--- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.yql;
-
-import org.antlr.v4.runtime.ANTLRInputStream;
-import org.antlr.v4.runtime.CharStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
- */
-class CaseInsensitiveInputStream extends ANTLRInputStream {
-
-
-    public CaseInsensitiveInputStream(InputStream input) throws IOException {
-        super(input);
-    }
-    
-    public CaseInsensitiveInputStream(String input) {
-        super(input);
-    }
-
-    @Override
-    public int LA(int i) {
-        if (i == 0) {
-            return 0;
-        }
-        if (i < 0) {
-            i++; // e.g., translate LA(-1) to use offset 0
-        }
-
-        if ((p + i - 1) >= n) {
-            return CharStream.EOF;
-        }
-        return Character.toLowerCase(data[p + i - 1]);
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
index d32033249f1..c84ed0a0565 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
@@ -56,6 +56,7 @@ import com.yahoo.search.yql.yqlplusParser.TimeoutContext;
 import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext;
 import com.yahoo.search.yql.yqlplusParser.WhereContext;
 
+import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.BaseErrorListener;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -69,7 +70,6 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.RuleNode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Iterator;
@@ -83,50 +83,39 @@ import java.util.Set;
 final class ProgramParser {
 
     public yqlplusParser prepareParser(String programName, InputStream input) throws IOException {
-        return prepareParser(programName, new CaseInsensitiveInputStream(input));
+        //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading
+        return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input)));
     }
 
     public yqlplusParser prepareParser(String programName, String input) throws IOException {
-        return prepareParser(programName, new CaseInsensitiveInputStream(input));
+        //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading
+        return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input)));
     }
 
-    public yqlplusParser prepareParser(File file) throws IOException {
-        return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath()));
+    private static class ErrorListener extends BaseErrorListener {
+        private final String programName;
+        ErrorListener(String programName) { this.programName = programName; }
+        @Override
+        public void syntaxError(Recognizer<?, ?> recognizer,
+                                Object offendingSymbol,
+                                int line,
+                                int charPositionInLine,
+                                String msg,
+                                RecognitionException e) {
+            throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
+        }
     }
 
     private yqlplusParser prepareParser(String programName, CharStream input) {
+        ErrorListener errorListener = new ErrorListener(programName);
         yqlplusLexer lexer = new yqlplusLexer(input);
         lexer.removeErrorListeners();
-        lexer.addErrorListener(new BaseErrorListener() {
-
-          @Override
-          public void syntaxError(Recognizer<?, ?> recognizer,
-                                  Object offendingSymbol,
-                                  int line,
-                                  int charPositionInLine,
-                                  String msg,
-                                  RecognitionException e) {
-            throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
-          }
-
-        });
+        lexer.addErrorListener(errorListener);
         TokenStream tokens = new CommonTokenStream(lexer);
 
         yqlplusParser parser = new yqlplusParser(tokens);
         parser.removeErrorListeners();
-        parser.addErrorListener(new BaseErrorListener() {
-
-          @Override
-          public void syntaxError(Recognizer<?, ?> recognizer,
-                                  Object offendingSymbol,
-                                  int line,
-                                  int charPositionInLine,
-                                  String msg,
-                                  RecognitionException e) {
-            throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
-          }
-
-        });
+        parser.addErrorListener(errorListener);
         parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
         return parser;
     }