diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
4 files changed, 97 insertions, 104 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java new file mode 100644 index 00000000000..1784aa77966 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.yql; + +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.CharStream; + +/** + * This class supports case-insensitive lexing by wrapping an existing + * {@link CharStream} and forcing the lexer to see only lowercase characters. + * Grammar literals should then be only lower case such as 'begin'. The text of the character + * stream is unaffected. Example: input 'BeGiN' would match lexer rule + * 'begin', but getText() would return 'BeGiN'. + * It is based on https://github.com/antlr/antlr4/blob/master/doc/resources/CaseChangingCharStream.java + */ +class CaseInsensitiveCharStream implements CharStream { + + final CharStream stream; + + /** + * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing + * all characters lower case. + * @param stream The stream to wrap. + */ + CaseInsensitiveCharStream(CharStream stream) { + this.stream = stream; + } + + @Override + public String getText(Interval interval) { + return stream.getText(interval); + } + + @Override + public void consume() { + stream.consume(); + } + + @Override + public int LA(int i) { + int c = stream.LA(i); + if (c <= 0) { + return c; + } + return Character.toLowerCase(c); + } + + @Override + public int mark() { + return stream.mark(); + } + + @Override + public void release(int marker) { + stream.release(marker); + } + + @Override + public int index() { + return stream.index(); + } + + @Override + public void seek(int index) { + stream.seek(index); + } + + @Override + public int size() { + return stream.size(); + } + + @Override + public String getSourceName() { + return stream.getSourceName(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java deleted file mode 100644 index 0f83fd40cc1..00000000000 --- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.yql; - -import org.antlr.v4.runtime.ANTLRFileStream; -import org.antlr.v4.runtime.CharStream; - -import java.io.IOException; - -/** - * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token. - */ - -class CaseInsensitiveFileStream extends ANTLRFileStream { - - public CaseInsensitiveFileStream(String fileName) throws IOException { - super(fileName); - } - - @Override - public int LA(int i) { - if (i == 0) { - return 0; - } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java deleted file mode 100644 index 8e14b544439..00000000000 --- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.yql; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token. - */ -class CaseInsensitiveInputStream extends ANTLRInputStream { - - - public CaseInsensitiveInputStream(InputStream input) throws IOException { - super(input); - } - - public CaseInsensitiveInputStream(String input) { - super(input); - } - - @Override - public int LA(int i) { - if (i == 0) { - return 0; - } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index d32033249f1..c84ed0a0565 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -56,6 +56,7 @@ import com.yahoo.search.yql.yqlplusParser.TimeoutContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; +import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; @@ -69,7 +70,6 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.TerminalNode; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; @@ -83,50 +83,39 @@ import java.util.Set; final class ProgramParser { public yqlplusParser prepareParser(String programName, InputStream input) throws IOException { - return prepareParser(programName, new CaseInsensitiveInputStream(input)); + //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading + return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input))); } public yqlplusParser prepareParser(String programName, String input) throws IOException { - return prepareParser(programName, new CaseInsensitiveInputStream(input)); + //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading + return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input))); } - public yqlplusParser prepareParser(File file) throws IOException { - return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath())); + private static class ErrorListener extends BaseErrorListener { + private final String programName; + ErrorListener(String programName) { this.programName = programName; } + @Override + public void syntaxError(Recognizer<?, ?> recognizer, + Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); + } } private yqlplusParser prepareParser(String programName, CharStream input) { + ErrorListener errorListener = new ErrorListener(programName); yqlplusLexer lexer = new yqlplusLexer(input); lexer.removeErrorListeners(); - lexer.addErrorListener(new BaseErrorListener() { - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); - } - - }); + lexer.addErrorListener(errorListener); TokenStream tokens = new CommonTokenStream(lexer); yqlplusParser parser = new yqlplusParser(tokens); parser.removeErrorListeners(); - parser.addErrorListener(new BaseErrorListener() { - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); - } - - }); + parser.addErrorListener(errorListener); parser.getInterpreter().setPredictionMode(PredictionMode.SLL); return parser; } |