Publish

2 files changed, 293 insertions, 0 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java b/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java
new file mode 100644
index 00000000000..a73fffc6c5c
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.javacc;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class FastCharStreamTestCase {
+
+    @Test
+    public void requireThatInputCanBeRead() throws IOException {
+        FastCharStream input = new FastCharStream("foo");
+        assertEquals('f', input.readChar());
+        assertEquals('o', input.readChar());
+        assertEquals('o', input.readChar());
+        try {
+            input.readChar();
+            fail();
+        } catch (IOException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatColumnIsTracked() throws IOException {
+        FastCharStream input = new FastCharStream("foo");
+        assertEquals(1, input.getColumn());
+        input.readChar();
+        assertEquals(2, input.getColumn());
+        input.readChar();
+        assertEquals(3, input.getColumn());
+        input.readChar();
+        assertEquals(4, input.getColumn());
+    }
+
+    @Test
+    public void requireThatLineIsNotTracked() throws IOException {
+        FastCharStream input = new FastCharStream("f\no");
+        assertEquals(-1, input.getLine());
+        input.readChar();
+        assertEquals(-1, input.getLine());
+        input.readChar();
+        assertEquals(-1, input.getLine());
+        input.readChar();
+        assertEquals(-1, input.getLine());
+    }
+
+
+    @Test
+    public void requireThatBackupIsSupported() throws IOException {
+        FastCharStream input = new FastCharStream("foo");
+        assertEquals('f', input.readChar());
+        input.backup(1);
+        assertEquals('f', input.readChar());
+        assertEquals('o', input.readChar());
+        input.backup(2);
+        assertEquals('f', input.readChar());
+        assertEquals('o', input.readChar());
+        assertEquals('o', input.readChar());
+        input.backup(3);
+        assertEquals('f', input.readChar());
+        assertEquals('o', input.readChar());
+        assertEquals('o', input.readChar());
+        input.backup(2);
+        assertEquals('o', input.readChar());
+        assertEquals('o', input.readChar());
+        input.backup(1);
+        assertEquals('o', input.readChar());
+    }
+
+    @Test
+    public void requireThatSuffixIsNotSupported() {
+        try {
+            new FastCharStream("foo").GetSuffix(0);
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatDoneDoesNotThrowException() {
+        FastCharStream input = new FastCharStream("foo");
+        input.Done();
+    }
+
+    @Test
+    public void requireThatTokensCanBeRetrieved() throws IOException {
+        FastCharStream input = new FastCharStream("foo bar baz");
+        input.readChar();
+        input.readChar();
+        input.readChar();
+        input.readChar();
+        assertEquals('b', input.BeginToken());
+        assertEquals(5, input.getBeginColumn());
+        assertEquals(-1, input.getBeginLine());
+        assertEquals(6, input.getEndColumn());
+        assertEquals(-1, input.getEndLine());
+        assertEquals('a', input.readChar());
+        assertEquals('r', input.readChar());
+        assertEquals(8, input.getEndColumn());
+        assertEquals(-1, input.getEndLine());
+        assertEquals("bar", input.GetImage());
+    }
+
+    @Test
+    public void requireThatExceptionsDetectLineNumber() {
+        FastCharStream input = new FastCharStream("foo\nbar");
+        assertEquals("line 2, column 1\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     "^",
+                     input.formatException("line -1, column 5"));
+        assertEquals("line 2, column 2\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     " ^",
+                     input.formatException("line -1, column 6"));
+        assertEquals("line 2, column 3\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     "  ^",
+                     input.formatException("line -1, column 7"));
+        assertEquals("line 2, column 4\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     "   ^",
+                     input.formatException("line -1, column 8"));
+        assertEquals("foo line 2, column 2\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     " ^",
+                     input.formatException("foo line -1, column 6"));
+        assertEquals("foo line 2, column 2 bar\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     " ^",
+                     input.formatException("foo line -1, column 6 bar"));
+        assertEquals("line 2, column 2 bar\n" +
+                     "At position:\n" +
+                     "bar\n" +
+                     " ^",
+                     input.formatException("line -1, column 6 bar"));
+    }
+
+    @Test
+    public void requireErrorMsgExceptionAtEOF() {
+        FastCharStream input = new FastCharStream("\n");
+        assertEquals("line 1, column 1\n" +
+                     "At position:\n" +
+                     "EOF\n" +
+                     "^",
+                     input.formatException("line -1, column 1"));
+    }
+    
+    @Test
+    public void requireThatUnknownExceptionFormatIsIgnored() {
+        FastCharStream input = new FastCharStream("foo\nbar");
+        assertEquals("",
+                     input.formatException(""));
+        assertEquals("foo",
+                     input.formatException("foo"));
+        assertEquals("foo line -1, column ",
+                     input.formatException("foo line -1, column "));
+        assertEquals("foo line -1, column bar",
+                     input.formatException("foo line -1, column bar"));
+    }
+
+    @Test
+    public void requireThatIllegalExceptionColumnIsIgnored() {
+        FastCharStream input = new FastCharStream("foo\nbar");
+        assertEquals("line -1, column 9",
+                     input.formatException("line -1, column 9"));
+    }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java b/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java
new file mode 100644
index 00000000000..81329e06308
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.javacc;
+
+import org.junit.Test;
+
+import static com.yahoo.javacc.UnicodeUtilities.quote;
+import static com.yahoo.javacc.UnicodeUtilities.unquote;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UnicodeUtilitiesTestCase {
+
+    @Test
+    public void testQuote() {
+        assertEquals("'\\f'", quote("\f", '\''));
+        assertEquals("'\\n'", quote("\n", '\''));
+        assertEquals("'\\r'", quote("\r", '\''));
+        assertEquals("'\\t'", quote("\t", '\''));
+
+        for (char c = 'a'; c <= 'z'; ++c) {
+            assertEquals("'" + c + "'", quote(String.valueOf(c), '\''));
+        }
+
+        assertEquals("'\\u4f73'", quote("\u4f73", '\''));
+        assertEquals("'\\u80fd'", quote("\u80fd", '\''));
+        assertEquals("'\\u7d22'", quote("\u7d22", '\''));
+        assertEquals("'\\u5c3c'", quote("\u5c3c", '\''));
+        assertEquals("'\\u60e0'", quote("\u60e0", '\''));
+        assertEquals("'\\u666e'", quote("\u666e", '\''));
+
+        assertEquals("\"foo\"", quote("foo", '"'));
+        assertEquals("\"'foo\"", quote("'foo", '"'));
+        assertEquals("\"foo'\"", quote("foo'", '"'));
+        assertEquals("\"'foo'\"", quote("'foo'", '"'));
+        assertEquals("\"\\\"foo\"", quote("\"foo", '"'));
+        assertEquals("\"foo\\\"\"", quote("foo\"", '"'));
+        assertEquals("\"\\\"foo\\\"\"", quote("\"foo\"", '"'));
+        assertEquals("\"\\\"'foo'\\\"\"", quote("\"'foo'\"", '"'));
+        assertEquals("\"'\\\"foo\\\"'\"", quote("'\"foo\"'", '"'));
+        assertEquals("\"'f\\\\'o\\\"o\\\\\\\\'\"", quote("'f\\'o\"o\\\\'", '"'));
+
+        assertEquals("\"\\female \\nude fa\\rt fe\\tish\"", quote("\female \nude fa\rt fe\tish", '"'));
+        assertEquals("\"\\u666e\"", quote("\u666e", '"'));
+    }
+
+    @Test
+    public void testQuoteUnquote() {
+        assertEquals("\"foo\"", quote(unquote("'foo'"), '"'));
+        assertEquals("\"\\foo\"", quote(unquote(quote("\foo", '"')), '"'));
+        assertEquals("\u666e", unquote(quote("\u666e", '"')));
+    }
+
+    @Test
+    public void testUnquote() {
+        assertEquals("foo", unquote("foo"));
+        assertEquals("'foo", unquote("'foo"));
+        assertEquals("foo'", unquote("foo'"));
+        assertEquals("foo", unquote("'foo'"));
+        assertEquals("\"foo", unquote("\"foo"));
+        assertEquals("foo\"", unquote("foo\""));
+        assertEquals("foo", unquote("\"foo\""));
+        assertEquals("'foo'", unquote("\"'foo'\""));
+        assertEquals("\"foo\"", unquote("'\"foo\"'"));
+        assertEquals("f'o\"o\\", unquote("'f\\'o\"o\\\\'"));
+
+        assertEquals("\female \nude fa\rt fe\tish", unquote("'\\female \\nude fa\\rt fe\\tish'"));
+        assertEquals("\u666e", unquote("\"\\u666e\""));
+
+        try {
+            unquote("\"\\uSiM0N\"");
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        assertEquals("simo\n", unquote("'\\s\\i\\m\\o\\n'"));
+        try {
+            unquote("\"foo\"bar\"");
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            unquote("'foo'bar'");
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatTokenIncludesOnlyAcceptedChars() {
+        assertEquals("\"\\u0000\",\"\\u7777\",\"\\uffff\",",
+                     UnicodeUtilities.generateToken(new UnicodeUtilities.Predicate() {
+
+                         @Override
+                         public boolean accepts(char c) {
+                             return c == 0x0000 || c == 0x7777 || c == 0xffff;
+                         }
+                     }));
+        assertEquals("\"\\u0006\",\"\\u0009\",\"\\u0060\"-\"\\u0069\",",
+                     UnicodeUtilities.generateToken(new UnicodeUtilities.Predicate() {
+
+                         @Override
+                         public boolean accepts(char c) {
+                             return c == 0x6 || c == 0x9 || (c >= 0x60 && c <= 0x69);
+                         }
+                     }));
+    }
+}