Publish

4 files changed, 420 insertions, 0 deletions

diff --git a/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java
new file mode 100644
index 00000000000..ce9854e7c44
--- /dev/null
+++ b/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.fsa.test;
+
+import com.yahoo.fsa.FSA;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.nio.BufferUnderflowException;
+
+/**
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class FSADataTestCase extends junit.framework.TestCase {
+
+    private static class Worker extends Thread {
+        FSA.State state;
+        String word;
+        String data;
+        long numRuns;
+        long numExceptions;
+        long numAsserts;
+        public Worker(FSA fsa, String word, String data, long numRuns) {
+            state = fsa.getState();
+            this.word = word;
+            this.data = data;
+            this.numRuns = numRuns;
+            this.numExceptions = 0;
+            this.numAsserts = 0;
+        }
+        public void run() {
+            for (long i = 0; i < numRuns; ++i) {
+                state.start();
+                state.delta(word);
+                try {
+                    String data = state.dataString();
+                    if (!this.data.equals(data)) {
+                        ++numAsserts;
+                    }
+                } catch (BufferUnderflowException e) {
+                    ++numExceptions;
+                }
+            }
+            System.out.println("Worker(" + word + "): numExceptions(" + numExceptions + "), numAsserts(" + numAsserts + ")");
+        }
+    };
+
+    private FSA fsa;
+
+    public FSADataTestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() throws IOException {
+        fsa = new FSA(new FileInputStream("src/test/fsa/test-data.fsa"));
+    }
+
+    public void testBasic() {
+        FSA.State state = fsa.getState();
+        state.delta("aa");
+        assertTrue(state.isFinal());
+        assertEquals("aa data", state.dataString());
+
+        state.start();
+        state.delta("bbbb");
+        assertTrue(state.isFinal());
+        assertEquals("bbbb data", state.dataString());
+
+        state.start();
+        state.delta("c");
+        assertTrue(state.isFinal());
+        assertEquals("c data", state.dataString());
+
+        state.start();
+        state.delta("dddddd");
+        assertTrue(state.isFinal());
+        assertEquals("dddddd data", state.dataString());
+    }
+
+    public void testMultipleThreads() {
+        long numRuns = 10000;
+        List<Worker> workers = new ArrayList<Worker>();
+        workers.add(new Worker(fsa, "aa", "aa data", numRuns));
+        workers.add(new Worker(fsa, "bbbb", "bbbb data", numRuns));
+        workers.add(new Worker(fsa, "c", "c data", numRuns));
+        workers.add(new Worker(fsa, "dddddd", "dddddd data", numRuns));
+        for (int i = 0; i < workers.size(); ++i) {
+            workers.get(i).start();
+        }
+        try {
+            for (int i = 0; i < workers.size(); ++i) {
+                workers.get(i).join();
+            }
+        } catch (InterruptedException e) {
+            assertTrue(false);
+        }
+        for (int i = 0; i < workers.size(); ++i) {
+            assertEquals(0, workers.get(i).numExceptions);
+            assertEquals(0, workers.get(i).numAsserts);
+        }
+    }
+
+}
diff --git a/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java
new file mode 100644
index 00000000000..21dc86f4925
--- /dev/null
+++ b/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.fsa.test;
+
+import com.yahoo.fsa.FSA;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class FSAIteratorTestCase extends junit.framework.TestCase {
+
+    private FSA fsa;
+
+    private FSA.State state;
+
+    private List<String> expected;
+
+    public FSAIteratorTestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() {
+        fsa = new FSA("src/test/fsa/test-iterator.fsa");
+        state = fsa.getState();
+
+        expected = new ArrayList<String>();
+
+        expected.add("abacus");
+        expected.add("abadan");
+        expected.add("abaisse");
+        expected.add("abdicate");
+        expected.add("abdomen");
+        expected.add("abdominous");
+        expected.add("dachs");
+        expected.add("dacia");
+        expected.add("daciaa");
+        expected.add("daciab");
+        expected.add("dacite");
+        expected.add("dacota");
+    }
+
+    private void checkIterator(int beginIdx, int endIdx, String prefix) {
+        System.out.println("checkIterator(" + beginIdx + ", " + endIdx + ", " + prefix + ")");
+        java.util.Iterator<FSA.Iterator.Item> i = fsa.iterator(state);
+        for (; i.hasNext() && beginIdx < endIdx; ++beginIdx) {
+            FSA.Iterator.Item item = i.next();
+            System.out.println("item: " + item);
+            String str = prefix + item.getString();
+            String data = item.getDataString();
+            System.out.println("str:  '" + expected.get(beginIdx) + "'.equals('" + str + "')?");
+            assertTrue(expected.get(beginIdx).equals(str));
+            System.out.println("data: '" + expected.get(beginIdx) + "'.equals('" + data + "')?");
+            assertTrue(expected.get(beginIdx).equals(data));
+        }
+        assertFalse(i.hasNext());
+        assertTrue(beginIdx == endIdx);
+    }
+
+    public void testIterator() {
+        checkIterator(0, expected.size(), "");
+    }
+
+    public void testIteratorSingle() {
+        state.delta("dach");
+        checkIterator(6, 7, "dach");
+    }
+
+    public void testIteratorSubset() {
+        state.delta("abd");
+        checkIterator(3, 6, "abd");
+    }
+
+    public void testIteratorFinalState() {
+        state.delta("dacia");
+        checkIterator(7, 10, "dacia");
+    }
+
+    public void testIteratorFinalStateOnly() {
+        state.delta("dachs");
+        checkIterator(6, 7, "dachs");
+    }
+
+    public void testIteratorEmpty1() {
+        state.delta("b");
+        java.util.Iterator i = fsa.iterator(state);
+        assertFalse(i.hasNext());
+        try {
+            i.next();
+            assertFalse(true);
+        } catch (NoSuchElementException e) {
+            assertTrue(true);
+        }
+    }
+
+    public void testIteratorEmpty2() {
+        state.delta("daciac");
+        java.util.Iterator i = fsa.iterator(state);
+        assertFalse(i.hasNext());
+        try {
+            i.next();
+            assertFalse(true);
+        } catch (NoSuchElementException e) {
+            assertTrue(true);
+        }
+    }
+
+    public void testIteratorRemove() {
+        java.util.Iterator i = fsa.iterator(state);
+        try {
+            i.remove();
+            assertFalse(true);
+        } catch (UnsupportedOperationException e) {
+            assertTrue(true);
+        }
+    }
+}
diff --git a/fsa/src/test/java/com/yahoo/fsa/test/FSATestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/FSATestCase.java
new file mode 100644
index 00000000000..4300c5938e1
--- /dev/null
+++ b/fsa/src/test/java/com/yahoo/fsa/test/FSATestCase.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.fsa.test;
+
+import com.yahoo.fsa.FSA;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+/**
+ * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class FSATestCase extends junit.framework.TestCase {
+
+    private FSA fsa;
+
+    private FSA.State state;
+
+    public FSATestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() throws IOException {
+        fsa=new FSA(new FileInputStream("src/test/fsa/test-fsa.fsa"));
+        state=fsa.getState();
+    }
+
+    public void testSingleWordDelta() {
+        state.delta("aword");
+        assertTrue(state.isValid());
+        assertTrue(state.isFinal());
+    }
+
+    public void testSingleWordDeltaWord() {
+        state.deltaWord("aword");
+        assertTrue(state.isValid());
+        assertTrue(state.isFinal());
+    }
+
+    public void testSingleWordDeltaPartialMatch() {
+        state.delta("awo");
+        assertTrue(state.isValid());
+        assertFalse(state.isFinal());
+    }
+
+    public void testSingleWordDeltaPartialMatchWord() {
+        state.deltaWord("awo");
+        assertTrue(state.isValid());
+        assertFalse(state.isFinal());
+    }
+
+    public void testMultiWordDelta() {
+        state.delta("th");
+        assertFalse(state.isFinal());
+        state.delta("is ");
+        assertFalse(state.isFinal());
+        state.delta("is ");
+        assertFalse(state.isFinal());
+        state.delta("a");
+        assertFalse(state.isFinal());
+        state.delta(" test");
+        assertTrue(state.isValid());
+        assertTrue(state.isFinal());
+    }
+
+    public void testMultiWordDeltaWord() {
+        state.deltaWord("this");
+        assertFalse(state.isFinal());
+        state.deltaWord("is");
+        assertFalse(state.isFinal());
+        state.deltaWord("a");
+        assertFalse(state.isFinal());
+        state.deltaWord("test");
+        assertTrue(state.isValid());
+        assertTrue(state.isFinal());
+    }
+
+    public void testMultiWordDeltaWordInvalid() {
+        state.deltaWord("th");
+        assertFalse(state.isFinal());
+        state.deltaWord("is ");
+        assertFalse(state.isFinal());
+        assertFalse(state.isValid());
+    }
+
+    public void testMultiWordDeltaTry() {
+        assertFalse(state.tryDeltaWord("thiss"));
+        assertTrue(state.isValid());
+        assertTrue(state.tryDeltaWord("this"));
+        state.deltaWord("is");
+        state.tryDeltaWord("a");
+        assertFalse(state.tryDeltaWord("tes"));
+        assertFalse(state.tryDeltaWord("tesz"));
+        assertFalse(state.tryDeltaWord("teszzzz"));
+        assertTrue(state.tryDeltaWord("test"));
+        assertTrue(state.isValid());
+        assertTrue(state.isFinal());
+
+    }
+
+}
diff --git a/fsa/src/test/java/com/yahoo/fsa/test/UTF8TestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/UTF8TestCase.java
new file mode 100644
index 00000000000..3f07816a914
--- /dev/null
+++ b/fsa/src/test/java/com/yahoo/fsa/test/UTF8TestCase.java
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.fsa.test;
+
+import com.yahoo.fsa.FSA;
+import java.util.Iterator;
+import java.nio.charset.Charset;
+
+/**
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class UTF8TestCase extends junit.framework.TestCase {
+
+    private Charset charset = Charset.forName("utf-8");
+    private FSA fsa;
+    private FSA.State state;
+    private byte prefixBuf[];
+    private byte suffixBuf[];
+    private String prefix;
+    private String suffix;
+    private String word;
+
+    private static byte [] convert(int [] buf) {
+        byte retval[] = new byte[buf.length];
+        for (int i = 0; i < buf.length; ++i) {
+            retval[i] = (byte)buf[i];
+        }
+        return retval;
+    }
+
+    public UTF8TestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() {
+        fsa = new FSA("src/test/fsa/utf8.fsa"); // fsa with one word (6 code points, 18 bytes)
+        state = fsa.getState();
+        int pbuf[] = {0xe0,0xa4,0xb9};
+        prefixBuf = convert(pbuf);
+        prefix = new String(prefixBuf, charset);
+        int sbuf[] = {0xe0,0xa4,0xbf,0xe0,0xa4,0xa8,0xe0,0xa5,0x8d,0xe0,0xa4,0xa6,0xe0,0xa5,0x80};
+        suffixBuf = convert(sbuf);
+        suffix = new String(suffixBuf, charset);
+        word = prefix + suffix;
+    }
+
+    public void testStringDelta() {
+        state.delta(word);
+        assertTrue(state.isFinal());
+    }
+
+    public void testCharDelta() {
+        assertEquals(6, word.length());
+        for (int i = 0; i < word.length(); ++i) {
+            state.delta(word.charAt(i));
+            assertTrue(state.isValid());
+        }
+        assertTrue(state.isFinal());
+    }
+
+    public void testByteDelta() {
+        FSA.State state = fsa.getState();
+        assertEquals(3, prefixBuf.length);
+        for (int i = 0; i < prefixBuf.length; ++i) {
+            state.delta(prefixBuf[i]);
+            assertTrue(state.isValid());
+        }
+        assertEquals(15, suffixBuf.length);
+        for (int i = 0; i < suffixBuf.length; ++i) {
+            state.delta(suffixBuf[i]);
+            assertTrue(state.isValid());
+        }
+        assertTrue(state.isFinal());
+    }
+
+    public void testIteratorAtStart() {
+        Iterator<FSA.Iterator.Item> itr = fsa.iterator(state);
+        FSA.Iterator.Item item = itr.next();
+        assertEquals(word, item.getString());
+        assertFalse(itr.hasNext());
+    }
+
+    public void testIteratorWithPrefix() {
+        state.delta(prefix);
+        Iterator<FSA.Iterator.Item> itr = fsa.iterator(state);
+        FSA.Iterator.Item item = itr.next();
+        assertEquals(suffix, item.getString());
+        assertFalse(itr.hasNext());
+    }
+
+    public void testIteratorWithCompleteWord() {
+        state.delta(word);
+        Iterator<FSA.Iterator.Item> itr = fsa.iterator(state);
+        FSA.Iterator.Item item = itr.next();
+        assertEquals("", item.getString());
+        assertFalse(itr.hasNext());
+    }
+}