Fold fastlib into vespalib and gc some unused code.

Also move some code only used by juniper up into juniper test module.

7 files changed, 243 insertions, 0 deletions

diff --git a/vespalib/src/tests/fastlib/io/.gitignore b/vespalib/src/tests/fastlib/io/.gitignore
new file mode 100644
index 00000000000..816281ccbfb
--- /dev/null
+++ b/vespalib/src/tests/fastlib/io/.gitignore
@@ -0,0 +1,23 @@
+*.So
+*.a
+*.ilk
+*.o
+*.pdb
+.cvsignore
+.depend
+.pure
+Debug
+Makefile
+SunWS_cache
+bufferedfiletest
+bufferedstreamtest
+bufferedstreamtest.tmp
+datastreamtest
+datastreamtestfile.tmp
+linenumberinputstreamtest
+outputfile.txt
+pushbackinputstreamtest
+streampumptest
+stringinputstreamtest
+fastlib_bufferedfiletest_app
+fastlib_bufferedstreamtest_app
diff --git a/vespalib/src/tests/fastlib/io/CMakeLists.txt b/vespalib/src/tests/fastlib/io/CMakeLists.txt
new file mode 100644
index 00000000000..345b3456bbf
--- /dev/null
+++ b/vespalib/src/tests/fastlib/io/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(fastlib_bufferedfiletest_app TEST
+    SOURCES
+    bufferedfiletest.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME fastlib_bufferedfiletest_app COMMAND fastlib_bufferedfiletest_app)
diff --git a/vespalib/src/tests/fastlib/io/bufferedfiletest.cpp b/vespalib/src/tests/fastlib/io/bufferedfiletest.cpp
new file mode 100644
index 00000000000..8aa9b943419
--- /dev/null
+++ b/vespalib/src/tests/fastlib/io/bufferedfiletest.cpp
@@ -0,0 +1,97 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastlib/io/bufferedfile.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
+
+TEST("main") {
+    int value = 0;
+    FastOS_StatInfo statInfo;
+
+    FastOS_File::Delete("testfile1");
+    FastOS_File::Delete("testfile2");
+    FastOS_File::Delete("testfile3");
+    FastOS_File::Delete("testfile4");
+    FastOS_File::Delete("testfile5");
+
+    Fast_BufferedFile bufFile(4096);
+
+    // test 1
+    printf ("testing 11 byte long file\n");
+    bufFile.WriteOpen("testfile1");
+    bufFile.addNum(1,10,' ');
+    ASSERT_TRUE(bufFile.CheckedWrite("\n",1));
+    ASSERT_TRUE(bufFile.Close());
+    FastOS_File::Stat("testfile1", &statInfo);
+    if (statInfo._size != 11) {
+        printf (" -- FAILURE\n\n");
+        TEST_FATAL("exit 1");
+    }
+    printf (" -- SUCCESS\n\n");
+
+    // test 2
+    printf ("testing 4095 byte long file\n");
+    bufFile.WriteOpen("testfile2");
+    char buf[8192]; // allocate 8K buffer
+    memset(buf,0xff,8192);
+    ASSERT_TRUE(bufFile.CheckedWrite(buf,4095)); // write almost 4K
+    ASSERT_TRUE(bufFile.Close());
+    FastOS_File::Stat("testfile2", &statInfo);
+    if (statInfo._size != 4095) {
+        printf (" -- FAILURE\n\n");
+        TEST_FATAL("exit 1");
+    }
+    printf (" -- SUCCESS\n\n");
+
+    // test 3
+    printf ("testing 4096 byte long file\n");
+    bufFile.WriteOpen("testfile3");
+    ASSERT_TRUE(bufFile.CheckedWrite(buf,4096));  // write exactly 4K
+    ASSERT_TRUE(bufFile.Close());
+    FastOS_File::Stat("testfile3", &statInfo);
+    if (statInfo._size != 4096) {
+        printf (" -- FAILURE\n\n");
+        TEST_FATAL("exit 1");
+    }
+    printf (" -- SUCCESS\n\n");
+
+    // test 4
+    printf ("testing 4097 byte long file\n");
+    bufFile.WriteOpen("testfile4");
+    ASSERT_TRUE(bufFile.CheckedWrite(buf,4097));   // write a bit over 4K
+    ASSERT_TRUE(bufFile.Close());
+    FastOS_File::Stat("testfile4", &statInfo);
+    if (statInfo._size != 4097) {
+        printf (" -- FAILURE\n\n");
+        TEST_FATAL("exit 1");
+    }
+    printf (" -- SUCCESS\n\n");
+
+    // test 5
+    printf ("testing 610000 byte long file with repeated addNum\n");
+    bufFile.WriteOpen("testfile5");
+    for (int i = 0; i < 10000; i++) {
+        for (int j = 0; j < 10; j++) {
+            bufFile.addNum(value,6,' ');
+            value++;
+        }
+        ASSERT_TRUE(bufFile.CheckedWrite("\n",1));
+    }
+    ASSERT_TRUE(bufFile.Close());
+    FastOS_File::Stat("testfile5", &statInfo);
+    if (statInfo._size != 610000) {
+        printf (" -- FAILURE\n\n");
+        TEST_FATAL("exit 1");
+    }
+    printf (" -- SUCCESS\n\n");
+
+    FastOS_File::Delete("testfile1");
+    FastOS_File::Delete("testfile2");
+    FastOS_File::Delete("testfile3");
+    FastOS_File::Delete("testfile4");
+    FastOS_File::Delete("testfile5");
+
+    printf ("All tests OK for bufferedfiletest\n");
+    printf (" -- SUCCESS\n\n");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/fastlib/text/.gitignore b/vespalib/src/tests/fastlib/text/.gitignore
new file mode 100644
index 00000000000..8134602778d
--- /dev/null
+++ b/vespalib/src/tests/fastlib/text/.gitignore
@@ -0,0 +1,23 @@
+*.So
+*.a
+*.ilk
+*.lib
+*.o
+*.obj
+*.pdb
+.cvsignore
+.depend
+.pure
+Debug
+Makefile
+SunWS_cache
+alpharanktest
+latintokenizertest
+limitstrtest
+list.html
+map.html
+unicodeutiltest
+wordfolderstest
+fastlib_latintokenizertest_app
+fastlib_unicodeutiltest_app
+fastlib_wordfolderstest_app
diff --git a/vespalib/src/tests/fastlib/text/CMakeLists.txt b/vespalib/src/tests/fastlib/text/CMakeLists.txt
new file mode 100644
index 00000000000..690da0a3d80
--- /dev/null
+++ b/vespalib/src/tests/fastlib/text/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(fastlib_unicodeutiltest_app TEST
+    SOURCES
+    unicodeutiltest.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME fastlib_unicodeutiltest_app NO_VALGRIND COMMAND fastlib_unicodeutiltest_app)
+vespa_add_executable(fastlib_wordfolderstest_app TEST
+    SOURCES
+    wordfolderstest.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME fastlib_wordfolderstest_app NO_VALGRIND COMMAND fastlib_wordfolderstest_app)
diff --git a/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp
new file mode 100644
index 00000000000..d734b3b6aab
--- /dev/null
+++ b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp
@@ -0,0 +1,31 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastlib/text/unicodeutil.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
+TEST("GetUTF8Char_WrongInput") {
+    const char *testdata = "ab\xF8";
+
+    ucs4_t the_char = 0;
+
+    const unsigned char *src = reinterpret_cast<const unsigned char *>(testdata);
+    while (*src != 0) {
+        the_char = Fast_UnicodeUtil::GetUTF8Char(src);
+    }
+    EXPECT_EQUAL(Fast_UnicodeUtil::_BadUTF8Char, the_char);
+}
+
+TEST("IsTerminalPunctuationChar") {
+    // test a small selection
+
+    EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('!'));
+    EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(','));
+    EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('.'));
+    EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(':'));
+    EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(';'));
+    EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar(' '));
+    EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('a'));
+    EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('A'));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/vespalib/src/tests/fastlib/text/wordfolderstest.cpp b/vespalib/src/tests/fastlib/text/wordfolderstest.cpp
new file mode 100644
index 00000000000..b2e05250951
--- /dev/null
+++ b/vespalib/src/tests/fastlib/text/wordfolderstest.cpp
@@ -0,0 +1,46 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastlib/text/normwordfolder.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
+TEST("NormalizeWordFolderConstruction") {
+    Fast_NormalizeWordFolder::Setup(
+            Fast_NormalizeWordFolder::DO_ACCENT_REMOVAL
+            | Fast_NormalizeWordFolder::DO_KATAKANA_TO_HIRAGANA
+            | Fast_NormalizeWordFolder::DO_SMALL_TO_NORMAL_KANA
+            | Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION
+            | Fast_NormalizeWordFolder::DO_LIGATURE_SUBSTITUTION
+            | Fast_NormalizeWordFolder::DO_MULTICHAR_EXPANSION);
+}
+
+TEST("TokenizeAnnotatedUCS4Buffer") {
+    auto nwf = std::make_unique<Fast_NormalizeWordFolder>();
+    const char *testinput = "This is a "
+                            "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB"
+                            " superduperextrafeaturecoolandlongplainword fun "
+                            "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA"
+                            "world wide web extra long annotation block" "\xEF\xBF\xBB"
+                            " test\nIt is cool.\n";
+    const char *correct[] = {
+            "this", "is", "a",
+            "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB",
+            "superduperextrafeaturecooland", "fun",
+            "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" "world wide web extra lon",
+            "test", "it", "is", "cool" };
+
+    const char *teststart = testinput;
+    const char *testend = testinput + strlen(testinput);
+    ucs4_t destbuf[32];
+    ucs4_t *destbufend = destbuf + 32;
+
+    const char *origstart = testinput;
+    size_t tokenlen = 0;
+
+    int tokencounter = 0;
+    while ((teststart = nwf->UCS4Tokenize(teststart, testend, destbuf, destbufend, origstart, tokenlen)) < testend) {
+        EXPECT_EQUAL(0, Fast_UnicodeUtil::utf8cmp(correct[tokencounter++], destbuf));
+    }
+
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file