Collapse vsm into streamingvisitors

23 files changed, 2131 insertions, 4 deletions

diff --git a/streamingvisitors/src/tests/charbuffer/.gitignore b/streamingvisitors/src/tests/charbuffer/.gitignore
new file mode 100644
index 00000000000..2c980038fb5
--- /dev/null
+++ b/streamingvisitors/src/tests/charbuffer/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+charbuffer_test
+vsm_charbuffer_test_app
diff --git a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt
new file mode 100644
index 00000000000..5d0c0068d37
--- /dev/null
+++ b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vsm_charbuffer_test_app TEST
+    SOURCES
+    charbuffer.cpp
+    DEPENDS
+    streamingvisitors
+)
+vespa_add_test(NAME vsm_charbuffer_test_app COMMAND vsm_charbuffer_test_app)
diff --git a/streamingvisitors/src/tests/charbuffer/charbuffer.cpp b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp
new file mode 100644
index 00000000000..736d35459cb
--- /dev/null
+++ b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp
@@ -0,0 +1,80 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/vsm/common/charbuffer.h>
+
+namespace vsm {
+
+class CharBufferTest : public vespalib::TestApp
+{
+private:
+    void test();
+public:
+    int Main() override;
+};
+
+void
+CharBufferTest::test()
+{
+    { // empty
+        CharBuffer buf;
+        EXPECT_EQUAL(buf.getLength(), 0u);
+        EXPECT_EQUAL(buf.getPos(), 0u);
+        EXPECT_EQUAL(buf.getRemaining(), 0u);
+    }
+    { // explicit length
+        CharBuffer buf(8);
+        EXPECT_EQUAL(buf.getLength(), 8u);
+        EXPECT_EQUAL(buf.getPos(), 0u);
+        EXPECT_EQUAL(buf.getRemaining(), 8u);
+    }
+    { // resize
+        CharBuffer buf(8);
+        EXPECT_EQUAL(buf.getLength(), 8u);
+        buf.resize(16);
+        EXPECT_EQUAL(buf.getLength(), 16u);
+        buf.resize(8);
+        EXPECT_EQUAL(buf.getLength(), 16u);
+    }
+    { // put with triggered resize
+        CharBuffer buf(8);
+        buf.put("123456", 6);
+        EXPECT_EQUAL(buf.getLength(), 8u);
+        EXPECT_EQUAL(buf.getPos(), 6u);
+        EXPECT_EQUAL(buf.getRemaining(), 2u);
+        EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456");
+        buf.put("789", 3);
+        EXPECT_EQUAL(buf.getLength(), 12u);
+        EXPECT_EQUAL(buf.getPos(), 9u);
+        EXPECT_EQUAL(buf.getRemaining(), 3u);
+        EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789");
+        buf.put('a');
+        EXPECT_EQUAL(buf.getLength(), 12u);
+        EXPECT_EQUAL(buf.getPos(), 10u);
+        EXPECT_EQUAL(buf.getRemaining(), 2u);
+        EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789a");
+        buf.reset();
+        EXPECT_EQUAL(buf.getLength(), 12u);
+        EXPECT_EQUAL(buf.getPos(), 0u);
+        EXPECT_EQUAL(buf.getRemaining(), 12u);
+        buf.put("bcd", 3);
+        EXPECT_EQUAL(buf.getLength(), 12u);
+        EXPECT_EQUAL(buf.getPos(), 3u);
+        EXPECT_EQUAL(buf.getRemaining(), 9u);
+        EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "bcd");
+    }
+}
+
+int
+CharBufferTest::Main()
+{
+    TEST_INIT("charbuffer_test");
+
+    test();
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(vsm::CharBufferTest);
diff --git a/streamingvisitors/src/tests/config/mail.cfg b/streamingvisitors/src/tests/config/mail.cfg
new file mode 100644
index 00000000000..ce830beac23
--- /dev/null
+++ b/streamingvisitors/src/tests/config/mail.cfg
@@ -0,0 +1,116 @@
+datatype[2]
+datatype[0].id 1012
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 12
+datatype[1].id 1013
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 13
+documenttype[1]
+documenttype[0].name mail
+documenttype[0].version 0
+documenttype[0].inherits[0]
+documenttype[0].field[26]
+documenttype[0].field[0].name mailid
+documenttype[0].field[0].id 2
+documenttype[0].field[0].header true
+documenttype[0].field[0].datatype 2
+documenttype[0].field[1].name date
+documenttype[0].field[1].id 3
+documenttype[0].field[1].header true
+documenttype[0].field[1].datatype 0
+documenttype[0].field[2].name from
+documenttype[0].field[2].id 4
+documenttype[0].field[2].header true
+documenttype[0].field[2].datatype 12
+documenttype[0].field[3].name replyto
+documenttype[0].field[3].id 5
+documenttype[0].field[3].header true
+documenttype[0].field[3].datatype 12
+documenttype[0].field[4].name to
+documenttype[0].field[4].id 6
+documenttype[0].field[4].header true
+documenttype[0].field[4].datatype 12
+documenttype[0].field[5].name cc
+documenttype[0].field[5].id 7
+documenttype[0].field[5].header true
+documenttype[0].field[5].datatype 12
+documenttype[0].field[6].name bcc
+documenttype[0].field[6].id 8
+documenttype[0].field[6].header true
+documenttype[0].field[6].datatype 12
+documenttype[0].field[7].name subject
+documenttype[0].field[7].id 9
+documenttype[0].field[7].header true
+documenttype[0].field[7].datatype 12
+documenttype[0].field[8].name body
+documenttype[0].field[8].id 10
+documenttype[0].field[8].header false
+documenttype[0].field[8].datatype 12
+documenttype[0].field[9].name attachmentcount
+documenttype[0].field[9].id 11
+documenttype[0].field[9].header false
+documenttype[0].field[9].datatype 0
+documenttype[0].field[10].name attachmentpartids
+documenttype[0].field[10].id 12
+documenttype[0].field[10].header false
+documenttype[0].field[10].datatype 2
+documenttype[0].field[11].name attachmentsizes
+documenttype[0].field[11].id 13
+documenttype[0].field[11].header false
+documenttype[0].field[11].datatype 2
+documenttype[0].field[12].name attachmentnames
+documenttype[0].field[12].id 14
+documenttype[0].field[12].header false
+documenttype[0].field[12].datatype 2
+documenttype[0].field[13].name attachmenttypes
+documenttype[0].field[13].id 15
+documenttype[0].field[13].header false
+documenttype[0].field[13].datatype 2
+documenttype[0].field[14].name attachmentlanguages
+documenttype[0].field[14].id 16
+documenttype[0].field[14].header false
+documenttype[0].field[14].datatype 2
+documenttype[0].field[15].name attachmentcontent
+documenttype[0].field[15].id 17
+documenttype[0].field[15].header false
+documenttype[0].field[15].datatype 2
+documenttype[0].field[16].name bodylanguage
+documenttype[0].field[16].id 18
+documenttype[0].field[16].header false
+documenttype[0].field[16].datatype 2
+documenttype[0].field[17].name bodyencoding
+documenttype[0].field[17].id 19
+documenttype[0].field[17].header false
+documenttype[0].field[17].datatype 2
+documenttype[0].field[18].name collectionid
+documenttype[0].field[18].id 20
+documenttype[0].field[18].header true
+documenttype[0].field[18].datatype 4
+documenttype[0].field[19].name content
+documenttype[0].field[19].id 21
+documenttype[0].field[19].header true
+documenttype[0].field[19].datatype 12
+documenttype[0].field[20].name bodymeta
+documenttype[0].field[20].id 50027053
+documenttype[0].field[20].header false
+documenttype[0].field[20].datatype 13
+documenttype[0].field[21].name attachments
+documenttype[0].field[21].id 1081629685
+documenttype[0].field[21].header false
+documenttype[0].field[21].datatype 1012
+documenttype[0].field[22].name attachmentsmeta
+documenttype[0].field[22].id 1203055625
+documenttype[0].field[22].header false
+documenttype[0].field[22].datatype 1013
+documenttype[0].field[23].name tolist
+documenttype[0].field[23].id 1084918181
+documenttype[0].field[23].header false
+documenttype[0].field[23].datatype 1012
+documenttype[0].field[24].name cclist
+documenttype[0].field[24].id 1733332403
+documenttype[0].field[24].header false
+documenttype[0].field[24].datatype 1012
+documenttype[0].field[25].name bcclist
+documenttype[0].field[25].id 410546306
+documenttype[0].field[25].header false
+documenttype[0].field[25].datatype 1012
diff --git a/streamingvisitors/src/tests/config/vsm.cfg b/streamingvisitors/src/tests/config/vsm.cfg
new file mode 100644
index 00000000000..dc50447f623
--- /dev/null
+++ b/streamingvisitors/src/tests/config/vsm.cfg
@@ -0,0 +1,3 @@
+doctype file:../config/mail.cfg
+storagecfg ""
+vsmfields file:../config/vsmfields.cfg
diff --git a/streamingvisitors/src/tests/config/vsmfields.cfg b/streamingvisitors/src/tests/config/vsmfields.cfg
new file mode 100644
index 00000000000..30f1c8ed8b1
--- /dev/null
+++ b/streamingvisitors/src/tests/config/vsmfields.cfg
@@ -0,0 +1,297 @@
+threadsperquery 4
+documentverificationlevel=0
+searchall 1
+fieldspec[17]
+fieldspec[0].name bcc
+fieldspec[0].searchmethod AUTOUTF8
+fieldspec[0].arg1 ""
+fieldspec[1].name cc
+fieldspec[1].searchmethod AUTOUTF8
+fieldspec[1].arg1 ""
+fieldspec[2].name from
+fieldspec[2].searchmethod AUTOUTF8
+fieldspec[2].arg1 ""
+fieldspec[3].name date
+fieldspec[3].searchmethod INT32
+fieldspec[3].arg1 ""
+fieldspec[4].name replyto
+fieldspec[4].searchmethod AUTOUTF8
+fieldspec[4].arg1 ""
+fieldspec[5].name subject
+fieldspec[5].searchmethod AUTOUTF8
+fieldspec[5].arg1 ""
+fieldspec[6].name to
+fieldspec[6].searchmethod AUTOUTF8
+fieldspec[6].arg1 ""
+fieldspec[7].name body
+fieldspec[7].searchmethod AUTOUTF8
+fieldspec[7].arg1 ""
+fieldspec[8].name bodymeta
+fieldspec[8].searchmethod AUTOUTF8
+fieldspec[8].arg1 ""
+fieldspec[9].name mailid
+fieldspec[9].searchmethod AUTOUTF8
+fieldspec[9].arg1 ""
+fieldspec[10].name attachmentcount
+fieldspec[10].searchmethod INT32
+fieldspec[10].arg1 ""
+fieldspec[11].name attachmentcontent
+fieldspec[11].searchmethod AUTOUTF8
+fieldspec[11].arg1 ""
+fieldspec[12].name attachmenttypes
+fieldspec[12].searchmethod AUTOUTF8
+fieldspec[12].arg1 ""
+fieldspec[13].name attachmentnames
+fieldspec[13].searchmethod AUTOUTF8
+fieldspec[13].arg1 ""
+fieldspec[14].name attachmentlanguages
+fieldspec[14].searchmethod AUTOUTF8
+fieldspec[14].arg1 ""
+fieldspec[15].name URI
+fieldspec[15].searchmethod AUTOUTF8
+fieldspec[15].arg1 ""
+fieldspec[16].name vsm_whichfieldmatched
+fieldspec[16].searchmethod AUTOUTF8
+fieldspec[16].arg1 ""
+index[26]
+index[0].name default
+index[0].field[10]
+index[0].field[0].name from
+index[0].field[1].name to
+index[0].field[2].name cc
+index[0].field[3].name bcc
+index[0].field[4].name subject
+index[0].field[5].name body
+index[0].field[6].name attachmentcontent
+index[0].field[7].name attachmentnames
+index[0].field[8].name attachmenttypes
+index[0].field[9].name date
+index[1].name all
+index[1].field[8]
+index[1].field[0].name to
+index[1].field[1].name cc
+index[1].field[2].name bcc
+index[1].field[3].name subject
+index[1].field[4].name body
+index[1].field[5].name attachmentcontent
+index[1].field[6].name attachmentnames
+index[1].field[7].name attachmenttypes
+index[2].name header
+index[2].field[6]
+index[2].field[0].name from
+index[2].field[1].name replyto
+index[2].field[2].name to
+index[2].field[3].name cc
+index[2].field[4].name bcc
+index[2].field[5].name subject
+index[3].name senders
+index[3].field[2]
+index[3].field[0].name from
+index[3].field[1].name replyto
+index[4].name recipients
+index[4].field[3]
+index[4].field[0].name to
+index[4].field[1].name cc
+index[4].field[2].name bcc
+index[5].name address
+index[5].field[5]
+index[5].field[0].name from
+index[5].field[1].name replyto
+index[5].field[2].name to
+index[5].field[3].name cc
+index[5].field[4].name bcc
+index[6].name body
+index[6].field[2]
+index[6].field[0].name subject
+index[6].field[1].name body
+index[7].name meta
+index[7].field[2]
+index[7].field[0].name attachmentcontent
+index[7].field[1].name attachmenttypes
+index[8].name index1
+index[8].field[1]
+index[8].field[0].name bcc
+index[9].name index2
+index[9].field[2]
+index[9].field[0].name bcc
+index[9].field[1].name cc
+index[10].name index3
+index[10].field[3]
+index[10].field[0].name bcc
+index[10].field[1].name cc
+index[10].field[2].name from
+index[11].name index4
+index[11].field[4]
+index[11].field[0].name bcc
+index[11].field[1].name cc
+index[11].field[2].name from
+index[11].field[3].name date
+index[12].name index5
+index[12].field[5]
+index[12].field[0].name bcc
+index[12].field[1].name cc
+index[12].field[2].name from
+index[12].field[3].name date
+index[12].field[4].name replyto
+index[13].name index6
+index[13].field[6]
+index[13].field[0].name bcc
+index[13].field[1].name cc
+index[13].field[2].name from
+index[13].field[3].name date
+index[13].field[4].name replyto
+index[13].field[5].name subject
+index[14].name index7
+index[14].field[7]
+index[14].field[0].name bcc
+index[14].field[1].name cc
+index[14].field[2].name from
+index[14].field[3].name date
+index[14].field[4].name replyto
+index[14].field[5].name subject
+index[14].field[6].name to
+index[15].name index8
+index[15].field[8]
+index[15].field[0].name bcc
+index[15].field[1].name cc
+index[15].field[2].name from
+index[15].field[3].name date
+index[15].field[4].name replyto
+index[15].field[5].name subject
+index[15].field[6].name to
+index[15].field[7].name body
+index[16].name index9
+index[16].field[9]
+index[16].field[0].name bcc
+index[16].field[1].name cc
+index[16].field[2].name from
+index[16].field[3].name date
+index[16].field[4].name replyto
+index[16].field[5].name subject
+index[16].field[6].name to
+index[16].field[7].name body
+index[16].field[8].name bodymeta
+index[17].name index10
+index[17].field[10]
+index[17].field[0].name bcc
+index[17].field[1].name cc
+index[17].field[2].name from
+index[17].field[3].name date
+index[17].field[4].name replyto
+index[17].field[5].name subject
+index[17].field[6].name to
+index[17].field[7].name body
+index[17].field[8].name bodymeta
+index[17].field[9].name mailid
+index[18].name index11
+index[18].field[11]
+index[18].field[0].name bcc
+index[18].field[1].name cc
+index[18].field[2].name from
+index[18].field[3].name date
+index[18].field[4].name replyto
+index[18].field[5].name subject
+index[18].field[6].name to
+index[18].field[7].name body
+index[18].field[8].name bodymeta
+index[18].field[9].name mailid
+index[18].field[10].name attachmentcount
+index[19].name index12
+index[19].field[12]
+index[19].field[0].name bcc
+index[19].field[1].name cc
+index[19].field[2].name from
+index[19].field[3].name date
+index[19].field[4].name replyto
+index[19].field[5].name subject
+index[19].field[6].name to
+index[19].field[7].name body
+index[19].field[8].name bodymeta
+index[19].field[9].name mailid
+index[19].field[10].name attachmentcount
+index[19].field[11].name attachmentcontent
+index[20].name index13
+index[20].field[13]
+index[20].field[0].name bcc
+index[20].field[1].name cc
+index[20].field[2].name from
+index[20].field[3].name date
+index[20].field[4].name replyto
+index[20].field[5].name subject
+index[20].field[6].name to
+index[20].field[7].name body
+index[20].field[8].name bodymeta
+index[20].field[9].name mailid
+index[20].field[10].name attachmentcount
+index[20].field[11].name attachmentcontent
+index[20].field[12].name attachmenttypes
+index[21].name index14
+index[21].field[14]
+index[21].field[0].name bcc
+index[21].field[1].name cc
+index[21].field[2].name from
+index[21].field[3].name date
+index[21].field[4].name replyto
+index[21].field[5].name subject
+index[21].field[6].name to
+index[21].field[7].name body
+index[21].field[8].name bodymeta
+index[21].field[9].name mailid
+index[21].field[10].name attachmentcount
+index[21].field[11].name attachmentcontent
+index[21].field[12].name attachmenttypes
+index[21].field[13].name attachmentnames
+index[22].name index15
+index[22].field[15]
+index[22].field[0].name bcc
+index[22].field[1].name cc
+index[22].field[2].name from
+index[22].field[3].name date
+index[22].field[4].name replyto
+index[22].field[5].name subject
+index[22].field[6].name to
+index[22].field[7].name body
+index[22].field[8].name bodymeta
+index[22].field[9].name mailid
+index[22].field[10].name attachmentcount
+index[22].field[11].name attachmentcontent
+index[22].field[12].name attachmenttypes
+index[22].field[13].name attachmentnames
+index[22].field[14].name attachmentlanguages
+index[23].name index16
+index[23].field[15]
+index[23].field[0].name bcc
+index[23].field[1].name cc
+index[23].field[2].name from
+index[23].field[3].name date
+index[23].field[4].name replyto
+index[23].field[5].name subject
+index[23].field[6].name to
+index[23].field[7].name body
+index[23].field[8].name bodymeta
+index[23].field[9].name mailid
+index[23].field[10].name attachmentcount
+index[23].field[11].name attachmentcontent
+index[23].field[12].name attachmenttypes
+index[23].field[13].name attachmentnames
+index[23].field[14].name attachmentlanguages
+index[24].name index17
+index[24].field[15]
+index[24].field[0].name bcc
+index[24].field[1].name cc
+index[24].field[2].name from
+index[24].field[3].name date
+index[24].field[4].name replyto
+index[24].field[5].name subject
+index[24].field[6].name to
+index[24].field[7].name body
+index[24].field[8].name bodymeta
+index[24].field[9].name mailid
+index[24].field[10].name attachmentcount
+index[24].field[11].name attachmentcontent
+index[24].field[12].name attachmenttypes
+index[24].field[13].name attachmentnames
+index[24].field[14].name attachmentlanguages
+index[25].name date
+index[25].field[1]
+index[25].field[0].name date
diff --git a/streamingvisitors/src/tests/docsum/.gitignore b/streamingvisitors/src/tests/docsum/.gitignore
new file mode 100644
index 00000000000..9a697a94de8
--- /dev/null
+++ b/streamingvisitors/src/tests/docsum/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+docsum_test
+vsm_docsum_test_app
diff --git a/streamingvisitors/src/tests/docsum/CMakeLists.txt b/streamingvisitors/src/tests/docsum/CMakeLists.txt
new file mode 100644
index 00000000000..87c46409053
--- /dev/null
+++ b/streamingvisitors/src/tests/docsum/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vsm_docsum_test_app TEST
+    SOURCES
+    docsum.cpp
+    DEPENDS
+    streamingvisitors
+)
+vespa_add_test(NAME vsm_docsum_test_app COMMAND vsm_docsum_test_app)
diff --git a/streamingvisitors/src/tests/docsum/docsum.cpp b/streamingvisitors/src/tests/docsum/docsum.cpp
new file mode 100644
index 00000000000..475489d2f5a
--- /dev/null
+++ b/streamingvisitors/src/tests/docsum/docsum.cpp
@@ -0,0 +1,293 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/vsm/common/docsum.h>
+#include <vespa/vsm/vsm/flattendocsumwriter.h>
+#include <vespa/vsm/vsm/slimefieldwriter.h>
+
+using namespace document;
+
+namespace vsm {
+
+template <typename T>
+class Vector : public std::vector<T>
+{
+public:
+    Vector<T> & add(T v) { this->push_back(v); return *this; }
+};
+
+typedef Vector<std::string>  StringList;
+typedef Vector<std::pair<std::string, int32_t> > WeightedStringList;
+
+
+class TestDocument : public vsm::Document
+{
+private:
+    std::vector<FieldValueContainer> _fields;
+
+public:
+    TestDocument(const search::DocumentIdT & docId, size_t numFields) : vsm::Document(docId, numFields), _fields(numFields) {}
+    virtual bool setField(FieldIdT fId, document::FieldValue::UP fv) override {
+        if (fId < _fields.size()) {
+            _fields[fId].reset(fv.release());
+            return true;
+        }
+        return false;
+    }
+    virtual const document::FieldValue * getField(FieldIdT fId) const override {
+        if (fId < _fields.size()) {
+            return _fields[fId].get();
+        }
+        return NULL;
+    }
+};
+
+
+class DocsumTest : public vespalib::TestApp
+{
+private:
+    ArrayFieldValue createFieldValue(const StringList & fv);
+    WeightedSetFieldValue createFieldValue(const WeightedStringList & fv);
+
+    void assertFlattenDocsumWriter(const FieldValue & fv, const std::string & exp) {
+        FlattenDocsumWriter fdw;
+        assertFlattenDocsumWriter(fdw, fv, exp);
+    }
+    void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp);
+    void assertSlimeFieldWriter(const FieldValue & fv, const std::string & exp) {
+        SlimeFieldWriter sfw;
+        TEST_DO(assertSlimeFieldWriter(sfw, fv, exp));
+    }
+    void assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp);
+
+    void testFlattenDocsumWriter();
+    void testSlimeFieldWriter();
+    void requireThatSlimeFieldWriterHandlesMap();
+    void testDocSumCache();
+
+public:
+    int Main() override;
+};
+
+ArrayFieldValue
+DocsumTest::createFieldValue(const StringList & fv)
+{
+
+    static ArrayDataType type(*DataType::STRING);
+    ArrayFieldValue afv(type);
+    for (size_t i = 0; i < fv.size(); ++i) {
+        afv.add(StringFieldValue(fv[i]));
+    }
+    return afv;
+}
+
+WeightedSetFieldValue
+DocsumTest::createFieldValue(const WeightedStringList & fv)
+{
+    static WeightedSetDataType type(*DataType::STRING, false, false);
+    WeightedSetFieldValue wsfv(type);
+    for (size_t i = 0; i < fv.size(); ++i) {
+        wsfv.add(StringFieldValue(fv[i].first), fv[i].second);
+    }
+    return wsfv;
+}
+
+void
+DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp)
+{
+    FieldPath empty;
+    fv.iterateNested(empty.getFullRange(), fdw);
+    std::string actual(fdw.getResult().getBuffer(), fdw.getResult().getPos());
+    EXPECT_EQUAL(actual, exp);
+}
+
+void
+DocsumTest::assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp)
+{
+    sfw.convert(fv);
+
+    vespalib::Slime gotSlime;
+    vespalib::Memory serialized(sfw.out());
+    size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, gotSlime);
+    ASSERT_EQUAL(decodeRes, serialized.size);
+
+    vespalib::Slime expSlime;
+    size_t used = vespalib::slime::JsonFormat::decode(exp, expSlime);
+    EXPECT_TRUE(used > 0);
+    EXPECT_EQUAL(expSlime, gotSlime);
+}
+
+void
+DocsumTest::testFlattenDocsumWriter()
+{
+    { // basic tests
+        TEST_DO(assertFlattenDocsumWriter(StringFieldValue("foo bar"), "foo bar"));
+        TEST_DO(assertFlattenDocsumWriter(RawFieldValue("foo bar"), "foo bar"));
+        TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(true), "true"));
+        TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(false), "false"));
+        TEST_DO(assertFlattenDocsumWriter(LongFieldValue(123456789), "123456789"));
+        TEST_DO(assertFlattenDocsumWriter(createFieldValue(StringList().add("foo bar").add("baz").add(" qux ")),
+                                  "foo bar baz  qux "));
+    }
+    { // test mulitple invokations
+        FlattenDocsumWriter fdw("#");
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("foo"), "foo"));
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("bar"), "foo#bar"));
+        fdw.clear();
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("baz"), "baz"));
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("qux"), "baz qux"));
+    }
+    { // test resizing
+        FlattenDocsumWriter fdw("#");
+        EXPECT_EQUAL(fdw.getResult().getPos(), 0u);
+        EXPECT_EQUAL(fdw.getResult().getLength(), 32u);
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaabbbbccccddddeeeeffffgggghhhh"),
+                                          "aaaabbbbccccddddeeeeffffgggghhhh"));
+        EXPECT_EQUAL(fdw.getResult().getPos(), 32u);
+        EXPECT_EQUAL(fdw.getResult().getLength(), 32u);
+        TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaa"), "aaaabbbbccccddddeeeeffffgggghhhh#aaaa"));
+        EXPECT_EQUAL(fdw.getResult().getPos(), 37u);
+        EXPECT_TRUE(fdw.getResult().getLength() >= 37u);
+        fdw.clear();
+        EXPECT_EQUAL(fdw.getResult().getPos(), 0u);
+        EXPECT_TRUE(fdw.getResult().getLength() >= 37u);
+    }
+}
+
+void
+DocsumTest::testSlimeFieldWriter()
+{
+    { // basic types
+        assertSlimeFieldWriter(LongFieldValue(123456789), "123456789");
+        assertSlimeFieldWriter(BoolFieldValue(true), "true");
+        assertSlimeFieldWriter(BoolFieldValue(false), "false");
+        assertSlimeFieldWriter(DoubleFieldValue(12.34), "12.34");
+        assertSlimeFieldWriter(StringFieldValue("foo bar"), "\"foo bar\"");
+    }
+    { // collection field values
+        assertSlimeFieldWriter(createFieldValue(StringList().add("foo").add("bar").add("baz")),
+                               "[\"foo\",\"bar\",\"baz\"]");
+        assertSlimeFieldWriter(createFieldValue(WeightedStringList().add(std::make_pair("bar", 20)).
+                                                                     add(std::make_pair("baz", 30)).
+                                                                     add(std::make_pair("foo", 10))),
+                               "[{item:\"bar\",weight:20},{item:\"baz\",weight:30},{item:\"foo\",weight:10}]");
+    }
+    { // struct field value
+        StructDataType subType("substruct");
+        Field fd("d", 0, *DataType::STRING);
+        Field fe("e", 1, *DataType::STRING);
+        subType.addField(fd);
+        subType.addField(fe);
+        StructFieldValue subValue(subType);
+        subValue.setValue(fd, StringFieldValue("baz"));
+        subValue.setValue(fe, StringFieldValue("qux"));
+
+        StructDataType type("struct");
+        Field fa("a", 0, *DataType::STRING);
+        Field fb("b", 1, *DataType::STRING);
+        Field fc("c", 2, subType);
+        type.addField(fa);
+        type.addField(fb);
+        type.addField(fc);
+        StructFieldValue value(type);
+        value.setValue(fa, StringFieldValue("foo"));
+        value.setValue(fb, StringFieldValue("bar"));
+        value.setValue(fc, subValue);
+
+
+        { // select a subset and then all
+            SlimeFieldWriter sfw;
+            DocsumFieldSpec::FieldIdentifierVector fields;
+            {
+                FieldPath path;
+                type.buildFieldPath(path, "a");
+                fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
+            }
+            {
+                FieldPath path;
+                type.buildFieldPath(path, "c.e");
+                fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
+            }
+            sfw.setInputFields(fields);
+            TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}"));
+            sfw.clear();
+            TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"b\":\"bar\",\"c\":{\"d\":\"baz\",\"e\":\"qux\"}}"));
+        }
+
+    { // multiple invocations
+        SlimeFieldWriter sfw;
+        TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("foo"), "\"foo\""));
+        sfw.clear();
+        TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("bar"), "\"bar\""));
+        sfw.clear();
+        TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("baz"), "\"baz\""));
+    }
+
+    }
+}
+
+void
+DocsumTest::requireThatSlimeFieldWriterHandlesMap()
+{
+    { // map<string, string>
+        MapDataType mapType(*DataType::STRING, *DataType::STRING);
+        MapFieldValue mapfv(mapType);
+        EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), StringFieldValue("v1")));
+        EXPECT_TRUE(mapfv.put(StringFieldValue("k2"), StringFieldValue("v2")));
+        assertSlimeFieldWriter(mapfv, "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]");
+    }
+    { // map<string, struct>
+        StructDataType structType("struct");
+        Field fa("a", 0, *DataType::STRING);
+        Field fb("b", 1, *DataType::STRING);
+        structType.addField(fa);
+        structType.addField(fb);
+        StructFieldValue structValue(structType);
+        structValue.setValue(fa, StringFieldValue("foo"));
+        structValue.setValue(fb, StringFieldValue("bar"));
+        MapDataType mapType(*DataType::STRING, structType);
+        MapFieldValue mapfv(mapType);
+        EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), structValue));
+        { // select a subset and then all
+            SlimeFieldWriter sfw;
+            DocsumFieldSpec::FieldIdentifierVector fields;
+            {
+                FieldPath path;
+                mapType.buildFieldPath(path, "value.b");
+                fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
+            }
+            sfw.setInputFields(fields);
+            TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]"));
+            {
+                FieldPath path;
+                mapType.buildFieldPath(path, "{k1}.a");
+                fields[0] = DocsumFieldSpec::FieldIdentifier(0, std::move(path));
+            }
+            sfw.clear();
+            sfw.setInputFields(fields);
+            TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\"}}]"));
+            sfw.clear(); // all fields implicit
+            TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\",\"b\":\"bar\"}}]"));
+        }
+    }
+}
+
+int
+DocsumTest::Main()
+{
+    TEST_INIT("docsum_test");
+
+    TEST_DO(testFlattenDocsumWriter());
+    TEST_DO(testSlimeFieldWriter());
+    TEST_DO(requireThatSlimeFieldWriterHandlesMap());
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(vsm::DocsumTest);
+
diff --git a/streamingvisitors/src/tests/document/.gitignore b/streamingvisitors/src/tests/document/.gitignore
new file mode 100644
index 00000000000..d47781eff63
--- /dev/null
+++ b/streamingvisitors/src/tests/document/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+document_test
+vsm_document_test_app
diff --git a/streamingvisitors/src/tests/document/CMakeLists.txt b/streamingvisitors/src/tests/document/CMakeLists.txt
new file mode 100644
index 00000000000..5ea12dc5e2d
--- /dev/null
+++ b/streamingvisitors/src/tests/document/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vsm_document_test_app TEST
+    SOURCES
+    document.cpp
+    DEPENDS
+    streamingvisitors
+)
+vespa_add_test(NAME vsm_document_test_app COMMAND vsm_document_test_app)
diff --git a/streamingvisitors/src/tests/document/document.cpp b/streamingvisitors/src/tests/document/document.cpp
new file mode 100644
index 00000000000..1e97d232a64
--- /dev/null
+++ b/streamingvisitors/src/tests/document/document.cpp
@@ -0,0 +1,129 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vsm/common/storagedocument.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+using namespace document;
+
+namespace vsm {
+
+class DocumentTest : public vespalib::TestApp
+{
+private:
+    void testStorageDocument();
+    void testStringFieldIdTMap();
+public:
+    int Main() override;
+};
+
+void
+DocumentTest::testStorageDocument()
+{
+    DocumentType dt("testdoc", 0);
+
+    Field fa("a", 0, *DataType::STRING);
+    Field fb("b", 1, *DataType::STRING);
+    dt.addField(fa);
+    dt.addField(fb);
+
+    document::Document::UP doc(new document::Document(dt, DocumentId()));
+    doc->setValue(fa, StringFieldValue("foo"));
+    doc->setValue(fb, StringFieldValue("bar"));
+
+    SharedFieldPathMap fpmap(new FieldPathMapT());
+    fpmap->emplace_back();
+    dt.buildFieldPath(fpmap->back(),"a");
+    fpmap->emplace_back();
+    dt.buildFieldPath(fpmap->back(), "b");
+    fpmap->emplace_back();
+    ASSERT_TRUE((*fpmap)[0].size() == 1);
+    ASSERT_TRUE((*fpmap)[1].size() == 1);
+    ASSERT_TRUE((*fpmap)[2].size() == 0);
+
+    StorageDocument sdoc(std::move(doc), fpmap, 3);
+    ASSERT_TRUE(sdoc.valid());
+
+    EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString());
+    EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
+    EXPECT_TRUE(sdoc.getField(2) == nullptr);
+    // test caching
+    EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString());
+    EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
+    EXPECT_TRUE(sdoc.getField(2) == nullptr);
+
+    // set new values
+    EXPECT_TRUE(sdoc.setField(0, FieldValue::UP(new StringFieldValue("baz"))));
+    EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
+    EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
+    EXPECT_TRUE(sdoc.getField(2) == nullptr);
+    EXPECT_TRUE(sdoc.setField(1, FieldValue::UP(new StringFieldValue("qux"))));
+    EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
+    EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString());
+    EXPECT_TRUE(sdoc.getField(2) == nullptr);
+    EXPECT_TRUE(sdoc.setField(2, FieldValue::UP(new StringFieldValue("quux"))));
+    EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
+    EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString());
+    EXPECT_EQUAL(std::string("quux"), sdoc.getField(2)->getAsString());
+
+    EXPECT_TRUE(!sdoc.setField(3, FieldValue::UP(new StringFieldValue("thud"))));
+
+    SharedFieldPathMap fim;
+    StorageDocument s2(std::make_unique<document::Document>(), fim, 0);
+    EXPECT_EQUAL(IdString().toString(), s2.docDoc().getId().toString());
+}
+
+void DocumentTest::testStringFieldIdTMap()
+{
+    StringFieldIdTMap m;
+    EXPECT_EQUAL(0u, m.highestFieldNo());
+    EXPECT_TRUE(StringFieldIdTMap::npos == m.fieldNo("unknown"));
+    m.add("f1");
+    EXPECT_EQUAL(0u, m.fieldNo("f1"));
+    EXPECT_EQUAL(1u, m.highestFieldNo());
+    m.add("f1");
+    EXPECT_EQUAL(0u, m.fieldNo("f1"));
+    EXPECT_EQUAL(1u, m.highestFieldNo());
+    m.add("f2");
+    EXPECT_EQUAL(1u, m.fieldNo("f2"));
+    EXPECT_EQUAL(2u, m.highestFieldNo());
+    m.add("f3", 7);
+    EXPECT_EQUAL(7u, m.fieldNo("f3"));
+    EXPECT_EQUAL(8u, m.highestFieldNo());
+    m.add("f3");
+    EXPECT_EQUAL(7u, m.fieldNo("f3"));
+    EXPECT_EQUAL(8u, m.highestFieldNo());
+    m.add("f2", 13);
+    EXPECT_EQUAL(13u, m.fieldNo("f2"));
+    EXPECT_EQUAL(14u, m.highestFieldNo());
+    m.add("f4");
+    EXPECT_EQUAL(3u, m.fieldNo("f4"));
+    EXPECT_EQUAL(14u, m.highestFieldNo());
+    {
+        vespalib::asciistream os;
+        StringFieldIdTMap t;
+        t.add("b");
+        t.add("a");
+        os << t;
+        EXPECT_EQUAL(vespalib::string("a = 1\nb = 0\n"), os.str());
+    }
+    
+}
+
+int
+DocumentTest::Main()
+{
+    TEST_INIT("document_test");
+
+    testStorageDocument();
+    testStringFieldIdTMap();
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(vsm::DocumentTest);
+
diff --git a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
index f25ab348265..dbec820a462 100644
--- a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
+++ b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_hitcollector_test_app TEST
     SOURCES
     hitcollector_test.cpp
     DEPENDS
-    streamingvisitors_searchvisitor
+    streamingvisitors
 )
 vespa_add_test(NAME streamingvisitors_hitcollector_test_app COMMAND streamingvisitors_hitcollector_test_app)
diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
index ef93d551912..5cc2977b3c3 100644
--- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
+++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST
     SOURCES
     matching_elements_filler_test.cpp
     DEPENDS
-    streamingvisitors_searchvisitor
+    streamingvisitors
     GTest::GTest
 )
 vespa_add_test(NAME streamingvisitors_matching_elements_filler_test_app COMMAND streamingvisitors_matching_elements_filler_test_app)
diff --git a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
index 7cae60e6a11..e0131d0c6cc 100644
--- a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
+++ b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_querywrapper_test_app TEST
     SOURCES
     querywrapper_test.cpp
     DEPENDS
-    streamingvisitors_searchvisitor
+    streamingvisitors
 )
 vespa_add_test(NAME streamingvisitors_querywrapper_test_app COMMAND streamingvisitors_querywrapper_test_app)
diff --git a/streamingvisitors/src/tests/searcher/.gitignore b/streamingvisitors/src/tests/searcher/.gitignore
new file mode 100644
index 00000000000..52a56dff405
--- /dev/null
+++ b/streamingvisitors/src/tests/searcher/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+searcher_test
+vsm_searcher_test_app
diff --git a/streamingvisitors/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt
new file mode 100644
index 00000000000..2277f5ef55f
--- /dev/null
+++ b/streamingvisitors/src/tests/searcher/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vsm_searcher_test_app TEST
+    SOURCES
+    searcher_test.cpp
+    DEPENDS
+    streamingvisitors
+)
+vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app)
diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp
new file mode 100644
index 00000000000..34fa66eaa90
--- /dev/null
+++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp
@@ -0,0 +1,864 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/vsm/searcher/fieldsearcher.h>
+#include <vespa/vsm/searcher/floatfieldsearcher.h>
+#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h>
+#include <vespa/vsm/searcher/intfieldsearcher.h>
+#include <vespa/vsm/searcher/boolfieldsearcher.h>
+#include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h>
+#include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h>
+#include <vespa/vsm/searcher/utf8substringsearcher.h>
+#include <vespa/vsm/searcher/utf8substringsnippetmodifier.h>
+#include <vespa/vsm/searcher/utf8suffixstringfieldsearcher.h>
+#include <vespa/vsm/vsm/snippetmodifier.h>
+#include <vespa/searchlib/query/streaming/queryterm.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+
+using namespace document;
+using search::streaming::HitList;
+using search::streaming::QueryNodeResultFactory;
+using search::streaming::QueryTerm;
+using search::streaming::QueryTermList;
+using TermType = QueryTerm::Type;
+using namespace vsm;
+
+template <typename T>
+class Vector : public std::vector<T>
+{
+public:
+    Vector() : std::vector<T>() {}
+    Vector<T> & add(T v) { this->push_back(v); return *this; }
+};
+
+typedef Vector<size_t>       Hits;
+typedef Vector<std::string>  StringList;
+typedef Vector<Hits>         HitsList;
+typedef Vector<bool>         BoolList;
+typedef Vector<int64_t>      LongList;
+typedef Vector<float>        FloatList;
+typedef QueryTerm::FieldInfo QTFieldInfo;
+typedef Vector<QTFieldInfo>  FieldInfoList;
+
+class String
+{
+private:
+    const std::string & _str;
+public:
+    String(const std::string & str) : _str(str) {}
+    bool operator==(const String & rhs) const {
+        return _str == rhs._str;
+    }
+};
+
+class Query
+{
+private:
+    void setupQuery(const StringList & terms) {
+        for (size_t i = 0; i < terms.size(); ++i) {
+            ParsedQueryTerm pqt = parseQueryTerm(terms[i]);
+            ParsedTerm pt = parseTerm(pqt.second);
+            qtv.push_back(std::make_unique<QueryTerm>(eqnr.create(), pt.first, pqt.first.empty() ? "index" : pqt.first, pt.second));
+        }
+        for (size_t i = 0; i < qtv.size(); ++i) {
+            qtl.push_back(qtv[i].get());
+        }
+    }
+public:
+    typedef std::pair<std::string, std::string> ParsedQueryTerm;
+    typedef std::pair<std::string, TermType> ParsedTerm;
+    QueryNodeResultFactory   eqnr;
+    std::vector<QueryTerm::UP> qtv;
+    QueryTermList          qtl;
+    Query(const StringList & terms);
+    ~Query();
+    static ParsedQueryTerm parseQueryTerm(const std::string & queryTerm) {
+        size_t i = queryTerm.find(':');
+        if (i != std::string::npos) {
+            return ParsedQueryTerm(queryTerm.substr(0, i), queryTerm.substr(i + 1));
+        }
+        return ParsedQueryTerm(std::string(), queryTerm);
+    }
+    static ParsedTerm parseTerm(const std::string & term) {
+        if (term[0] == '*' && term[term.size() - 1] == '*') {
+            return std::make_pair(term.substr(1, term.size() - 2), TermType::SUBSTRINGTERM);
+        } else if (term[0] == '*') {
+            return std::make_pair(term.substr(1, term.size() - 1), TermType::SUFFIXTERM);
+        } else if (term[term.size() - 1] == '*') {
+            return std::make_pair(term.substr(0, term.size() - 1), TermType::PREFIXTERM);
+        } else {
+            return std::make_pair(term, TermType::WORD);
+        }
+    }
+};
+
+Query::Query(const StringList & terms) : eqnr(), qtv(), qtl() {
+    setupQuery(terms);
+}
+Query::~Query() = default;
+
+struct SnippetModifierSetup
+{
+    Query                            query;
+    UTF8SubstringSnippetModifier::SP searcher;
+    SharedSearcherBuf                buf;
+    SnippetModifier                  modifier;
+    explicit SnippetModifierSetup(const StringList & terms);
+    ~SnippetModifierSetup();
+};
+
+SnippetModifierSetup::SnippetModifierSetup(const StringList & terms)
+    : query(terms),
+      searcher(new UTF8SubstringSnippetModifier()),
+      buf(new SearcherBuf(8)),
+      modifier(searcher)
+{
+    searcher->prepare(query.qtl, buf);
+}
+SnippetModifierSetup::~SnippetModifierSetup() = default;
+
+// helper functions
+ArrayFieldValue getFieldValue(const StringList &fv);
+ArrayFieldValue getFieldValue(const LongList &fv);
+ArrayFieldValue getFieldValue(const FloatList &fv);
+
+bool assertMatchTermSuffix(const std::string &term, const std::string &word);
+void assertSnippetModifier(const StringList &query, const std::string &fv, const std::string &exp);
+void assertSnippetModifier(SnippetModifierSetup &setup, const FieldValue &fv, const std::string &exp);
+void assertQueryTerms(const SnippetModifierManager &man, FieldIdT fId, const StringList &terms);
+void assertNumeric(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const BoolList &exp);
+std::vector<QueryTerm::UP> performSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv);
+void assertSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const HitsList &exp);
+bool assertCountWords(size_t numWords, const std::string &field);
+bool assertFieldInfo(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const FieldInfoList &exp);
+
+void assertString(StrChrFieldSearcher &fs, const StringList &query, const std::string &field, const HitsList &exp) {
+    assertSearch(fs, query, StringFieldValue(field), exp);
+}
+
+void assertString(StrChrFieldSearcher &fs, const StringList &query, const StringList &field, const HitsList &exp) {
+    assertSearch(fs, query, getFieldValue(field), exp);
+}
+
+void assertString(StrChrFieldSearcher &fs, const std::string &term, const std::string &field, const Hits &exp) {
+    assertString(fs, StringList().add(term), field, HitsList().add(exp));
+}
+void assertString(StrChrFieldSearcher &fs, const std::string &term, const StringList &field, const Hits &exp) {
+    assertString(fs, StringList().add(term), field, HitsList().add(exp));
+}
+
+void assertInt(IntFieldSearcher & fs, const StringList &query, int64_t field, const BoolList &exp) {
+    assertNumeric(fs, query, LongFieldValue(field), exp);
+}
+
+void assertInt(IntFieldSearcher & fs, const std::string &term, int64_t field, bool exp) {
+    assertInt(fs, StringList().add(term), field, BoolList().add(exp));
+}
+
+void assertBool(BoolFieldSearcher & fs, const StringList &query, bool field, const BoolList &exp) {
+    assertNumeric(fs, query, BoolFieldValue(field), exp);
+}
+void assertBool(BoolFieldSearcher & fs, const std::string &term, bool field, bool exp) {
+    assertBool(fs, StringList().add(term), field, BoolList().add(exp));
+}
+
+void assertInt(IntFieldSearcher & fs, const StringList &query, const LongList &field, const HitsList &exp) {
+    assertSearch(fs, query, getFieldValue(field), exp);
+}
+
+void assertInt(IntFieldSearcher & fs, const std::string &term, const LongList &field, const Hits &exp) {
+    assertInt(fs, StringList().add(term), field, HitsList().add(exp));
+}
+
+void assertFloat(FloatFieldSearcher & fs, const StringList &query, float field, const BoolList &exp) {
+    assertNumeric(fs, query, FloatFieldValue(field), exp);
+}
+
+void assertFloat(FloatFieldSearcher & fs, const std::string &term, float field, bool exp) {
+    assertFloat(fs, StringList().add(term), field, BoolList().add(exp));
+}
+
+void assertFloat(FloatFieldSearcher & fs, const StringList &query, const FloatList &field, const HitsList &exp) {
+    assertSearch(fs, query, getFieldValue(field), exp);
+}
+
+void assertFloat(FloatFieldSearcher & fs, const std::string &term, const FloatList &field, const Hits &exp) {
+    assertFloat(fs, StringList().add(term), field, HitsList().add(exp));
+}
+
+bool
+assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const std::string &fv, const FieldInfoList &exp) {
+    return assertFieldInfo(fs, query, StringFieldValue(fv), exp);
+}
+
+bool
+assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const StringList &fv, const FieldInfoList &exp) {
+    return assertFieldInfo(fs, query, getFieldValue(fv), exp);
+}
+bool
+assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const StringList &fv, const QTFieldInfo &exp) {
+    return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+bool
+assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const std::string &fv, const QTFieldInfo &exp) {
+    return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, int64_t fv, const FieldInfoList &exp) {
+    assertFieldInfo(fs, query, LongFieldValue(fv), exp);
+}
+
+void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, const LongList &fv, const FieldInfoList &exp) {
+    assertFieldInfo(fs, query, getFieldValue(fv), exp);
+}
+
+void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, int64_t fv, const QTFieldInfo &exp) {
+    assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, const LongList &fv, const QTFieldInfo &exp) {
+    assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+void assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, float fv, const FieldInfoList &exp) {
+    assertFieldInfo(fs, query, FloatFieldValue(fv), exp);
+}
+
+void
+assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, const FloatList &fv, const FieldInfoList &exp) {
+    assertFieldInfo(fs, query, getFieldValue(fv), exp);
+}
+
+/** float field searcher **/
+void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, float fv, const QTFieldInfo &exp) {
+    assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, const FloatList &fv, const QTFieldInfo &exp) {
+    assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
+}
+
+
+/** snippet modifer searcher **/
+void assertSnippetModifier(const std::string &term, const std::string &fv, const std::string &exp) {
+    assertSnippetModifier(StringList().add(term), fv, exp);
+}
+
+
+ArrayFieldValue
+getFieldValue(const StringList & fv)
+{
+
+    static ArrayDataType type(*DataType::STRING);
+    ArrayFieldValue afv(type);
+    for (size_t i = 0; i < fv.size(); ++i) {
+        afv.add(StringFieldValue(fv[i]));
+    }
+    return afv;
+}
+
+ArrayFieldValue
+getFieldValue(const LongList & fv)
+{
+    static ArrayDataType type(*DataType::LONG);
+    ArrayFieldValue afv(type);
+    for (size_t i = 0; i < fv.size(); ++i) {
+        afv.add(LongFieldValue(fv[i]));
+    }
+    return afv;
+}
+
+ArrayFieldValue
+getFieldValue(const FloatList & fv)
+{
+    static ArrayDataType type(*DataType::FLOAT);
+    ArrayFieldValue afv(type);
+    for (size_t i = 0; i < fv.size(); ++i) {
+        afv.add(FloatFieldValue(fv[i]));
+    }
+    return afv;
+}
+
+bool
+assertMatchTermSuffix(const std::string & term, const std::string & word)
+{
+    QueryNodeResultFactory eqnr;
+    QueryTerm qa(eqnr.create(), term, "index", TermType::WORD);
+    QueryTerm qb(eqnr.create(), word, "index", TermType::WORD);
+    const ucs4_t * a;
+    size_t alen = qa.term(a);
+    const ucs4_t * b;
+    size_t blen = qb.term(b);
+    return UTF8StringFieldSearcherBase::matchTermSuffix(a, alen, b, blen);
+}
+
+void
+assertNumeric(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const BoolList & exp)
+{
+    HitsList hl;
+    for (size_t i = 0; i < exp.size(); ++i) {
+        hl.push_back(exp[i] ? Hits().add(0) : Hits());
+    }
+    assertSearch(fs, query, fv, hl);
+}
+
+std::vector<QueryTerm::UP>
+performSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv)
+{
+    Query q(query);
+
+    // prepare field searcher
+    SharedSearcherBuf ssb = SharedSearcherBuf(new SearcherBuf());
+    fs.prepare(q.qtl, ssb);
+
+    // setup document
+    SharedFieldPathMap sfim(new FieldPathMapT());
+    sfim->push_back(FieldPath());
+    StorageDocument doc(std::make_unique<document::Document>(), sfim, 1);
+    doc.setField(0, document::FieldValue::UP(fv.clone()));
+
+    fs.search(doc);
+    return std::move(q.qtv);
+}
+
+void
+assertSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const HitsList & exp)
+{
+    auto qtv = performSearch(fs, query, fv);
+    EXPECT_EQUAL(qtv.size(), exp.size());
+    ASSERT_TRUE(qtv.size() == exp.size());
+    for (size_t i = 0; i < qtv.size(); ++i) {
+        const HitList & hl = qtv[i]->getHitList();
+        EXPECT_EQUAL(hl.size(), exp[i].size());
+        ASSERT_TRUE(hl.size() == exp[i].size());
+        for (size_t j = 0; j < hl.size(); ++j) {
+            EXPECT_EQUAL((size_t)hl[j].pos(), exp[i][j]);
+        }
+    }
+}
+
+bool
+assertFieldInfo(FieldSearcher & fs, const StringList & query,
+                              const FieldValue & fv, const FieldInfoList & exp)
+{
+    auto qtv = performSearch(fs, query, fv);
+    if (!EXPECT_EQUAL(qtv.size(), exp.size())) return false;
+    bool retval = true;
+    for (size_t i = 0; i < qtv.size(); ++i) {
+        if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitOffset(), exp[i].getHitOffset())) retval = false;
+        if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitCount(), exp[i].getHitCount())) retval = false;
+        if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getFieldLength(), exp[i].getFieldLength())) retval = false;
+    }
+    return retval;
+}
+
+void
+assertSnippetModifier(const StringList & query, const std::string & fv, const std::string & exp)
+{
+    UTF8SubstringSnippetModifier mod;
+    performSearch(mod, query, StringFieldValue(fv));
+    EXPECT_EQUAL(mod.getModifiedBuf().getPos(), exp.size());
+    std::string actual(mod.getModifiedBuf().getBuffer(), mod.getModifiedBuf().getPos());
+    EXPECT_EQUAL(actual.size(), exp.size());
+    EXPECT_EQUAL(actual, exp);
+}
+
+void assertSnippetModifier(SnippetModifierSetup & setup, const FieldValue & fv, const std::string & exp)
+{
+    FieldValue::UP mfv = setup.modifier.modify(fv);
+    const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(*mfv.get());
+    const std::string & actual = lfv.getValue();
+    EXPECT_EQUAL(actual.size(), exp.size());
+    EXPECT_EQUAL(actual, exp);
+}
+
+void assertQueryTerms(const SnippetModifierManager & man, FieldIdT fId, const StringList & terms)
+{
+    if (terms.size() == 0) {
+        ASSERT_TRUE(man.getModifiers().getModifier(fId) == NULL);
+        return;
+    }
+    ASSERT_TRUE(man.getModifiers().getModifier(fId) != NULL);
+    UTF8SubstringSnippetModifier * searcher =
+        (static_cast<SnippetModifier *>(man.getModifiers().getModifier(fId)))->getSearcher().get();
+    EXPECT_EQUAL(searcher->getQueryTerms().size(), terms.size());
+    ASSERT_TRUE(searcher->getQueryTerms().size() == terms.size());
+    for (size_t i = 0; i < terms.size(); ++i) {
+        EXPECT_EQUAL(std::string(searcher->getQueryTerms()[i]->getTerm()), terms[i]);
+    }
+}
+
+bool assertCountWords(size_t numWords, const std::string & field)
+{
+    FieldRef ref(field.c_str(), field.size());
+    return EXPECT_EQUAL(numWords, FieldSearcher::countWords(ref));
+}
+
+bool
+testStringFieldInfo(StrChrFieldSearcher & fs)
+{
+    assertString(fs,    "foo", StringList().add("foo bar baz").add("foo bar").add("baz foo"), Hits().add(0).add(3).add(6));
+    assertString(fs,    StringList().add("foo").add("bar"), StringList().add("foo bar baz").add("foo bar").add("baz foo"),
+                 HitsList().add(Hits().add(0).add(3).add(6)).add(Hits().add(1).add(4)));
+
+    bool retval = true;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo", QTFieldInfo(0, 1, 1)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo", QTFieldInfo(0, 0, 1)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "baz", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "qux", "foo bar baz", QTFieldInfo(0, 0, 3)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo foo foo", QTFieldInfo(0, 3, 3)))) retval = false;
+    // query term size > last term size
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "runner", "Road Runner Disco", QTFieldInfo(0, 1, 3)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("roadrun").add("runner"), "Road Runner Disco",
+                                     FieldInfoList().add(QTFieldInfo(0, 0, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false;
+    // multiple terms
+    if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", StringList().add("foo bar baz").add("foo bar"),
+                                     QTFieldInfo(0, 2, 5)))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), "foo bar baz",
+                                     FieldInfoList().add(QTFieldInfo(0, 1, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false;
+    if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), StringList().add("foo bar baz").add("foo bar"),
+                                     FieldInfoList().add(QTFieldInfo(0, 2, 5)).add(QTFieldInfo(0, 1, 5))))) retval = false;
+    return retval;
+}
+bool
+testStrChrFieldSearcher(StrChrFieldSearcher & fs)
+{
+    std::string field = "operators and operator overloading with utf8 char oe = \xc3\x98";
+    assertString(fs, "oper",  field, Hits());
+    assertString(fs, "tor",   field, Hits());
+    assertString(fs, "oper*", field, Hits().add(0).add(2));
+    assertString(fs, "and",   field, Hits().add(1));
+
+    assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits()).add(Hits()));
+    assertString(fs, StringList().add("and").add("overloading"), field, HitsList().add(Hits().add(1)).add(Hits().add(3)));
+
+    fs.setMatchType(FieldSearcher::PREFIX);
+    assertString(fs, "oper",  field, Hits().add(0).add(2));
+    assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits().add(0).add(2)).add(Hits()));
+
+    fs.setMatchType(FieldSearcher::REGULAR);
+    if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false;
+
+    { // test handling of several underscores
+        StringList query = StringList().add("foo").add("bar");
+        HitsList exp = HitsList().add(Hits().add(0)).add(Hits().add(1));
+        assertString(fs, query, "foo_bar", exp);
+        assertString(fs, query, "foo__bar", exp);
+        assertString(fs, query, "foo___bar", exp);
+        assertString(fs, query, "foo________bar", exp);
+        assertString(fs, query, "foo____________________bar", exp);
+        assertString(fs, query, "________________________________________foo________________________________________bar________________________________________", exp);
+        query = StringList().add("foo").add("thisisaveryveryverylongword");
+        assertString(fs, query, "foo____________________thisisaveryveryverylongword", exp);
+
+        assertString(fs, "bar", "foo                    bar", Hits().add(1));
+        assertString(fs, "bar", "foo____________________bar", Hits().add(1));
+        assertString(fs, "bar", "foo____________________thisisaveryveryverylongword____________________bar", Hits().add(2));
+    }
+    return true;
+}
+
+    TEST("verify correct term parsing") {
+        ASSERT_TRUE(Query::parseQueryTerm("index:term").first == "index");
+        ASSERT_TRUE(Query::parseQueryTerm("index:term").second == "term");
+        ASSERT_TRUE(Query::parseQueryTerm("term").first == "");
+        ASSERT_TRUE(Query::parseQueryTerm("term").second == "term");
+        ASSERT_TRUE(Query::parseTerm("*substr*").first == "substr");
+        ASSERT_TRUE(Query::parseTerm("*substr*").second == TermType::SUBSTRINGTERM);
+        ASSERT_TRUE(Query::parseTerm("*suffix").first == "suffix");
+        ASSERT_TRUE(Query::parseTerm("*suffix").second == TermType::SUFFIXTERM);
+        ASSERT_TRUE(Query::parseTerm("prefix*").first == "prefix");
+        ASSERT_TRUE(Query::parseTerm("prefix*").second == TermType::PREFIXTERM);
+        ASSERT_TRUE(Query::parseTerm("term").first == "term");
+        ASSERT_TRUE(Query::parseTerm("term").second == TermType::WORD);
+    }
+
+    TEST("suffix matching") {
+        EXPECT_EQUAL(assertMatchTermSuffix("a",      "vespa"), true);
+        EXPECT_EQUAL(assertMatchTermSuffix("spa",    "vespa"), true);
+        EXPECT_EQUAL(assertMatchTermSuffix("vespa",  "vespa"), true);
+        EXPECT_EQUAL(assertMatchTermSuffix("vvespa", "vespa"), false);
+        EXPECT_EQUAL(assertMatchTermSuffix("fspa",   "vespa"), false);
+        EXPECT_EQUAL(assertMatchTermSuffix("v",      "vespa"), false);
+    }
+
+TEST("Test basic strchrfield searchers") {
+    {
+        UTF8StrChrFieldSearcher fs(0);
+        EXPECT_TRUE(testStrChrFieldSearcher(fs));
+    }
+    {
+        FUTF8StrChrFieldSearcher fs(0);
+        EXPECT_TRUE(testStrChrFieldSearcher(fs));
+    }
+}
+
+bool
+testUTF8SubStringFieldSearcher(StrChrFieldSearcher & fs)
+{
+    std::string field = "operators and operator overloading";
+    assertString(fs, "rsand", field, Hits());
+    assertString(fs, "ove",   field, Hits().add(3));
+    assertString(fs, "ing",   field, Hits().add(3));
+    assertString(fs, "era",   field, Hits().add(0).add(2));
+    assertString(fs, "a",     field, Hits().add(0).add(1).add(2).add(3));
+
+    assertString(fs, StringList().add("dn").add("gn"), field, HitsList().add(Hits()).add(Hits()));
+    assertString(fs, StringList().add("ato").add("load"), field, HitsList().add(Hits().add(0).add(2)).add(Hits().add(3)));
+
+    assertString(fs, StringList().add("aa").add("ab"), "aaaab",
+                 HitsList().add(Hits().add(0).add(0).add(0)).add(Hits().add(0)));
+
+    if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false;
+    return true;
+}
+
+TEST("utf8 substring search") {
+    {
+        UTF8SubStringFieldSearcher fs(0);
+        EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
+        assertString(fs, "aa", "aaaa", Hits().add(0).add(0));
+    }
+    {
+        UTF8SubStringFieldSearcher fs(0);
+        EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
+        assertString(fs, "abc", "abc bcd abc", Hits().add(0).add(2));
+        fs.maxFieldLength(4);
+        assertString(fs, "abc", "abc bcd abc", Hits().add(0));
+    }
+    {
+        UTF8SubstringSnippetModifier fs(0);
+        EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
+        // we don't have 1 term optimization
+        assertString(fs, "aa", "aaaa", Hits().add(0).add(0).add(0));
+    }
+}
+
+TEST("utf8 substring search with empty term")
+{
+    UTF8SubStringFieldSearcher fs(0);
+    EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
+    assertString(fs, "", "abc", Hits());
+    assertFieldInfo(fs, "", "abc", QTFieldInfo().setFieldLength(0));
+}
+
+TEST("utf8 suffix search") {
+    UTF8SuffixStringFieldSearcher fs(0);
+    std::string field = "operators and operator overloading";
+    assertString(fs, "rsand", field, Hits());
+    assertString(fs, "tor",   field, Hits().add(2));
+    assertString(fs, "tors",  field, Hits().add(0));
+
+    assertString(fs, StringList().add("an").add("din"), field, HitsList().add(Hits()).add(Hits()));
+    assertString(fs, StringList().add("nd").add("g"), field, HitsList().add(Hits().add(1)).add(Hits().add(3)));
+
+    EXPECT_TRUE(testStringFieldInfo(fs));
+}
+
+TEST("utf8 exact match") {
+    UTF8ExactStringFieldSearcher fs(0);
+    // regular
+    TEST_DO(assertString(fs, "vespa", "vespa", Hits().add(0)));
+    TEST_DO(assertString(fs, "vespar", "vespa", Hits()));
+    TEST_DO(assertString(fs, "vespa", "vespar", Hits()));
+    TEST_DO(assertString(fs, "vespa", "vespa vespa", Hits()));
+    TEST_DO(assertString(fs, "vesp",  "vespa", Hits()));
+    TEST_DO(assertString(fs, "vesp*",  "vespa", Hits().add(0)));
+    TEST_DO(assertString(fs, "hutte",  "hutte", Hits().add(0)));
+    TEST_DO(assertString(fs, "hütte",  "hütte", Hits().add(0)));
+    TEST_DO(assertString(fs, "hutte",  "hütte", Hits()));
+    TEST_DO(assertString(fs, "hütte",  "hutte", Hits()));
+    TEST_DO(assertString(fs, "hütter", "hütte", Hits()));
+    TEST_DO(assertString(fs, "hütte",  "hütter", Hits()));
+}
+
+TEST("utf8 flexible searcher"){
+    UTF8FlexibleStringFieldSearcher fs(0);
+    // regular
+    assertString(fs, "vespa", "vespa", Hits().add(0));
+    assertString(fs, "vesp",  "vespa", Hits());
+    assertString(fs, "esp",   "vespa", Hits());
+    assertString(fs, "espa",  "vespa", Hits());
+
+    // prefix
+    assertString(fs, "vesp*",  "vespa", Hits().add(0));
+    fs.setMatchType(FieldSearcher::PREFIX);
+    assertString(fs, "vesp",   "vespa", Hits().add(0));
+
+    // substring
+    fs.setMatchType(FieldSearcher::REGULAR);
+    assertString(fs, "*esp*",  "vespa", Hits().add(0));
+    fs.setMatchType(FieldSearcher::SUBSTRING);
+    assertString(fs, "esp",  "vespa", Hits().add(0));
+
+    // suffix
+    fs.setMatchType(FieldSearcher::REGULAR);
+    assertString(fs, "*espa",  "vespa", Hits().add(0));
+    fs.setMatchType(FieldSearcher::SUFFIX);
+    assertString(fs, "espa",  "vespa", Hits().add(0));
+
+    fs.setMatchType(FieldSearcher::REGULAR);
+    EXPECT_TRUE(testStringFieldInfo(fs));
+}
+
+TEST("bool search") {
+    BoolFieldSearcher fs(0);
+    TEST_DO(assertBool(fs,     "true",  true, true));
+    TEST_DO(assertBool(fs,     "true",  false, false));
+    TEST_DO(assertBool(fs,     "1",  true, true));
+    TEST_DO(assertBool(fs,     "1",  false, false));
+    TEST_DO(assertBool(fs,     "false",  true, false));
+    TEST_DO(assertBool(fs,     "false",  false, true));
+    TEST_DO(assertBool(fs,     "0",  true, false));
+    TEST_DO(assertBool(fs,     "0",  false, true));
+    TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"),  true, BoolList().add(true).add(false).add(true)));
+    TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"),  false, BoolList().add(false).add(true).add(false)));
+}
+
+TEST("integer search")
+{
+    IntFieldSearcher fs(0);
+    TEST_DO(assertInt(fs,     "10",  10, true));
+    TEST_DO(assertInt(fs,      "9",  10, false));
+    TEST_DO(assertInt(fs,     ">9",  10, true));
+    TEST_DO(assertInt(fs,     ">9",   9, false));
+    TEST_DO(assertInt(fs,    "<11",  10, true));
+    TEST_DO(assertInt(fs,    "<11",  11, false));
+    TEST_DO(assertInt(fs,    "-10", -10, true));
+    TEST_DO(assertInt(fs,     "-9", -10, false));
+    TEST_DO(assertInt(fs,      "a",  10, false));
+    TEST_DO(assertInt(fs, "[-5;5]",  -5, true));
+    TEST_DO(assertInt(fs, "[-5;5]",   0, true));
+    TEST_DO(assertInt(fs, "[-5;5]",   5, true));
+    TEST_DO(assertInt(fs, "[-5;5]",  -6, false));
+    TEST_DO(assertInt(fs, "[-5;5]",   6, false));
+
+    TEST_DO(assertInt(fs, StringList().add("9").add("11"),  10, BoolList().add(false).add(false)));
+    TEST_DO(assertInt(fs, StringList().add("9").add("10"),  10, BoolList().add(false).add(true)));
+    TEST_DO(assertInt(fs, StringList().add("10").add(">9"), 10, BoolList().add(true).add(true)));
+
+    TEST_DO(assertInt(fs, "10", LongList().add(10).add(20).add(10).add(30), Hits().add(0).add(2)));
+    TEST_DO(assertInt(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30),
+                      HitsList().add(Hits().add(0).add(2)).add(Hits().add(1))));
+
+    TEST_DO(assertFieldInfo(fs, "10", 10, QTFieldInfo(0, 1, 1)));
+    TEST_DO(assertFieldInfo(fs, "10", LongList().add(10).add(20).add(10).add(30), QTFieldInfo(0, 2, 4)));
+    TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), 10,
+                            FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1))));
+    TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30),
+                            FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4))));
+}
+
+TEST("floating point search")
+{
+    FloatFieldSearcher fs;
+    TEST_DO(assertFloat(fs,         "10",    10, true));
+    TEST_DO(assertFloat(fs,       "10.5",  10.5, true));
+    TEST_DO(assertFloat(fs,      "-10.5", -10.5, true));
+    TEST_DO(assertFloat(fs,      ">10.5",  10.6, true));
+    TEST_DO(assertFloat(fs,      ">10.5",  10.5, false));
+    TEST_DO(assertFloat(fs,      "<10.5",  10.4, true));
+    TEST_DO(assertFloat(fs,      "<10.5",  10.5, false));
+    TEST_DO(assertFloat(fs,       "10.4",  10.5, false));
+    TEST_DO(assertFloat(fs,      "-10.4", -10.5, false));
+    TEST_DO(assertFloat(fs,          "a",  10.5, false));
+    TEST_DO(assertFloat(fs, "[-5.5;5.5]",  -5.5, true));
+    TEST_DO(assertFloat(fs, "[-5.5;5.5]",     0, true));
+    TEST_DO(assertFloat(fs, "[-5.5;5.5]",   5.5, true));
+    TEST_DO(assertFloat(fs, "[-5.5;5.5]",  -5.6, false));
+    TEST_DO(assertFloat(fs, "[-5.5;5.5]",   5.6, false));
+
+    TEST_DO(assertFloat(fs, StringList().add("10").add("11"),      10.5, BoolList().add(false).add(false)));
+    TEST_DO(assertFloat(fs, StringList().add("10").add("10.5"),    10.5, BoolList().add(false).add(true)));
+    TEST_DO(assertFloat(fs, StringList().add(">10.4").add("10.5"), 10.5, BoolList().add(true).add(true)));
+
+    TEST_DO(assertFloat(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), Hits().add(0).add(2)));
+    TEST_DO(assertFloat(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5),
+                    HitsList().add(Hits().add(0).add(2)).add(Hits().add(1))));
+
+    TEST_DO(assertFieldInfo(fs, "10.5", 10.5, QTFieldInfo(0, 1, 1)));
+    TEST_DO(assertFieldInfo(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), QTFieldInfo(0, 2, 4)));
+    TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), 10.5,
+                    FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1))));
+    TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5),
+                    FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4))));
+}
+
+TEST("Snippet modifier search") {
+    // ascii
+    assertSnippetModifier("f", "foo", "\x1F""f\x1Foo");
+    assertSnippetModifier("o", "foo", "f\x1Fo\x1F\x1Fo\x1F");
+    assertSnippetModifier("r", "bar", "ba\x1Fr\x1F");
+    assertSnippetModifier("foo", "foo foo", "\x1F""foo\x1F \x1F""foo\x1F");
+    assertSnippetModifier("aa", "aaaaaa", "\x1F""aa\x1F\x1F""aa\x1F\x1F""aa\x1F");
+    assertSnippetModifier("ab", "abcd\x1F""efgh", "\x1F""ab\x1F""cd\x1F""efgh");
+    assertSnippetModifier("ef", "abcd\x1F""efgh", "abcd\x1F\x1F""ef\x1Fgh");
+    assertSnippetModifier("fg", "abcd\x1F""efgh", "abcd\x1F""e\x1F""fg\x1Fh");
+    // the separator overlapping the match is skipped
+    assertSnippetModifier("cdef", "abcd\x1F""efgh", "ab\x1F""cdef\x1F""gh");
+    // no hits
+    assertSnippetModifier("bb", "aaaaaa", "aaaaaa");
+
+
+    // multiple query terms
+    assertSnippetModifier(StringList().add("ab").add("cd"), "abcd", "\x1F""ab\x1F\x1F""cd\x1F");
+    // when we have overlap we only get the first match
+    assertSnippetModifier(StringList().add("ab").add("bc"), "abcd", "\x1F""ab\x1F""cd");
+    assertSnippetModifier(StringList().add("bc").add("ab"), "abcd", "\x1F""ab\x1F""cd");
+    // the separator overlapping the match is skipped
+    assertSnippetModifier(StringList().add("de").add("ef"), "abcd\x1F""efgh", "abc\x1F""de\x1F""fgh");
+
+    // cjk
+    assertSnippetModifier("\xe7\x9f\xb3", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8",
+                                      "\x1f\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8");
+    assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8",
+                                                      "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8");
+    // the separator overlapping the match is skipped
+    assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\x1f\xe5\x87\xb1\xe5\x9c\xa8",
+                                                      "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8");
+
+    { // check that resizing works
+        UTF8SubstringSnippetModifier mod;
+        EXPECT_EQUAL(mod.getModifiedBuf().getLength(), 32u);
+        EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 0u);
+        performSearch(mod, StringList().add("a"), StringFieldValue("aaaaaaaaaaaaaaaa"));
+        EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 16u + 2 * 16u);
+        EXPECT_TRUE(mod.getModifiedBuf().getLength() >= mod.getModifiedBuf().getPos());
+    }
+}
+
+TEST("snippet modifier") {
+    { // string field value
+        SnippetModifierSetup sms(StringList().add("ab"));
+        // multiple invokations
+        assertSnippetModifier(sms, StringFieldValue("ab"), "\x1F""ab\x1F");
+        assertSnippetModifier(sms, StringFieldValue("xxxxabxxxxabxxxx"), "xxxx\x1F""ab\x1Fxxxx\x1F""ab\x1Fxxxx");
+        assertSnippetModifier(sms, StringFieldValue("xxabxx"), "xx\x1F""ab\x1Fxx");
+    }
+    { // collection field value
+        SnippetModifierSetup sms(StringList().add("ab"));
+        // multiple invokations
+        assertSnippetModifier(sms, getFieldValue(StringList().add("ab")), "\x1F""ab\x1F");
+        assertSnippetModifier(sms, getFieldValue(StringList().add("xxabxx")), "xx\x1F""ab\x1Fxx");
+        assertSnippetModifier(sms, getFieldValue(StringList().add("ab").add("xxabxx").add("xxxxxx")),
+                              "\x1F""ab\x1F\x1E""xx\x1F""ab\x1F""xx\x1E""xxxxxx");
+        assertSnippetModifier(sms, getFieldValue(StringList().add("cd").add("ef").add("gh")),
+                              "cd\x1E""ef\x1E""gh");
+    }
+    { // check that resizing works
+        SnippetModifierSetup sms(StringList().add("a"));
+        EXPECT_EQUAL(sms.modifier.getValueBuf().getLength(), 32u);
+        EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 0u);
+        sms.modifier.modify(StringFieldValue("aaaaaaaaaaaaaaaa"));
+        EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 16u + 2 * 16u);
+        EXPECT_TRUE(sms.modifier.getValueBuf().getLength() >= sms.modifier.getValueBuf().getPos());
+    }
+}
+
+TEST("FieldSearchSpec constrution") {
+    {
+        FieldSearchSpec f;
+        EXPECT_FALSE(f.valid());
+        EXPECT_EQUAL(0u, f.id());
+        EXPECT_EQUAL("", f.name());
+        EXPECT_EQUAL(0x100000u, f.maxLength());
+    }
+    {
+        FieldSearchSpec f(7, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 789);
+        EXPECT_TRUE(f.valid());
+        EXPECT_EQUAL(7u, f.id());
+        EXPECT_EQUAL("f0", f.name());
+        EXPECT_EQUAL(789u, f.maxLength());
+        EXPECT_EQUAL(789u, f.searcher().maxFieldLength());
+    }
+}
+
+TEST("snippet modifier manager") {
+    FieldSearchSpecMapT specMap;
+    specMap[0] = FieldSearchSpec(0, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 1000);
+    specMap[1] = FieldSearchSpec(1, "f1", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "", 1000);
+    IndexFieldMapT indexMap;
+    indexMap["i0"].push_back(0);
+    indexMap["i1"].push_back(1);
+    indexMap["i2"].push_back(0);
+    indexMap["i2"].push_back(1);
+
+    {
+        SnippetModifierManager man;
+        Query query(StringList().add("i0:foo"));
+        man.setup(query.qtl, specMap, indexMap);
+        assertQueryTerms(man, 0, StringList().add("foo"));
+        assertQueryTerms(man, 1, StringList());
+    }
+    {
+        SnippetModifierManager man;
+        Query query(StringList().add("i1:foo"));
+        man.setup(query.qtl, specMap, indexMap);
+        assertQueryTerms(man, 0, StringList());
+        assertQueryTerms(man, 1, StringList());
+    }
+    {
+        SnippetModifierManager man;
+        Query query(StringList().add("i1:*foo*"));
+        man.setup(query.qtl, specMap, indexMap);
+        assertQueryTerms(man, 0, StringList());
+        assertQueryTerms(man, 1, StringList().add("foo"));
+    }
+    {
+        SnippetModifierManager man;
+        Query query(StringList().add("i2:foo").add("i2:*bar*"));
+        man.setup(query.qtl, specMap, indexMap);
+        assertQueryTerms(man, 0, StringList().add("foo").add("bar"));
+        assertQueryTerms(man, 1, StringList().add("bar"));
+    }
+    { // check buffer sizes
+        SnippetModifierManager man;
+        Query query(StringList().add("i2:foo").add("i2:*bar*"));
+        man.setup(query.qtl, specMap, indexMap);
+        {
+            SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(0));
+            UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get();
+            EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u);
+            EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u);
+        }
+        {
+            SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(1));
+            UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get();
+            EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u);
+            EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u);
+        }
+    }
+}
+
+TEST("Stripping of indexes")
+{
+    EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f"));
+    EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f[0]"));
+    EXPECT_EQUAL("f[a]", FieldSearchSpecMap::stripNonFields("f[a]"));
+
+    EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a}"));
+    EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a0}"));
+    EXPECT_EQUAL("f{a 0}", FieldSearchSpecMap::stripNonFields("f{a 0}"));
+    EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{\"a 0\"}"));
+}
+
+TEST("counting of words") {
+    EXPECT_TRUE(assertCountWords(0, ""));
+    EXPECT_TRUE(assertCountWords(0, "?"));
+    EXPECT_TRUE(assertCountWords(1, "foo"));
+    EXPECT_TRUE(assertCountWords(2, "foo bar"));
+    EXPECT_TRUE(assertCountWords(2, "? foo bar"));
+    EXPECT_TRUE(assertCountWords(2, "foo bar ?"));
+
+    // check that 'a' is counted as 1 word
+    UTF8StrChrFieldSearcher fs(0);
+    StringList field = StringList().add("a").add("aa bb cc");
+    assertString(fs, "bb", field, Hits().add(2));
+    assertString(fs, StringList().add("bb").add("not"), field, HitsList().add(Hits().add(2)).add(Hits()));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
index fdbd60ce30a..01b625b6b3b 100644
--- a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
+++ b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_searchvisitor_test_app TEST
     SOURCES
     searchvisitor_test.cpp
     DEPENDS
-    streamingvisitors_searchvisitor
+    streamingvisitors
 )
 vespa_add_test(NAME streamingvisitors_searchvisitor_test_app COMMAND streamingvisitors_searchvisitor_test_app)
diff --git a/streamingvisitors/src/tests/textutil/.gitignore b/streamingvisitors/src/tests/textutil/.gitignore
new file mode 100644
index 00000000000..1103f79800a
--- /dev/null
+++ b/streamingvisitors/src/tests/textutil/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+textutil_test
+vsm_textutil_test_app
diff --git a/streamingvisitors/src/tests/textutil/CMakeLists.txt b/streamingvisitors/src/tests/textutil/CMakeLists.txt
new file mode 100644
index 00000000000..59817d01137
--- /dev/null
+++ b/streamingvisitors/src/tests/textutil/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vsm_textutil_test_app TEST
+    SOURCES
+    textutil.cpp
+    DEPENDS
+    streamingvisitors
+)
+vespa_add_test(NAME vsm_textutil_test_app COMMAND vsm_textutil_test_app)
diff --git a/streamingvisitors/src/tests/textutil/textutil.cpp b/streamingvisitors/src/tests/textutil/textutil.cpp
new file mode 100644
index 00000000000..2a1390eaa01
--- /dev/null
+++ b/streamingvisitors/src/tests/textutil/textutil.cpp
@@ -0,0 +1,285 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/fastlib/text/unicodeutil.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vsm/searcher/fold.h>
+#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h>
+#include <vespa/vsm/searcher/utf8stringfieldsearcherbase.h>
+
+using search::byte; // unsigned char
+
+namespace vsm {
+
+template <typename T>
+class Vector : public std::vector<T>
+{
+public:
+    Vector() : std::vector<T>() {}
+    Vector<T> & a(T v) { this->push_back(v); return *this; }
+};
+
+typedef Vector<ucs4_t> UCS4V;
+typedef Vector<size_t> SizeV;
+typedef UTF8StringFieldSearcherBase SFSB;
+typedef FUTF8StrChrFieldSearcher FSFS;
+
+class TextUtilTest : public vespalib::TestApp
+{
+private:
+    ucs4_t getUTF8Char(const char * src);
+    template <typename BW, bool OFF>
+    void assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets);
+    void assertAnsiFold(const std::string & toFold, const std::string & exp);
+    void assertAnsiFold(char c, char exp);
+#ifdef __x86_64__
+    void assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp);
+    void assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded = 16);
+#endif
+
+    template <typename BW, bool OFF>
+    void testSkipSeparators();
+    void testSkipSeparators();
+    void testSeparatorCharacter();
+    void testAnsiFold();
+    void test_lfoldua();
+#ifdef __x86_64__
+    void test_sse2_foldua();
+#endif
+
+public:
+    int Main() override;
+};
+
+ucs4_t
+TextUtilTest::getUTF8Char(const char * src)
+{
+    ucs4_t retval = Fast_UnicodeUtil::GetUTF8Char(src);
+    ASSERT_TRUE(retval != Fast_UnicodeUtil::_BadUTF8Char);
+    return retval;
+}
+
+template <typename BW, bool OFF>
+void
+TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets)
+{
+    const byte * srcbuf = reinterpret_cast<const byte *>(input);
+    auto dstbuf = std::make_unique<ucs4_t[]>(len + 1);
+    auto offsets = std::make_unique<size_t[]>(len + 1);
+    UTF8StrChrFieldSearcher fs;
+    BW bw(dstbuf.get(), offsets.get());
+    size_t dstlen = fs.skipSeparators(srcbuf, len, bw);
+    EXPECT_EQUAL(dstlen, expdstbuf.size());
+    ASSERT_TRUE(dstlen == expdstbuf.size());
+    for (size_t i = 0; i < dstlen; ++i) {
+        EXPECT_EQUAL(dstbuf[i], expdstbuf[i]);
+        if (OFF) {
+            EXPECT_EQUAL(offsets[i], expoffsets[i]);
+        }
+    }
+}
+
+void
+TextUtilTest::assertAnsiFold(const std::string & toFold, const std::string & exp)
+{
+    char folded[256];
+    EXPECT_TRUE(FSFS::ansiFold(toFold.c_str(), toFold.size(), folded));
+    EXPECT_EQUAL(std::string(folded, toFold.size()), exp);
+}
+
+void
+TextUtilTest::assertAnsiFold(char c, char exp)
+{
+    char folded;
+    EXPECT_TRUE(FSFS::ansiFold(&c, 1, &folded));
+    EXPECT_EQUAL((int32_t)folded, (int32_t)exp);
+}
+
+#ifdef __x86_64__
+void
+TextUtilTest::assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp)
+{
+    char folded[256];
+    size_t alignedStart =  0xF - (size_t(folded + 0xF) % 0x10);
+    const unsigned char * toFoldOrg = reinterpret_cast<const unsigned char *>(toFold.c_str());
+    const unsigned char * retval =
+        sse2_foldua(toFoldOrg, toFold.size(), reinterpret_cast<unsigned char *>(folded + alignedStart));
+    EXPECT_EQUAL((size_t)(retval - toFoldOrg), charFolded);
+    EXPECT_EQUAL(std::string(folded + alignedStart, charFolded), exp);
+}
+
+void
+TextUtilTest::assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded)
+{
+    unsigned char toFold[16];
+    memset(toFold, c, 16);
+    unsigned char folded[32];
+    size_t alignedStart =  0xF - (size_t(folded + 0xF) % 0x10);
+    const unsigned char * retval = sse2_foldua(toFold, 16, folded + alignedStart);
+    EXPECT_EQUAL((size_t)(retval - toFold), charFolded);
+    for (size_t i = 0; i < charFolded; ++i) {
+        EXPECT_EQUAL((int32_t)folded[i + alignedStart], (int32_t)exp);
+    }
+}
+#endif
+
+template <typename BW, bool OFF>
+void
+TextUtilTest::testSkipSeparators()
+{
+    // ascii characters
+    assertSkipSeparators<BW, OFF>("foo",    3, UCS4V().a('f').a('o').a('o'),  SizeV().a(0).a(1).a(2));
+    assertSkipSeparators<BW, OFF>("f\x1Fo", 3, UCS4V().a('f').a('o'),         SizeV().a(0).a(2));
+    assertSkipSeparators<BW, OFF>("f\no",   3, UCS4V().a('f').a('\n').a('o'), SizeV().a(0).a(1).a(2));
+    assertSkipSeparators<BW, OFF>("f\to",   3, UCS4V().a('f').a('\t').a('o'), SizeV().a(0).a(1).a(2));
+
+    // utf8 char
+    assertSkipSeparators<BW, OFF>("\xC2\x80\x66",         3, UCS4V().a(getUTF8Char("\xC2\x80")).a('f'),
+                                                             SizeV().a(0).a(2));
+    assertSkipSeparators<BW, OFF>("\xE0\xA0\x80\x66",     4, UCS4V().a(getUTF8Char("\xE0\xA0\x80")).a('f'),
+                                                             SizeV().a(0).a(3));
+    assertSkipSeparators<BW, OFF>("\xF0\x90\x80\x80\x66", 5, UCS4V().a(getUTF8Char("\xF0\x90\x80\x80")).a('f'),
+                                                             SizeV().a(0).a(4));
+
+    // replacement string (sharp s -> ss)
+    assertSkipSeparators<BW, OFF>("\xC3\x9F\x66\xC3\x9F", 5, UCS4V().a('s').a('s').a('f').a('s').a('s'),
+                                                             SizeV().a(0).a(0).a(2).a(3).a(3));
+}
+
+void
+TextUtilTest::testSkipSeparators()
+{
+    Fast_NormalizeWordFolder::Setup(Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION);
+
+    testSkipSeparators<SFSB::BufferWrapper, false>();
+    testSkipSeparators<SFSB::OffsetWrapper, true>();
+}
+
+void
+TextUtilTest::testSeparatorCharacter()
+{
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x00'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x01'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x02'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x03'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x04'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x05'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x06'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x07'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x08'));
+    EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x09')); // '\t'
+    EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x0a')); // '\n'
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0b'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0c'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0d'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0e'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0f'));
+
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x10'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x11'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x12'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x13'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x14'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x15'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x16'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x17'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x18'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x19'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1a'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1b'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1c'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1d'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1e'));
+    EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1f'));
+
+    EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x20')); // space
+}
+
+void
+TextUtilTest::testAnsiFold()
+{
+    FieldSearcher::init();
+    assertAnsiFold("", "");
+    assertAnsiFold("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz");
+    assertAnsiFold("abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz");
+    assertAnsiFold("0123456789", "0123456789");
+    for (int i = 0; i < 128; ++i) {
+        if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) {
+            assertAnsiFold(i, i);
+        } else if (i >= 'A' && i <= 'Z') {
+            assertAnsiFold(i, i + 32);
+        } else {
+            assertAnsiFold(i, 0);
+        }
+    }
+
+    // non-ascii is ignored
+    for (int i = 128; i < 256; ++i) {
+        char toFold = i;
+        char folded;
+        EXPECT_TRUE(!FSFS::ansiFold(&toFold, 1, &folded));
+    }
+}
+
+void
+TextUtilTest::test_lfoldua()
+{
+    FieldSearcher::init();
+    char folded[256];
+    size_t alignedStart = 0;
+    const char * toFold = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    size_t len = strlen(toFold);
+    EXPECT_TRUE(FSFS::lfoldua(toFold, len, folded, alignedStart));
+    EXPECT_EQUAL(std::string(folded + alignedStart, len), "abcdefghijklmnopqrstuvwxyz");
+}
+
+#ifdef __x86_64__
+void
+TextUtilTest::test_sse2_foldua()
+{
+    assert_sse2_foldua("", 0, "");
+    assert_sse2_foldua("ABCD", 0, "");
+    assert_sse2_foldua("ABCDEFGHIJKLMNO",   0,  "");
+    assert_sse2_foldua("ABCDEFGHIJKLMNOP",  16, "abcdefghijklmnop");
+    assert_sse2_foldua("ABCDEFGHIJKLMNOPQ", 16, "abcdefghijklmnop");
+    assert_sse2_foldua("KLMNOPQRSTUVWXYZ",  16, "klmnopqrstuvwxyz");
+    assert_sse2_foldua("abcdefghijklmnop",  16, "abcdefghijklmnop");
+    assert_sse2_foldua("klmnopqrstuvwxyz",  16, "klmnopqrstuvwxyz");
+    assert_sse2_foldua("0123456789abcdef",  16, "0123456789abcdef");
+
+    for (int i = 0; i < 128; ++i) {
+        if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) {
+            assert_sse2_foldua(i, i);
+        } else if (i >= 'A' && i <= 'Z') {
+            assert_sse2_foldua(i, i + 32);
+        } else {
+            assert_sse2_foldua(i, 0);
+        }
+    }
+
+    // non-ascii is ignored
+    for (int i = 128; i < 256; ++i) {
+        assert_sse2_foldua(i, '?', 0);
+    }
+}
+#endif
+
+int
+TextUtilTest::Main()
+{
+    TEST_INIT("textutil_test");
+
+    testSkipSeparators();
+    testSeparatorCharacter();
+    testAnsiFold();
+    test_lfoldua();
+#ifdef __x86_64__
+    test_sse2_foldua();
+#endif
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(vsm::TextUtilTest);
diff --git a/streamingvisitors/src/tests/utilapps/.gitignore b/streamingvisitors/src/tests/utilapps/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/streamingvisitors/src/tests/utilapps/.gitignore