aboutsummaryrefslogtreecommitdiffstats
path: root/streamingvisitors/src/tests
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2022-05-15 00:40:43 +0200
committerGitHub <noreply@github.com>2022-05-15 00:40:43 +0200
commitdacf557add1c6a3ffab036cdf2f7dfdf9750b22e (patch)
tree3a9dfff58b98898e2e28c0337925f4f04e5eaeb0 /streamingvisitors/src/tests
parent2722ce9d1d1ec12d57ebd3833ce37b0958afb752 (diff)
Revert "Collapse vsm into streamingvisitors"
Diffstat (limited to 'streamingvisitors/src/tests')
-rw-r--r--streamingvisitors/src/tests/charbuffer/.gitignore4
-rw-r--r--streamingvisitors/src/tests/charbuffer/CMakeLists.txt8
-rw-r--r--streamingvisitors/src/tests/charbuffer/charbuffer.cpp80
-rw-r--r--streamingvisitors/src/tests/config/mail.cfg116
-rw-r--r--streamingvisitors/src/tests/config/vsm.cfg3
-rw-r--r--streamingvisitors/src/tests/config/vsmfields.cfg297
-rw-r--r--streamingvisitors/src/tests/docsum/.gitignore4
-rw-r--r--streamingvisitors/src/tests/docsum/CMakeLists.txt8
-rw-r--r--streamingvisitors/src/tests/docsum/docsum.cpp293
-rw-r--r--streamingvisitors/src/tests/document/.gitignore4
-rw-r--r--streamingvisitors/src/tests/document/CMakeLists.txt8
-rw-r--r--streamingvisitors/src/tests/document/document.cpp129
-rw-r--r--streamingvisitors/src/tests/hitcollector/CMakeLists.txt2
-rw-r--r--streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt2
-rw-r--r--streamingvisitors/src/tests/querywrapper/CMakeLists.txt2
-rw-r--r--streamingvisitors/src/tests/searcher/.gitignore4
-rw-r--r--streamingvisitors/src/tests/searcher/CMakeLists.txt8
-rw-r--r--streamingvisitors/src/tests/searcher/searcher_test.cpp864
-rw-r--r--streamingvisitors/src/tests/searchvisitor/CMakeLists.txt2
-rw-r--r--streamingvisitors/src/tests/textutil/.gitignore4
-rw-r--r--streamingvisitors/src/tests/textutil/CMakeLists.txt8
-rw-r--r--streamingvisitors/src/tests/textutil/textutil.cpp285
-rw-r--r--streamingvisitors/src/tests/utilapps/.gitignore0
23 files changed, 4 insertions, 2131 deletions
diff --git a/streamingvisitors/src/tests/charbuffer/.gitignore b/streamingvisitors/src/tests/charbuffer/.gitignore
deleted file mode 100644
index 2c980038fb5..00000000000
--- a/streamingvisitors/src/tests/charbuffer/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-charbuffer_test
-vsm_charbuffer_test_app
diff --git a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt
deleted file mode 100644
index 5d0c0068d37..00000000000
--- a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vsm_charbuffer_test_app TEST
- SOURCES
- charbuffer.cpp
- DEPENDS
- streamingvisitors
-)
-vespa_add_test(NAME vsm_charbuffer_test_app COMMAND vsm_charbuffer_test_app)
diff --git a/streamingvisitors/src/tests/charbuffer/charbuffer.cpp b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp
deleted file mode 100644
index 736d35459cb..00000000000
--- a/streamingvisitors/src/tests/charbuffer/charbuffer.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/vsm/common/charbuffer.h>
-
-namespace vsm {
-
-class CharBufferTest : public vespalib::TestApp
-{
-private:
- void test();
-public:
- int Main() override;
-};
-
-void
-CharBufferTest::test()
-{
- { // empty
- CharBuffer buf;
- EXPECT_EQUAL(buf.getLength(), 0u);
- EXPECT_EQUAL(buf.getPos(), 0u);
- EXPECT_EQUAL(buf.getRemaining(), 0u);
- }
- { // explicit length
- CharBuffer buf(8);
- EXPECT_EQUAL(buf.getLength(), 8u);
- EXPECT_EQUAL(buf.getPos(), 0u);
- EXPECT_EQUAL(buf.getRemaining(), 8u);
- }
- { // resize
- CharBuffer buf(8);
- EXPECT_EQUAL(buf.getLength(), 8u);
- buf.resize(16);
- EXPECT_EQUAL(buf.getLength(), 16u);
- buf.resize(8);
- EXPECT_EQUAL(buf.getLength(), 16u);
- }
- { // put with triggered resize
- CharBuffer buf(8);
- buf.put("123456", 6);
- EXPECT_EQUAL(buf.getLength(), 8u);
- EXPECT_EQUAL(buf.getPos(), 6u);
- EXPECT_EQUAL(buf.getRemaining(), 2u);
- EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456");
- buf.put("789", 3);
- EXPECT_EQUAL(buf.getLength(), 12u);
- EXPECT_EQUAL(buf.getPos(), 9u);
- EXPECT_EQUAL(buf.getRemaining(), 3u);
- EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789");
- buf.put('a');
- EXPECT_EQUAL(buf.getLength(), 12u);
- EXPECT_EQUAL(buf.getPos(), 10u);
- EXPECT_EQUAL(buf.getRemaining(), 2u);
- EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789a");
- buf.reset();
- EXPECT_EQUAL(buf.getLength(), 12u);
- EXPECT_EQUAL(buf.getPos(), 0u);
- EXPECT_EQUAL(buf.getRemaining(), 12u);
- buf.put("bcd", 3);
- EXPECT_EQUAL(buf.getLength(), 12u);
- EXPECT_EQUAL(buf.getPos(), 3u);
- EXPECT_EQUAL(buf.getRemaining(), 9u);
- EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "bcd");
- }
-}
-
-int
-CharBufferTest::Main()
-{
- TEST_INIT("charbuffer_test");
-
- test();
-
- TEST_DONE();
-}
-
-}
-
-TEST_APPHOOK(vsm::CharBufferTest);
diff --git a/streamingvisitors/src/tests/config/mail.cfg b/streamingvisitors/src/tests/config/mail.cfg
deleted file mode 100644
index ce830beac23..00000000000
--- a/streamingvisitors/src/tests/config/mail.cfg
+++ /dev/null
@@ -1,116 +0,0 @@
-datatype[2]
-datatype[0].id 1012
-datatype[0].arraytype[1]
-datatype[0].arraytype[0].datatype 12
-datatype[1].id 1013
-datatype[1].arraytype[1]
-datatype[1].arraytype[0].datatype 13
-documenttype[1]
-documenttype[0].name mail
-documenttype[0].version 0
-documenttype[0].inherits[0]
-documenttype[0].field[26]
-documenttype[0].field[0].name mailid
-documenttype[0].field[0].id 2
-documenttype[0].field[0].header true
-documenttype[0].field[0].datatype 2
-documenttype[0].field[1].name date
-documenttype[0].field[1].id 3
-documenttype[0].field[1].header true
-documenttype[0].field[1].datatype 0
-documenttype[0].field[2].name from
-documenttype[0].field[2].id 4
-documenttype[0].field[2].header true
-documenttype[0].field[2].datatype 12
-documenttype[0].field[3].name replyto
-documenttype[0].field[3].id 5
-documenttype[0].field[3].header true
-documenttype[0].field[3].datatype 12
-documenttype[0].field[4].name to
-documenttype[0].field[4].id 6
-documenttype[0].field[4].header true
-documenttype[0].field[4].datatype 12
-documenttype[0].field[5].name cc
-documenttype[0].field[5].id 7
-documenttype[0].field[5].header true
-documenttype[0].field[5].datatype 12
-documenttype[0].field[6].name bcc
-documenttype[0].field[6].id 8
-documenttype[0].field[6].header true
-documenttype[0].field[6].datatype 12
-documenttype[0].field[7].name subject
-documenttype[0].field[7].id 9
-documenttype[0].field[7].header true
-documenttype[0].field[7].datatype 12
-documenttype[0].field[8].name body
-documenttype[0].field[8].id 10
-documenttype[0].field[8].header false
-documenttype[0].field[8].datatype 12
-documenttype[0].field[9].name attachmentcount
-documenttype[0].field[9].id 11
-documenttype[0].field[9].header false
-documenttype[0].field[9].datatype 0
-documenttype[0].field[10].name attachmentpartids
-documenttype[0].field[10].id 12
-documenttype[0].field[10].header false
-documenttype[0].field[10].datatype 2
-documenttype[0].field[11].name attachmentsizes
-documenttype[0].field[11].id 13
-documenttype[0].field[11].header false
-documenttype[0].field[11].datatype 2
-documenttype[0].field[12].name attachmentnames
-documenttype[0].field[12].id 14
-documenttype[0].field[12].header false
-documenttype[0].field[12].datatype 2
-documenttype[0].field[13].name attachmenttypes
-documenttype[0].field[13].id 15
-documenttype[0].field[13].header false
-documenttype[0].field[13].datatype 2
-documenttype[0].field[14].name attachmentlanguages
-documenttype[0].field[14].id 16
-documenttype[0].field[14].header false
-documenttype[0].field[14].datatype 2
-documenttype[0].field[15].name attachmentcontent
-documenttype[0].field[15].id 17
-documenttype[0].field[15].header false
-documenttype[0].field[15].datatype 2
-documenttype[0].field[16].name bodylanguage
-documenttype[0].field[16].id 18
-documenttype[0].field[16].header false
-documenttype[0].field[16].datatype 2
-documenttype[0].field[17].name bodyencoding
-documenttype[0].field[17].id 19
-documenttype[0].field[17].header false
-documenttype[0].field[17].datatype 2
-documenttype[0].field[18].name collectionid
-documenttype[0].field[18].id 20
-documenttype[0].field[18].header true
-documenttype[0].field[18].datatype 4
-documenttype[0].field[19].name content
-documenttype[0].field[19].id 21
-documenttype[0].field[19].header true
-documenttype[0].field[19].datatype 12
-documenttype[0].field[20].name bodymeta
-documenttype[0].field[20].id 50027053
-documenttype[0].field[20].header false
-documenttype[0].field[20].datatype 13
-documenttype[0].field[21].name attachments
-documenttype[0].field[21].id 1081629685
-documenttype[0].field[21].header false
-documenttype[0].field[21].datatype 1012
-documenttype[0].field[22].name attachmentsmeta
-documenttype[0].field[22].id 1203055625
-documenttype[0].field[22].header false
-documenttype[0].field[22].datatype 1013
-documenttype[0].field[23].name tolist
-documenttype[0].field[23].id 1084918181
-documenttype[0].field[23].header false
-documenttype[0].field[23].datatype 1012
-documenttype[0].field[24].name cclist
-documenttype[0].field[24].id 1733332403
-documenttype[0].field[24].header false
-documenttype[0].field[24].datatype 1012
-documenttype[0].field[25].name bcclist
-documenttype[0].field[25].id 410546306
-documenttype[0].field[25].header false
-documenttype[0].field[25].datatype 1012
diff --git a/streamingvisitors/src/tests/config/vsm.cfg b/streamingvisitors/src/tests/config/vsm.cfg
deleted file mode 100644
index dc50447f623..00000000000
--- a/streamingvisitors/src/tests/config/vsm.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-doctype file:../config/mail.cfg
-storagecfg ""
-vsmfields file:../config/vsmfields.cfg
diff --git a/streamingvisitors/src/tests/config/vsmfields.cfg b/streamingvisitors/src/tests/config/vsmfields.cfg
deleted file mode 100644
index 30f1c8ed8b1..00000000000
--- a/streamingvisitors/src/tests/config/vsmfields.cfg
+++ /dev/null
@@ -1,297 +0,0 @@
-threadsperquery 4
-documentverificationlevel=0
-searchall 1
-fieldspec[17]
-fieldspec[0].name bcc
-fieldspec[0].searchmethod AUTOUTF8
-fieldspec[0].arg1 ""
-fieldspec[1].name cc
-fieldspec[1].searchmethod AUTOUTF8
-fieldspec[1].arg1 ""
-fieldspec[2].name from
-fieldspec[2].searchmethod AUTOUTF8
-fieldspec[2].arg1 ""
-fieldspec[3].name date
-fieldspec[3].searchmethod INT32
-fieldspec[3].arg1 ""
-fieldspec[4].name replyto
-fieldspec[4].searchmethod AUTOUTF8
-fieldspec[4].arg1 ""
-fieldspec[5].name subject
-fieldspec[5].searchmethod AUTOUTF8
-fieldspec[5].arg1 ""
-fieldspec[6].name to
-fieldspec[6].searchmethod AUTOUTF8
-fieldspec[6].arg1 ""
-fieldspec[7].name body
-fieldspec[7].searchmethod AUTOUTF8
-fieldspec[7].arg1 ""
-fieldspec[8].name bodymeta
-fieldspec[8].searchmethod AUTOUTF8
-fieldspec[8].arg1 ""
-fieldspec[9].name mailid
-fieldspec[9].searchmethod AUTOUTF8
-fieldspec[9].arg1 ""
-fieldspec[10].name attachmentcount
-fieldspec[10].searchmethod INT32
-fieldspec[10].arg1 ""
-fieldspec[11].name attachmentcontent
-fieldspec[11].searchmethod AUTOUTF8
-fieldspec[11].arg1 ""
-fieldspec[12].name attachmenttypes
-fieldspec[12].searchmethod AUTOUTF8
-fieldspec[12].arg1 ""
-fieldspec[13].name attachmentnames
-fieldspec[13].searchmethod AUTOUTF8
-fieldspec[13].arg1 ""
-fieldspec[14].name attachmentlanguages
-fieldspec[14].searchmethod AUTOUTF8
-fieldspec[14].arg1 ""
-fieldspec[15].name URI
-fieldspec[15].searchmethod AUTOUTF8
-fieldspec[15].arg1 ""
-fieldspec[16].name vsm_whichfieldmatched
-fieldspec[16].searchmethod AUTOUTF8
-fieldspec[16].arg1 ""
-index[26]
-index[0].name default
-index[0].field[10]
-index[0].field[0].name from
-index[0].field[1].name to
-index[0].field[2].name cc
-index[0].field[3].name bcc
-index[0].field[4].name subject
-index[0].field[5].name body
-index[0].field[6].name attachmentcontent
-index[0].field[7].name attachmentnames
-index[0].field[8].name attachmenttypes
-index[0].field[9].name date
-index[1].name all
-index[1].field[8]
-index[1].field[0].name to
-index[1].field[1].name cc
-index[1].field[2].name bcc
-index[1].field[3].name subject
-index[1].field[4].name body
-index[1].field[5].name attachmentcontent
-index[1].field[6].name attachmentnames
-index[1].field[7].name attachmenttypes
-index[2].name header
-index[2].field[6]
-index[2].field[0].name from
-index[2].field[1].name replyto
-index[2].field[2].name to
-index[2].field[3].name cc
-index[2].field[4].name bcc
-index[2].field[5].name subject
-index[3].name senders
-index[3].field[2]
-index[3].field[0].name from
-index[3].field[1].name replyto
-index[4].name recipients
-index[4].field[3]
-index[4].field[0].name to
-index[4].field[1].name cc
-index[4].field[2].name bcc
-index[5].name address
-index[5].field[5]
-index[5].field[0].name from
-index[5].field[1].name replyto
-index[5].field[2].name to
-index[5].field[3].name cc
-index[5].field[4].name bcc
-index[6].name body
-index[6].field[2]
-index[6].field[0].name subject
-index[6].field[1].name body
-index[7].name meta
-index[7].field[2]
-index[7].field[0].name attachmentcontent
-index[7].field[1].name attachmenttypes
-index[8].name index1
-index[8].field[1]
-index[8].field[0].name bcc
-index[9].name index2
-index[9].field[2]
-index[9].field[0].name bcc
-index[9].field[1].name cc
-index[10].name index3
-index[10].field[3]
-index[10].field[0].name bcc
-index[10].field[1].name cc
-index[10].field[2].name from
-index[11].name index4
-index[11].field[4]
-index[11].field[0].name bcc
-index[11].field[1].name cc
-index[11].field[2].name from
-index[11].field[3].name date
-index[12].name index5
-index[12].field[5]
-index[12].field[0].name bcc
-index[12].field[1].name cc
-index[12].field[2].name from
-index[12].field[3].name date
-index[12].field[4].name replyto
-index[13].name index6
-index[13].field[6]
-index[13].field[0].name bcc
-index[13].field[1].name cc
-index[13].field[2].name from
-index[13].field[3].name date
-index[13].field[4].name replyto
-index[13].field[5].name subject
-index[14].name index7
-index[14].field[7]
-index[14].field[0].name bcc
-index[14].field[1].name cc
-index[14].field[2].name from
-index[14].field[3].name date
-index[14].field[4].name replyto
-index[14].field[5].name subject
-index[14].field[6].name to
-index[15].name index8
-index[15].field[8]
-index[15].field[0].name bcc
-index[15].field[1].name cc
-index[15].field[2].name from
-index[15].field[3].name date
-index[15].field[4].name replyto
-index[15].field[5].name subject
-index[15].field[6].name to
-index[15].field[7].name body
-index[16].name index9
-index[16].field[9]
-index[16].field[0].name bcc
-index[16].field[1].name cc
-index[16].field[2].name from
-index[16].field[3].name date
-index[16].field[4].name replyto
-index[16].field[5].name subject
-index[16].field[6].name to
-index[16].field[7].name body
-index[16].field[8].name bodymeta
-index[17].name index10
-index[17].field[10]
-index[17].field[0].name bcc
-index[17].field[1].name cc
-index[17].field[2].name from
-index[17].field[3].name date
-index[17].field[4].name replyto
-index[17].field[5].name subject
-index[17].field[6].name to
-index[17].field[7].name body
-index[17].field[8].name bodymeta
-index[17].field[9].name mailid
-index[18].name index11
-index[18].field[11]
-index[18].field[0].name bcc
-index[18].field[1].name cc
-index[18].field[2].name from
-index[18].field[3].name date
-index[18].field[4].name replyto
-index[18].field[5].name subject
-index[18].field[6].name to
-index[18].field[7].name body
-index[18].field[8].name bodymeta
-index[18].field[9].name mailid
-index[18].field[10].name attachmentcount
-index[19].name index12
-index[19].field[12]
-index[19].field[0].name bcc
-index[19].field[1].name cc
-index[19].field[2].name from
-index[19].field[3].name date
-index[19].field[4].name replyto
-index[19].field[5].name subject
-index[19].field[6].name to
-index[19].field[7].name body
-index[19].field[8].name bodymeta
-index[19].field[9].name mailid
-index[19].field[10].name attachmentcount
-index[19].field[11].name attachmentcontent
-index[20].name index13
-index[20].field[13]
-index[20].field[0].name bcc
-index[20].field[1].name cc
-index[20].field[2].name from
-index[20].field[3].name date
-index[20].field[4].name replyto
-index[20].field[5].name subject
-index[20].field[6].name to
-index[20].field[7].name body
-index[20].field[8].name bodymeta
-index[20].field[9].name mailid
-index[20].field[10].name attachmentcount
-index[20].field[11].name attachmentcontent
-index[20].field[12].name attachmenttypes
-index[21].name index14
-index[21].field[14]
-index[21].field[0].name bcc
-index[21].field[1].name cc
-index[21].field[2].name from
-index[21].field[3].name date
-index[21].field[4].name replyto
-index[21].field[5].name subject
-index[21].field[6].name to
-index[21].field[7].name body
-index[21].field[8].name bodymeta
-index[21].field[9].name mailid
-index[21].field[10].name attachmentcount
-index[21].field[11].name attachmentcontent
-index[21].field[12].name attachmenttypes
-index[21].field[13].name attachmentnames
-index[22].name index15
-index[22].field[15]
-index[22].field[0].name bcc
-index[22].field[1].name cc
-index[22].field[2].name from
-index[22].field[3].name date
-index[22].field[4].name replyto
-index[22].field[5].name subject
-index[22].field[6].name to
-index[22].field[7].name body
-index[22].field[8].name bodymeta
-index[22].field[9].name mailid
-index[22].field[10].name attachmentcount
-index[22].field[11].name attachmentcontent
-index[22].field[12].name attachmenttypes
-index[22].field[13].name attachmentnames
-index[22].field[14].name attachmentlanguages
-index[23].name index16
-index[23].field[15]
-index[23].field[0].name bcc
-index[23].field[1].name cc
-index[23].field[2].name from
-index[23].field[3].name date
-index[23].field[4].name replyto
-index[23].field[5].name subject
-index[23].field[6].name to
-index[23].field[7].name body
-index[23].field[8].name bodymeta
-index[23].field[9].name mailid
-index[23].field[10].name attachmentcount
-index[23].field[11].name attachmentcontent
-index[23].field[12].name attachmenttypes
-index[23].field[13].name attachmentnames
-index[23].field[14].name attachmentlanguages
-index[24].name index17
-index[24].field[15]
-index[24].field[0].name bcc
-index[24].field[1].name cc
-index[24].field[2].name from
-index[24].field[3].name date
-index[24].field[4].name replyto
-index[24].field[5].name subject
-index[24].field[6].name to
-index[24].field[7].name body
-index[24].field[8].name bodymeta
-index[24].field[9].name mailid
-index[24].field[10].name attachmentcount
-index[24].field[11].name attachmentcontent
-index[24].field[12].name attachmenttypes
-index[24].field[13].name attachmentnames
-index[24].field[14].name attachmentlanguages
-index[25].name date
-index[25].field[1]
-index[25].field[0].name date
diff --git a/streamingvisitors/src/tests/docsum/.gitignore b/streamingvisitors/src/tests/docsum/.gitignore
deleted file mode 100644
index 9a697a94de8..00000000000
--- a/streamingvisitors/src/tests/docsum/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-docsum_test
-vsm_docsum_test_app
diff --git a/streamingvisitors/src/tests/docsum/CMakeLists.txt b/streamingvisitors/src/tests/docsum/CMakeLists.txt
deleted file mode 100644
index 87c46409053..00000000000
--- a/streamingvisitors/src/tests/docsum/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vsm_docsum_test_app TEST
- SOURCES
- docsum.cpp
- DEPENDS
- streamingvisitors
-)
-vespa_add_test(NAME vsm_docsum_test_app COMMAND vsm_docsum_test_app)
diff --git a/streamingvisitors/src/tests/docsum/docsum.cpp b/streamingvisitors/src/tests/docsum/docsum.cpp
deleted file mode 100644
index 475489d2f5a..00000000000
--- a/streamingvisitors/src/tests/docsum/docsum.cpp
+++ /dev/null
@@ -1,293 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/datatype/structdatatype.h>
-#include <vespa/document/datatype/weightedsetdatatype.h>
-#include <vespa/document/datatype/mapdatatype.h>
-#include <vespa/vsm/common/docsum.h>
-#include <vespa/vsm/vsm/flattendocsumwriter.h>
-#include <vespa/vsm/vsm/slimefieldwriter.h>
-
-using namespace document;
-
-namespace vsm {
-
-template <typename T>
-class Vector : public std::vector<T>
-{
-public:
- Vector<T> & add(T v) { this->push_back(v); return *this; }
-};
-
-typedef Vector<std::string> StringList;
-typedef Vector<std::pair<std::string, int32_t> > WeightedStringList;
-
-
-class TestDocument : public vsm::Document
-{
-private:
- std::vector<FieldValueContainer> _fields;
-
-public:
- TestDocument(const search::DocumentIdT & docId, size_t numFields) : vsm::Document(docId, numFields), _fields(numFields) {}
- virtual bool setField(FieldIdT fId, document::FieldValue::UP fv) override {
- if (fId < _fields.size()) {
- _fields[fId].reset(fv.release());
- return true;
- }
- return false;
- }
- virtual const document::FieldValue * getField(FieldIdT fId) const override {
- if (fId < _fields.size()) {
- return _fields[fId].get();
- }
- return NULL;
- }
-};
-
-
-class DocsumTest : public vespalib::TestApp
-{
-private:
- ArrayFieldValue createFieldValue(const StringList & fv);
- WeightedSetFieldValue createFieldValue(const WeightedStringList & fv);
-
- void assertFlattenDocsumWriter(const FieldValue & fv, const std::string & exp) {
- FlattenDocsumWriter fdw;
- assertFlattenDocsumWriter(fdw, fv, exp);
- }
- void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp);
- void assertSlimeFieldWriter(const FieldValue & fv, const std::string & exp) {
- SlimeFieldWriter sfw;
- TEST_DO(assertSlimeFieldWriter(sfw, fv, exp));
- }
- void assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp);
-
- void testFlattenDocsumWriter();
- void testSlimeFieldWriter();
- void requireThatSlimeFieldWriterHandlesMap();
- void testDocSumCache();
-
-public:
- int Main() override;
-};
-
-ArrayFieldValue
-DocsumTest::createFieldValue(const StringList & fv)
-{
-
- static ArrayDataType type(*DataType::STRING);
- ArrayFieldValue afv(type);
- for (size_t i = 0; i < fv.size(); ++i) {
- afv.add(StringFieldValue(fv[i]));
- }
- return afv;
-}
-
-WeightedSetFieldValue
-DocsumTest::createFieldValue(const WeightedStringList & fv)
-{
- static WeightedSetDataType type(*DataType::STRING, false, false);
- WeightedSetFieldValue wsfv(type);
- for (size_t i = 0; i < fv.size(); ++i) {
- wsfv.add(StringFieldValue(fv[i].first), fv[i].second);
- }
- return wsfv;
-}
-
-void
-DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp)
-{
- FieldPath empty;
- fv.iterateNested(empty.getFullRange(), fdw);
- std::string actual(fdw.getResult().getBuffer(), fdw.getResult().getPos());
- EXPECT_EQUAL(actual, exp);
-}
-
-void
-DocsumTest::assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp)
-{
- sfw.convert(fv);
-
- vespalib::Slime gotSlime;
- vespalib::Memory serialized(sfw.out());
- size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, gotSlime);
- ASSERT_EQUAL(decodeRes, serialized.size);
-
- vespalib::Slime expSlime;
- size_t used = vespalib::slime::JsonFormat::decode(exp, expSlime);
- EXPECT_TRUE(used > 0);
- EXPECT_EQUAL(expSlime, gotSlime);
-}
-
-void
-DocsumTest::testFlattenDocsumWriter()
-{
- { // basic tests
- TEST_DO(assertFlattenDocsumWriter(StringFieldValue("foo bar"), "foo bar"));
- TEST_DO(assertFlattenDocsumWriter(RawFieldValue("foo bar"), "foo bar"));
- TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(true), "true"));
- TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(false), "false"));
- TEST_DO(assertFlattenDocsumWriter(LongFieldValue(123456789), "123456789"));
- TEST_DO(assertFlattenDocsumWriter(createFieldValue(StringList().add("foo bar").add("baz").add(" qux ")),
- "foo bar baz qux "));
- }
- { // test mulitple invokations
- FlattenDocsumWriter fdw("#");
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("foo"), "foo"));
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("bar"), "foo#bar"));
- fdw.clear();
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("baz"), "baz"));
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("qux"), "baz qux"));
- }
- { // test resizing
- FlattenDocsumWriter fdw("#");
- EXPECT_EQUAL(fdw.getResult().getPos(), 0u);
- EXPECT_EQUAL(fdw.getResult().getLength(), 32u);
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaabbbbccccddddeeeeffffgggghhhh"),
- "aaaabbbbccccddddeeeeffffgggghhhh"));
- EXPECT_EQUAL(fdw.getResult().getPos(), 32u);
- EXPECT_EQUAL(fdw.getResult().getLength(), 32u);
- TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaa"), "aaaabbbbccccddddeeeeffffgggghhhh#aaaa"));
- EXPECT_EQUAL(fdw.getResult().getPos(), 37u);
- EXPECT_TRUE(fdw.getResult().getLength() >= 37u);
- fdw.clear();
- EXPECT_EQUAL(fdw.getResult().getPos(), 0u);
- EXPECT_TRUE(fdw.getResult().getLength() >= 37u);
- }
-}
-
-void
-DocsumTest::testSlimeFieldWriter()
-{
- { // basic types
- assertSlimeFieldWriter(LongFieldValue(123456789), "123456789");
- assertSlimeFieldWriter(BoolFieldValue(true), "true");
- assertSlimeFieldWriter(BoolFieldValue(false), "false");
- assertSlimeFieldWriter(DoubleFieldValue(12.34), "12.34");
- assertSlimeFieldWriter(StringFieldValue("foo bar"), "\"foo bar\"");
- }
- { // collection field values
- assertSlimeFieldWriter(createFieldValue(StringList().add("foo").add("bar").add("baz")),
- "[\"foo\",\"bar\",\"baz\"]");
- assertSlimeFieldWriter(createFieldValue(WeightedStringList().add(std::make_pair("bar", 20)).
- add(std::make_pair("baz", 30)).
- add(std::make_pair("foo", 10))),
- "[{item:\"bar\",weight:20},{item:\"baz\",weight:30},{item:\"foo\",weight:10}]");
- }
- { // struct field value
- StructDataType subType("substruct");
- Field fd("d", 0, *DataType::STRING);
- Field fe("e", 1, *DataType::STRING);
- subType.addField(fd);
- subType.addField(fe);
- StructFieldValue subValue(subType);
- subValue.setValue(fd, StringFieldValue("baz"));
- subValue.setValue(fe, StringFieldValue("qux"));
-
- StructDataType type("struct");
- Field fa("a", 0, *DataType::STRING);
- Field fb("b", 1, *DataType::STRING);
- Field fc("c", 2, subType);
- type.addField(fa);
- type.addField(fb);
- type.addField(fc);
- StructFieldValue value(type);
- value.setValue(fa, StringFieldValue("foo"));
- value.setValue(fb, StringFieldValue("bar"));
- value.setValue(fc, subValue);
-
-
- { // select a subset and then all
- SlimeFieldWriter sfw;
- DocsumFieldSpec::FieldIdentifierVector fields;
- {
- FieldPath path;
- type.buildFieldPath(path, "a");
- fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
- }
- {
- FieldPath path;
- type.buildFieldPath(path, "c.e");
- fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
- }
- sfw.setInputFields(fields);
- TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}"));
- sfw.clear();
- TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"b\":\"bar\",\"c\":{\"d\":\"baz\",\"e\":\"qux\"}}"));
- }
-
- { // multiple invocations
- SlimeFieldWriter sfw;
- TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("foo"), "\"foo\""));
- sfw.clear();
- TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("bar"), "\"bar\""));
- sfw.clear();
- TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("baz"), "\"baz\""));
- }
-
- }
-}
-
-void
-DocsumTest::requireThatSlimeFieldWriterHandlesMap()
-{
- { // map<string, string>
- MapDataType mapType(*DataType::STRING, *DataType::STRING);
- MapFieldValue mapfv(mapType);
- EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), StringFieldValue("v1")));
- EXPECT_TRUE(mapfv.put(StringFieldValue("k2"), StringFieldValue("v2")));
- assertSlimeFieldWriter(mapfv, "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]");
- }
- { // map<string, struct>
- StructDataType structType("struct");
- Field fa("a", 0, *DataType::STRING);
- Field fb("b", 1, *DataType::STRING);
- structType.addField(fa);
- structType.addField(fb);
- StructFieldValue structValue(structType);
- structValue.setValue(fa, StringFieldValue("foo"));
- structValue.setValue(fb, StringFieldValue("bar"));
- MapDataType mapType(*DataType::STRING, structType);
- MapFieldValue mapfv(mapType);
- EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), structValue));
- { // select a subset and then all
- SlimeFieldWriter sfw;
- DocsumFieldSpec::FieldIdentifierVector fields;
- {
- FieldPath path;
- mapType.buildFieldPath(path, "value.b");
- fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path)));
- }
- sfw.setInputFields(fields);
- TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]"));
- {
- FieldPath path;
- mapType.buildFieldPath(path, "{k1}.a");
- fields[0] = DocsumFieldSpec::FieldIdentifier(0, std::move(path));
- }
- sfw.clear();
- sfw.setInputFields(fields);
- TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\"}}]"));
- sfw.clear(); // all fields implicit
- TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\",\"b\":\"bar\"}}]"));
- }
- }
-}
-
-int
-DocsumTest::Main()
-{
- TEST_INIT("docsum_test");
-
- TEST_DO(testFlattenDocsumWriter());
- TEST_DO(testSlimeFieldWriter());
- TEST_DO(requireThatSlimeFieldWriterHandlesMap());
-
- TEST_DONE();
-}
-
-}
-
-TEST_APPHOOK(vsm::DocsumTest);
-
diff --git a/streamingvisitors/src/tests/document/.gitignore b/streamingvisitors/src/tests/document/.gitignore
deleted file mode 100644
index d47781eff63..00000000000
--- a/streamingvisitors/src/tests/document/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-document_test
-vsm_document_test_app
diff --git a/streamingvisitors/src/tests/document/CMakeLists.txt b/streamingvisitors/src/tests/document/CMakeLists.txt
deleted file mode 100644
index 5ea12dc5e2d..00000000000
--- a/streamingvisitors/src/tests/document/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vsm_document_test_app TEST
- SOURCES
- document.cpp
- DEPENDS
- streamingvisitors
-)
-vespa_add_test(NAME vsm_document_test_app COMMAND vsm_document_test_app)
diff --git a/streamingvisitors/src/tests/document/document.cpp b/streamingvisitors/src/tests/document/document.cpp
deleted file mode 100644
index 1e97d232a64..00000000000
--- a/streamingvisitors/src/tests/document/document.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/datatype/documenttype.h>
-#include <vespa/vsm/common/storagedocument.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-
-using namespace document;
-
-namespace vsm {
-
-class DocumentTest : public vespalib::TestApp
-{
-private:
- void testStorageDocument();
- void testStringFieldIdTMap();
-public:
- int Main() override;
-};
-
-void
-DocumentTest::testStorageDocument()
-{
- DocumentType dt("testdoc", 0);
-
- Field fa("a", 0, *DataType::STRING);
- Field fb("b", 1, *DataType::STRING);
- dt.addField(fa);
- dt.addField(fb);
-
- document::Document::UP doc(new document::Document(dt, DocumentId()));
- doc->setValue(fa, StringFieldValue("foo"));
- doc->setValue(fb, StringFieldValue("bar"));
-
- SharedFieldPathMap fpmap(new FieldPathMapT());
- fpmap->emplace_back();
- dt.buildFieldPath(fpmap->back(),"a");
- fpmap->emplace_back();
- dt.buildFieldPath(fpmap->back(), "b");
- fpmap->emplace_back();
- ASSERT_TRUE((*fpmap)[0].size() == 1);
- ASSERT_TRUE((*fpmap)[1].size() == 1);
- ASSERT_TRUE((*fpmap)[2].size() == 0);
-
- StorageDocument sdoc(std::move(doc), fpmap, 3);
- ASSERT_TRUE(sdoc.valid());
-
- EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString());
- EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
- EXPECT_TRUE(sdoc.getField(2) == nullptr);
- // test caching
- EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString());
- EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
- EXPECT_TRUE(sdoc.getField(2) == nullptr);
-
- // set new values
- EXPECT_TRUE(sdoc.setField(0, FieldValue::UP(new StringFieldValue("baz"))));
- EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
- EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString());
- EXPECT_TRUE(sdoc.getField(2) == nullptr);
- EXPECT_TRUE(sdoc.setField(1, FieldValue::UP(new StringFieldValue("qux"))));
- EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
- EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString());
- EXPECT_TRUE(sdoc.getField(2) == nullptr);
- EXPECT_TRUE(sdoc.setField(2, FieldValue::UP(new StringFieldValue("quux"))));
- EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString());
- EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString());
- EXPECT_EQUAL(std::string("quux"), sdoc.getField(2)->getAsString());
-
- EXPECT_TRUE(!sdoc.setField(3, FieldValue::UP(new StringFieldValue("thud"))));
-
- SharedFieldPathMap fim;
- StorageDocument s2(std::make_unique<document::Document>(), fim, 0);
- EXPECT_EQUAL(IdString().toString(), s2.docDoc().getId().toString());
-}
-
-void DocumentTest::testStringFieldIdTMap()
-{
- StringFieldIdTMap m;
- EXPECT_EQUAL(0u, m.highestFieldNo());
- EXPECT_TRUE(StringFieldIdTMap::npos == m.fieldNo("unknown"));
- m.add("f1");
- EXPECT_EQUAL(0u, m.fieldNo("f1"));
- EXPECT_EQUAL(1u, m.highestFieldNo());
- m.add("f1");
- EXPECT_EQUAL(0u, m.fieldNo("f1"));
- EXPECT_EQUAL(1u, m.highestFieldNo());
- m.add("f2");
- EXPECT_EQUAL(1u, m.fieldNo("f2"));
- EXPECT_EQUAL(2u, m.highestFieldNo());
- m.add("f3", 7);
- EXPECT_EQUAL(7u, m.fieldNo("f3"));
- EXPECT_EQUAL(8u, m.highestFieldNo());
- m.add("f3");
- EXPECT_EQUAL(7u, m.fieldNo("f3"));
- EXPECT_EQUAL(8u, m.highestFieldNo());
- m.add("f2", 13);
- EXPECT_EQUAL(13u, m.fieldNo("f2"));
- EXPECT_EQUAL(14u, m.highestFieldNo());
- m.add("f4");
- EXPECT_EQUAL(3u, m.fieldNo("f4"));
- EXPECT_EQUAL(14u, m.highestFieldNo());
- {
- vespalib::asciistream os;
- StringFieldIdTMap t;
- t.add("b");
- t.add("a");
- os << t;
- EXPECT_EQUAL(vespalib::string("a = 1\nb = 0\n"), os.str());
- }
-
-}
-
-int
-DocumentTest::Main()
-{
- TEST_INIT("document_test");
-
- testStorageDocument();
- testStringFieldIdTMap();
-
- TEST_DONE();
-}
-
-}
-
-TEST_APPHOOK(vsm::DocumentTest);
-
diff --git a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
index dbec820a462..f25ab348265 100644
--- a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
+++ b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_hitcollector_test_app TEST
SOURCES
hitcollector_test.cpp
DEPENDS
- streamingvisitors
+ streamingvisitors_searchvisitor
)
vespa_add_test(NAME streamingvisitors_hitcollector_test_app COMMAND streamingvisitors_hitcollector_test_app)
diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
index 5cc2977b3c3..ef93d551912 100644
--- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
+++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST
SOURCES
matching_elements_filler_test.cpp
DEPENDS
- streamingvisitors
+ streamingvisitors_searchvisitor
GTest::GTest
)
vespa_add_test(NAME streamingvisitors_matching_elements_filler_test_app COMMAND streamingvisitors_matching_elements_filler_test_app)
diff --git a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
index e0131d0c6cc..7cae60e6a11 100644
--- a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
+++ b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_querywrapper_test_app TEST
SOURCES
querywrapper_test.cpp
DEPENDS
- streamingvisitors
+ streamingvisitors_searchvisitor
)
vespa_add_test(NAME streamingvisitors_querywrapper_test_app COMMAND streamingvisitors_querywrapper_test_app)
diff --git a/streamingvisitors/src/tests/searcher/.gitignore b/streamingvisitors/src/tests/searcher/.gitignore
deleted file mode 100644
index 52a56dff405..00000000000
--- a/streamingvisitors/src/tests/searcher/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-searcher_test
-vsm_searcher_test_app
diff --git a/streamingvisitors/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt
deleted file mode 100644
index 2277f5ef55f..00000000000
--- a/streamingvisitors/src/tests/searcher/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vsm_searcher_test_app TEST
- SOURCES
- searcher_test.cpp
- DEPENDS
- streamingvisitors
-)
-vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app)
diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp
deleted file mode 100644
index 34fa66eaa90..00000000000
--- a/streamingvisitors/src/tests/searcher/searcher_test.cpp
+++ /dev/null
@@ -1,864 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/vsm/searcher/fieldsearcher.h>
-#include <vespa/vsm/searcher/floatfieldsearcher.h>
-#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h>
-#include <vespa/vsm/searcher/intfieldsearcher.h>
-#include <vespa/vsm/searcher/boolfieldsearcher.h>
-#include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h>
-#include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h>
-#include <vespa/vsm/searcher/utf8substringsearcher.h>
-#include <vespa/vsm/searcher/utf8substringsnippetmodifier.h>
-#include <vespa/vsm/searcher/utf8suffixstringfieldsearcher.h>
-#include <vespa/vsm/vsm/snippetmodifier.h>
-#include <vespa/searchlib/query/streaming/queryterm.h>
-#include <vespa/document/fieldvalue/fieldvalues.h>
-
-using namespace document;
-using search::streaming::HitList;
-using search::streaming::QueryNodeResultFactory;
-using search::streaming::QueryTerm;
-using search::streaming::QueryTermList;
-using TermType = QueryTerm::Type;
-using namespace vsm;
-
-template <typename T>
-class Vector : public std::vector<T>
-{
-public:
- Vector() : std::vector<T>() {}
- Vector<T> & add(T v) { this->push_back(v); return *this; }
-};
-
-typedef Vector<size_t> Hits;
-typedef Vector<std::string> StringList;
-typedef Vector<Hits> HitsList;
-typedef Vector<bool> BoolList;
-typedef Vector<int64_t> LongList;
-typedef Vector<float> FloatList;
-typedef QueryTerm::FieldInfo QTFieldInfo;
-typedef Vector<QTFieldInfo> FieldInfoList;
-
-class String
-{
-private:
- const std::string & _str;
-public:
- String(const std::string & str) : _str(str) {}
- bool operator==(const String & rhs) const {
- return _str == rhs._str;
- }
-};
-
-class Query
-{
-private:
- void setupQuery(const StringList & terms) {
- for (size_t i = 0; i < terms.size(); ++i) {
- ParsedQueryTerm pqt = parseQueryTerm(terms[i]);
- ParsedTerm pt = parseTerm(pqt.second);
- qtv.push_back(std::make_unique<QueryTerm>(eqnr.create(), pt.first, pqt.first.empty() ? "index" : pqt.first, pt.second));
- }
- for (size_t i = 0; i < qtv.size(); ++i) {
- qtl.push_back(qtv[i].get());
- }
- }
-public:
- typedef std::pair<std::string, std::string> ParsedQueryTerm;
- typedef std::pair<std::string, TermType> ParsedTerm;
- QueryNodeResultFactory eqnr;
- std::vector<QueryTerm::UP> qtv;
- QueryTermList qtl;
- Query(const StringList & terms);
- ~Query();
- static ParsedQueryTerm parseQueryTerm(const std::string & queryTerm) {
- size_t i = queryTerm.find(':');
- if (i != std::string::npos) {
- return ParsedQueryTerm(queryTerm.substr(0, i), queryTerm.substr(i + 1));
- }
- return ParsedQueryTerm(std::string(), queryTerm);
- }
- static ParsedTerm parseTerm(const std::string & term) {
- if (term[0] == '*' && term[term.size() - 1] == '*') {
- return std::make_pair(term.substr(1, term.size() - 2), TermType::SUBSTRINGTERM);
- } else if (term[0] == '*') {
- return std::make_pair(term.substr(1, term.size() - 1), TermType::SUFFIXTERM);
- } else if (term[term.size() - 1] == '*') {
- return std::make_pair(term.substr(0, term.size() - 1), TermType::PREFIXTERM);
- } else {
- return std::make_pair(term, TermType::WORD);
- }
- }
-};
-
-Query::Query(const StringList & terms) : eqnr(), qtv(), qtl() {
- setupQuery(terms);
-}
-Query::~Query() = default;
-
-struct SnippetModifierSetup
-{
- Query query;
- UTF8SubstringSnippetModifier::SP searcher;
- SharedSearcherBuf buf;
- SnippetModifier modifier;
- explicit SnippetModifierSetup(const StringList & terms);
- ~SnippetModifierSetup();
-};
-
-SnippetModifierSetup::SnippetModifierSetup(const StringList & terms)
- : query(terms),
- searcher(new UTF8SubstringSnippetModifier()),
- buf(new SearcherBuf(8)),
- modifier(searcher)
-{
- searcher->prepare(query.qtl, buf);
-}
-SnippetModifierSetup::~SnippetModifierSetup() = default;
-
-// helper functions
-ArrayFieldValue getFieldValue(const StringList &fv);
-ArrayFieldValue getFieldValue(const LongList &fv);
-ArrayFieldValue getFieldValue(const FloatList &fv);
-
-bool assertMatchTermSuffix(const std::string &term, const std::string &word);
-void assertSnippetModifier(const StringList &query, const std::string &fv, const std::string &exp);
-void assertSnippetModifier(SnippetModifierSetup &setup, const FieldValue &fv, const std::string &exp);
-void assertQueryTerms(const SnippetModifierManager &man, FieldIdT fId, const StringList &terms);
-void assertNumeric(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const BoolList &exp);
-std::vector<QueryTerm::UP> performSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv);
-void assertSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const HitsList &exp);
-bool assertCountWords(size_t numWords, const std::string &field);
-bool assertFieldInfo(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const FieldInfoList &exp);
-
-void assertString(StrChrFieldSearcher &fs, const StringList &query, const std::string &field, const HitsList &exp) {
- assertSearch(fs, query, StringFieldValue(field), exp);
-}
-
-void assertString(StrChrFieldSearcher &fs, const StringList &query, const StringList &field, const HitsList &exp) {
- assertSearch(fs, query, getFieldValue(field), exp);
-}
-
-void assertString(StrChrFieldSearcher &fs, const std::string &term, const std::string &field, const Hits &exp) {
- assertString(fs, StringList().add(term), field, HitsList().add(exp));
-}
-void assertString(StrChrFieldSearcher &fs, const std::string &term, const StringList &field, const Hits &exp) {
- assertString(fs, StringList().add(term), field, HitsList().add(exp));
-}
-
-void assertInt(IntFieldSearcher & fs, const StringList &query, int64_t field, const BoolList &exp) {
- assertNumeric(fs, query, LongFieldValue(field), exp);
-}
-
-void assertInt(IntFieldSearcher & fs, const std::string &term, int64_t field, bool exp) {
- assertInt(fs, StringList().add(term), field, BoolList().add(exp));
-}
-
-void assertBool(BoolFieldSearcher & fs, const StringList &query, bool field, const BoolList &exp) {
- assertNumeric(fs, query, BoolFieldValue(field), exp);
-}
-void assertBool(BoolFieldSearcher & fs, const std::string &term, bool field, bool exp) {
- assertBool(fs, StringList().add(term), field, BoolList().add(exp));
-}
-
-void assertInt(IntFieldSearcher & fs, const StringList &query, const LongList &field, const HitsList &exp) {
- assertSearch(fs, query, getFieldValue(field), exp);
-}
-
-void assertInt(IntFieldSearcher & fs, const std::string &term, const LongList &field, const Hits &exp) {
- assertInt(fs, StringList().add(term), field, HitsList().add(exp));
-}
-
-void assertFloat(FloatFieldSearcher & fs, const StringList &query, float field, const BoolList &exp) {
- assertNumeric(fs, query, FloatFieldValue(field), exp);
-}
-
-void assertFloat(FloatFieldSearcher & fs, const std::string &term, float field, bool exp) {
- assertFloat(fs, StringList().add(term), field, BoolList().add(exp));
-}
-
-void assertFloat(FloatFieldSearcher & fs, const StringList &query, const FloatList &field, const HitsList &exp) {
- assertSearch(fs, query, getFieldValue(field), exp);
-}
-
-void assertFloat(FloatFieldSearcher & fs, const std::string &term, const FloatList &field, const Hits &exp) {
- assertFloat(fs, StringList().add(term), field, HitsList().add(exp));
-}
-
-bool
-assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const std::string &fv, const FieldInfoList &exp) {
- return assertFieldInfo(fs, query, StringFieldValue(fv), exp);
-}
-
-bool
-assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const StringList &fv, const FieldInfoList &exp) {
- return assertFieldInfo(fs, query, getFieldValue(fv), exp);
-}
-bool
-assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const StringList &fv, const QTFieldInfo &exp) {
- return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-bool
-assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const std::string &fv, const QTFieldInfo &exp) {
- return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, int64_t fv, const FieldInfoList &exp) {
- assertFieldInfo(fs, query, LongFieldValue(fv), exp);
-}
-
-void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, const LongList &fv, const FieldInfoList &exp) {
- assertFieldInfo(fs, query, getFieldValue(fv), exp);
-}
-
-void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, int64_t fv, const QTFieldInfo &exp) {
- assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, const LongList &fv, const QTFieldInfo &exp) {
- assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-void assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, float fv, const FieldInfoList &exp) {
- assertFieldInfo(fs, query, FloatFieldValue(fv), exp);
-}
-
-void
-assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, const FloatList &fv, const FieldInfoList &exp) {
- assertFieldInfo(fs, query, getFieldValue(fv), exp);
-}
-
-/** float field searcher **/
-void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, float fv, const QTFieldInfo &exp) {
- assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, const FloatList &fv, const QTFieldInfo &exp) {
- assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp));
-}
-
-
-/** snippet modifer searcher **/
-void assertSnippetModifier(const std::string &term, const std::string &fv, const std::string &exp) {
- assertSnippetModifier(StringList().add(term), fv, exp);
-}
-
-
-ArrayFieldValue
-getFieldValue(const StringList & fv)
-{
-
- static ArrayDataType type(*DataType::STRING);
- ArrayFieldValue afv(type);
- for (size_t i = 0; i < fv.size(); ++i) {
- afv.add(StringFieldValue(fv[i]));
- }
- return afv;
-}
-
-ArrayFieldValue
-getFieldValue(const LongList & fv)
-{
- static ArrayDataType type(*DataType::LONG);
- ArrayFieldValue afv(type);
- for (size_t i = 0; i < fv.size(); ++i) {
- afv.add(LongFieldValue(fv[i]));
- }
- return afv;
-}
-
-ArrayFieldValue
-getFieldValue(const FloatList & fv)
-{
- static ArrayDataType type(*DataType::FLOAT);
- ArrayFieldValue afv(type);
- for (size_t i = 0; i < fv.size(); ++i) {
- afv.add(FloatFieldValue(fv[i]));
- }
- return afv;
-}
-
-bool
-assertMatchTermSuffix(const std::string & term, const std::string & word)
-{
- QueryNodeResultFactory eqnr;
- QueryTerm qa(eqnr.create(), term, "index", TermType::WORD);
- QueryTerm qb(eqnr.create(), word, "index", TermType::WORD);
- const ucs4_t * a;
- size_t alen = qa.term(a);
- const ucs4_t * b;
- size_t blen = qb.term(b);
- return UTF8StringFieldSearcherBase::matchTermSuffix(a, alen, b, blen);
-}
-
-void
-assertNumeric(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const BoolList & exp)
-{
- HitsList hl;
- for (size_t i = 0; i < exp.size(); ++i) {
- hl.push_back(exp[i] ? Hits().add(0) : Hits());
- }
- assertSearch(fs, query, fv, hl);
-}
-
-std::vector<QueryTerm::UP>
-performSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv)
-{
- Query q(query);
-
- // prepare field searcher
- SharedSearcherBuf ssb = SharedSearcherBuf(new SearcherBuf());
- fs.prepare(q.qtl, ssb);
-
- // setup document
- SharedFieldPathMap sfim(new FieldPathMapT());
- sfim->push_back(FieldPath());
- StorageDocument doc(std::make_unique<document::Document>(), sfim, 1);
- doc.setField(0, document::FieldValue::UP(fv.clone()));
-
- fs.search(doc);
- return std::move(q.qtv);
-}
-
-void
-assertSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const HitsList & exp)
-{
- auto qtv = performSearch(fs, query, fv);
- EXPECT_EQUAL(qtv.size(), exp.size());
- ASSERT_TRUE(qtv.size() == exp.size());
- for (size_t i = 0; i < qtv.size(); ++i) {
- const HitList & hl = qtv[i]->getHitList();
- EXPECT_EQUAL(hl.size(), exp[i].size());
- ASSERT_TRUE(hl.size() == exp[i].size());
- for (size_t j = 0; j < hl.size(); ++j) {
- EXPECT_EQUAL((size_t)hl[j].pos(), exp[i][j]);
- }
- }
-}
-
-bool
-assertFieldInfo(FieldSearcher & fs, const StringList & query,
- const FieldValue & fv, const FieldInfoList & exp)
-{
- auto qtv = performSearch(fs, query, fv);
- if (!EXPECT_EQUAL(qtv.size(), exp.size())) return false;
- bool retval = true;
- for (size_t i = 0; i < qtv.size(); ++i) {
- if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitOffset(), exp[i].getHitOffset())) retval = false;
- if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitCount(), exp[i].getHitCount())) retval = false;
- if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getFieldLength(), exp[i].getFieldLength())) retval = false;
- }
- return retval;
-}
-
-void
-assertSnippetModifier(const StringList & query, const std::string & fv, const std::string & exp)
-{
- UTF8SubstringSnippetModifier mod;
- performSearch(mod, query, StringFieldValue(fv));
- EXPECT_EQUAL(mod.getModifiedBuf().getPos(), exp.size());
- std::string actual(mod.getModifiedBuf().getBuffer(), mod.getModifiedBuf().getPos());
- EXPECT_EQUAL(actual.size(), exp.size());
- EXPECT_EQUAL(actual, exp);
-}
-
-void assertSnippetModifier(SnippetModifierSetup & setup, const FieldValue & fv, const std::string & exp)
-{
- FieldValue::UP mfv = setup.modifier.modify(fv);
- const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(*mfv.get());
- const std::string & actual = lfv.getValue();
- EXPECT_EQUAL(actual.size(), exp.size());
- EXPECT_EQUAL(actual, exp);
-}
-
-void assertQueryTerms(const SnippetModifierManager & man, FieldIdT fId, const StringList & terms)
-{
- if (terms.size() == 0) {
- ASSERT_TRUE(man.getModifiers().getModifier(fId) == NULL);
- return;
- }
- ASSERT_TRUE(man.getModifiers().getModifier(fId) != NULL);
- UTF8SubstringSnippetModifier * searcher =
- (static_cast<SnippetModifier *>(man.getModifiers().getModifier(fId)))->getSearcher().get();
- EXPECT_EQUAL(searcher->getQueryTerms().size(), terms.size());
- ASSERT_TRUE(searcher->getQueryTerms().size() == terms.size());
- for (size_t i = 0; i < terms.size(); ++i) {
- EXPECT_EQUAL(std::string(searcher->getQueryTerms()[i]->getTerm()), terms[i]);
- }
-}
-
-bool assertCountWords(size_t numWords, const std::string & field)
-{
- FieldRef ref(field.c_str(), field.size());
- return EXPECT_EQUAL(numWords, FieldSearcher::countWords(ref));
-}
-
-bool
-testStringFieldInfo(StrChrFieldSearcher & fs)
-{
- assertString(fs, "foo", StringList().add("foo bar baz").add("foo bar").add("baz foo"), Hits().add(0).add(3).add(6));
- assertString(fs, StringList().add("foo").add("bar"), StringList().add("foo bar baz").add("foo bar").add("baz foo"),
- HitsList().add(Hits().add(0).add(3).add(6)).add(Hits().add(1).add(4)));
-
- bool retval = true;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo", QTFieldInfo(0, 1, 1)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo", QTFieldInfo(0, 0, 1)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "baz", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "qux", "foo bar baz", QTFieldInfo(0, 0, 3)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo foo foo", QTFieldInfo(0, 3, 3)))) retval = false;
- // query term size > last term size
- if (!EXPECT_TRUE(assertFieldInfo(fs, "runner", "Road Runner Disco", QTFieldInfo(0, 1, 3)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("roadrun").add("runner"), "Road Runner Disco",
- FieldInfoList().add(QTFieldInfo(0, 0, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false;
- // multiple terms
- if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", StringList().add("foo bar baz").add("foo bar"),
- QTFieldInfo(0, 2, 5)))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), "foo bar baz",
- FieldInfoList().add(QTFieldInfo(0, 1, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false;
- if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), StringList().add("foo bar baz").add("foo bar"),
- FieldInfoList().add(QTFieldInfo(0, 2, 5)).add(QTFieldInfo(0, 1, 5))))) retval = false;
- return retval;
-}
-bool
-testStrChrFieldSearcher(StrChrFieldSearcher & fs)
-{
- std::string field = "operators and operator overloading with utf8 char oe = \xc3\x98";
- assertString(fs, "oper", field, Hits());
- assertString(fs, "tor", field, Hits());
- assertString(fs, "oper*", field, Hits().add(0).add(2));
- assertString(fs, "and", field, Hits().add(1));
-
- assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits()).add(Hits()));
- assertString(fs, StringList().add("and").add("overloading"), field, HitsList().add(Hits().add(1)).add(Hits().add(3)));
-
- fs.setMatchType(FieldSearcher::PREFIX);
- assertString(fs, "oper", field, Hits().add(0).add(2));
- assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits().add(0).add(2)).add(Hits()));
-
- fs.setMatchType(FieldSearcher::REGULAR);
- if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false;
-
- { // test handling of several underscores
- StringList query = StringList().add("foo").add("bar");
- HitsList exp = HitsList().add(Hits().add(0)).add(Hits().add(1));
- assertString(fs, query, "foo_bar", exp);
- assertString(fs, query, "foo__bar", exp);
- assertString(fs, query, "foo___bar", exp);
- assertString(fs, query, "foo________bar", exp);
- assertString(fs, query, "foo____________________bar", exp);
- assertString(fs, query, "________________________________________foo________________________________________bar________________________________________", exp);
- query = StringList().add("foo").add("thisisaveryveryverylongword");
- assertString(fs, query, "foo____________________thisisaveryveryverylongword", exp);
-
- assertString(fs, "bar", "foo bar", Hits().add(1));
- assertString(fs, "bar", "foo____________________bar", Hits().add(1));
- assertString(fs, "bar", "foo____________________thisisaveryveryverylongword____________________bar", Hits().add(2));
- }
- return true;
-}
-
- TEST("verify correct term parsing") {
- ASSERT_TRUE(Query::parseQueryTerm("index:term").first == "index");
- ASSERT_TRUE(Query::parseQueryTerm("index:term").second == "term");
- ASSERT_TRUE(Query::parseQueryTerm("term").first == "");
- ASSERT_TRUE(Query::parseQueryTerm("term").second == "term");
- ASSERT_TRUE(Query::parseTerm("*substr*").first == "substr");
- ASSERT_TRUE(Query::parseTerm("*substr*").second == TermType::SUBSTRINGTERM);
- ASSERT_TRUE(Query::parseTerm("*suffix").first == "suffix");
- ASSERT_TRUE(Query::parseTerm("*suffix").second == TermType::SUFFIXTERM);
- ASSERT_TRUE(Query::parseTerm("prefix*").first == "prefix");
- ASSERT_TRUE(Query::parseTerm("prefix*").second == TermType::PREFIXTERM);
- ASSERT_TRUE(Query::parseTerm("term").first == "term");
- ASSERT_TRUE(Query::parseTerm("term").second == TermType::WORD);
- }
-
- TEST("suffix matching") {
- EXPECT_EQUAL(assertMatchTermSuffix("a", "vespa"), true);
- EXPECT_EQUAL(assertMatchTermSuffix("spa", "vespa"), true);
- EXPECT_EQUAL(assertMatchTermSuffix("vespa", "vespa"), true);
- EXPECT_EQUAL(assertMatchTermSuffix("vvespa", "vespa"), false);
- EXPECT_EQUAL(assertMatchTermSuffix("fspa", "vespa"), false);
- EXPECT_EQUAL(assertMatchTermSuffix("v", "vespa"), false);
- }
-
-TEST("Test basic strchrfield searchers") {
- {
- UTF8StrChrFieldSearcher fs(0);
- EXPECT_TRUE(testStrChrFieldSearcher(fs));
- }
- {
- FUTF8StrChrFieldSearcher fs(0);
- EXPECT_TRUE(testStrChrFieldSearcher(fs));
- }
-}
-
-bool
-testUTF8SubStringFieldSearcher(StrChrFieldSearcher & fs)
-{
- std::string field = "operators and operator overloading";
- assertString(fs, "rsand", field, Hits());
- assertString(fs, "ove", field, Hits().add(3));
- assertString(fs, "ing", field, Hits().add(3));
- assertString(fs, "era", field, Hits().add(0).add(2));
- assertString(fs, "a", field, Hits().add(0).add(1).add(2).add(3));
-
- assertString(fs, StringList().add("dn").add("gn"), field, HitsList().add(Hits()).add(Hits()));
- assertString(fs, StringList().add("ato").add("load"), field, HitsList().add(Hits().add(0).add(2)).add(Hits().add(3)));
-
- assertString(fs, StringList().add("aa").add("ab"), "aaaab",
- HitsList().add(Hits().add(0).add(0).add(0)).add(Hits().add(0)));
-
- if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false;
- return true;
-}
-
-TEST("utf8 substring search") {
- {
- UTF8SubStringFieldSearcher fs(0);
- EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
- assertString(fs, "aa", "aaaa", Hits().add(0).add(0));
- }
- {
- UTF8SubStringFieldSearcher fs(0);
- EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
- assertString(fs, "abc", "abc bcd abc", Hits().add(0).add(2));
- fs.maxFieldLength(4);
- assertString(fs, "abc", "abc bcd abc", Hits().add(0));
- }
- {
- UTF8SubstringSnippetModifier fs(0);
- EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
- // we don't have 1 term optimization
- assertString(fs, "aa", "aaaa", Hits().add(0).add(0).add(0));
- }
-}
-
-TEST("utf8 substring search with empty term")
-{
- UTF8SubStringFieldSearcher fs(0);
- EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs));
- assertString(fs, "", "abc", Hits());
- assertFieldInfo(fs, "", "abc", QTFieldInfo().setFieldLength(0));
-}
-
-TEST("utf8 suffix search") {
- UTF8SuffixStringFieldSearcher fs(0);
- std::string field = "operators and operator overloading";
- assertString(fs, "rsand", field, Hits());
- assertString(fs, "tor", field, Hits().add(2));
- assertString(fs, "tors", field, Hits().add(0));
-
- assertString(fs, StringList().add("an").add("din"), field, HitsList().add(Hits()).add(Hits()));
- assertString(fs, StringList().add("nd").add("g"), field, HitsList().add(Hits().add(1)).add(Hits().add(3)));
-
- EXPECT_TRUE(testStringFieldInfo(fs));
-}
-
-TEST("utf8 exact match") {
- UTF8ExactStringFieldSearcher fs(0);
- // regular
- TEST_DO(assertString(fs, "vespa", "vespa", Hits().add(0)));
- TEST_DO(assertString(fs, "vespar", "vespa", Hits()));
- TEST_DO(assertString(fs, "vespa", "vespar", Hits()));
- TEST_DO(assertString(fs, "vespa", "vespa vespa", Hits()));
- TEST_DO(assertString(fs, "vesp", "vespa", Hits()));
- TEST_DO(assertString(fs, "vesp*", "vespa", Hits().add(0)));
- TEST_DO(assertString(fs, "hutte", "hutte", Hits().add(0)));
- TEST_DO(assertString(fs, "hütte", "hütte", Hits().add(0)));
- TEST_DO(assertString(fs, "hutte", "hütte", Hits()));
- TEST_DO(assertString(fs, "hütte", "hutte", Hits()));
- TEST_DO(assertString(fs, "hütter", "hütte", Hits()));
- TEST_DO(assertString(fs, "hütte", "hütter", Hits()));
-}
-
-TEST("utf8 flexible searcher"){
- UTF8FlexibleStringFieldSearcher fs(0);
- // regular
- assertString(fs, "vespa", "vespa", Hits().add(0));
- assertString(fs, "vesp", "vespa", Hits());
- assertString(fs, "esp", "vespa", Hits());
- assertString(fs, "espa", "vespa", Hits());
-
- // prefix
- assertString(fs, "vesp*", "vespa", Hits().add(0));
- fs.setMatchType(FieldSearcher::PREFIX);
- assertString(fs, "vesp", "vespa", Hits().add(0));
-
- // substring
- fs.setMatchType(FieldSearcher::REGULAR);
- assertString(fs, "*esp*", "vespa", Hits().add(0));
- fs.setMatchType(FieldSearcher::SUBSTRING);
- assertString(fs, "esp", "vespa", Hits().add(0));
-
- // suffix
- fs.setMatchType(FieldSearcher::REGULAR);
- assertString(fs, "*espa", "vespa", Hits().add(0));
- fs.setMatchType(FieldSearcher::SUFFIX);
- assertString(fs, "espa", "vespa", Hits().add(0));
-
- fs.setMatchType(FieldSearcher::REGULAR);
- EXPECT_TRUE(testStringFieldInfo(fs));
-}
-
-TEST("bool search") {
- BoolFieldSearcher fs(0);
- TEST_DO(assertBool(fs, "true", true, true));
- TEST_DO(assertBool(fs, "true", false, false));
- TEST_DO(assertBool(fs, "1", true, true));
- TEST_DO(assertBool(fs, "1", false, false));
- TEST_DO(assertBool(fs, "false", true, false));
- TEST_DO(assertBool(fs, "false", false, true));
- TEST_DO(assertBool(fs, "0", true, false));
- TEST_DO(assertBool(fs, "0", false, true));
- TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"), true, BoolList().add(true).add(false).add(true)));
- TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"), false, BoolList().add(false).add(true).add(false)));
-}
-
-TEST("integer search")
-{
- IntFieldSearcher fs(0);
- TEST_DO(assertInt(fs, "10", 10, true));
- TEST_DO(assertInt(fs, "9", 10, false));
- TEST_DO(assertInt(fs, ">9", 10, true));
- TEST_DO(assertInt(fs, ">9", 9, false));
- TEST_DO(assertInt(fs, "<11", 10, true));
- TEST_DO(assertInt(fs, "<11", 11, false));
- TEST_DO(assertInt(fs, "-10", -10, true));
- TEST_DO(assertInt(fs, "-9", -10, false));
- TEST_DO(assertInt(fs, "a", 10, false));
- TEST_DO(assertInt(fs, "[-5;5]", -5, true));
- TEST_DO(assertInt(fs, "[-5;5]", 0, true));
- TEST_DO(assertInt(fs, "[-5;5]", 5, true));
- TEST_DO(assertInt(fs, "[-5;5]", -6, false));
- TEST_DO(assertInt(fs, "[-5;5]", 6, false));
-
- TEST_DO(assertInt(fs, StringList().add("9").add("11"), 10, BoolList().add(false).add(false)));
- TEST_DO(assertInt(fs, StringList().add("9").add("10"), 10, BoolList().add(false).add(true)));
- TEST_DO(assertInt(fs, StringList().add("10").add(">9"), 10, BoolList().add(true).add(true)));
-
- TEST_DO(assertInt(fs, "10", LongList().add(10).add(20).add(10).add(30), Hits().add(0).add(2)));
- TEST_DO(assertInt(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30),
- HitsList().add(Hits().add(0).add(2)).add(Hits().add(1))));
-
- TEST_DO(assertFieldInfo(fs, "10", 10, QTFieldInfo(0, 1, 1)));
- TEST_DO(assertFieldInfo(fs, "10", LongList().add(10).add(20).add(10).add(30), QTFieldInfo(0, 2, 4)));
- TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), 10,
- FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1))));
- TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30),
- FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4))));
-}
-
-TEST("floating point search")
-{
- FloatFieldSearcher fs;
- TEST_DO(assertFloat(fs, "10", 10, true));
- TEST_DO(assertFloat(fs, "10.5", 10.5, true));
- TEST_DO(assertFloat(fs, "-10.5", -10.5, true));
- TEST_DO(assertFloat(fs, ">10.5", 10.6, true));
- TEST_DO(assertFloat(fs, ">10.5", 10.5, false));
- TEST_DO(assertFloat(fs, "<10.5", 10.4, true));
- TEST_DO(assertFloat(fs, "<10.5", 10.5, false));
- TEST_DO(assertFloat(fs, "10.4", 10.5, false));
- TEST_DO(assertFloat(fs, "-10.4", -10.5, false));
- TEST_DO(assertFloat(fs, "a", 10.5, false));
- TEST_DO(assertFloat(fs, "[-5.5;5.5]", -5.5, true));
- TEST_DO(assertFloat(fs, "[-5.5;5.5]", 0, true));
- TEST_DO(assertFloat(fs, "[-5.5;5.5]", 5.5, true));
- TEST_DO(assertFloat(fs, "[-5.5;5.5]", -5.6, false));
- TEST_DO(assertFloat(fs, "[-5.5;5.5]", 5.6, false));
-
- TEST_DO(assertFloat(fs, StringList().add("10").add("11"), 10.5, BoolList().add(false).add(false)));
- TEST_DO(assertFloat(fs, StringList().add("10").add("10.5"), 10.5, BoolList().add(false).add(true)));
- TEST_DO(assertFloat(fs, StringList().add(">10.4").add("10.5"), 10.5, BoolList().add(true).add(true)));
-
- TEST_DO(assertFloat(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), Hits().add(0).add(2)));
- TEST_DO(assertFloat(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5),
- HitsList().add(Hits().add(0).add(2)).add(Hits().add(1))));
-
- TEST_DO(assertFieldInfo(fs, "10.5", 10.5, QTFieldInfo(0, 1, 1)));
- TEST_DO(assertFieldInfo(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), QTFieldInfo(0, 2, 4)));
- TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), 10.5,
- FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1))));
- TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5),
- FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4))));
-}
-
-TEST("Snippet modifier search") {
- // ascii
- assertSnippetModifier("f", "foo", "\x1F""f\x1Foo");
- assertSnippetModifier("o", "foo", "f\x1Fo\x1F\x1Fo\x1F");
- assertSnippetModifier("r", "bar", "ba\x1Fr\x1F");
- assertSnippetModifier("foo", "foo foo", "\x1F""foo\x1F \x1F""foo\x1F");
- assertSnippetModifier("aa", "aaaaaa", "\x1F""aa\x1F\x1F""aa\x1F\x1F""aa\x1F");
- assertSnippetModifier("ab", "abcd\x1F""efgh", "\x1F""ab\x1F""cd\x1F""efgh");
- assertSnippetModifier("ef", "abcd\x1F""efgh", "abcd\x1F\x1F""ef\x1Fgh");
- assertSnippetModifier("fg", "abcd\x1F""efgh", "abcd\x1F""e\x1F""fg\x1Fh");
- // the separator overlapping the match is skipped
- assertSnippetModifier("cdef", "abcd\x1F""efgh", "ab\x1F""cdef\x1F""gh");
- // no hits
- assertSnippetModifier("bb", "aaaaaa", "aaaaaa");
-
-
- // multiple query terms
- assertSnippetModifier(StringList().add("ab").add("cd"), "abcd", "\x1F""ab\x1F\x1F""cd\x1F");
- // when we have overlap we only get the first match
- assertSnippetModifier(StringList().add("ab").add("bc"), "abcd", "\x1F""ab\x1F""cd");
- assertSnippetModifier(StringList().add("bc").add("ab"), "abcd", "\x1F""ab\x1F""cd");
- // the separator overlapping the match is skipped
- assertSnippetModifier(StringList().add("de").add("ef"), "abcd\x1F""efgh", "abc\x1F""de\x1F""fgh");
-
- // cjk
- assertSnippetModifier("\xe7\x9f\xb3", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8",
- "\x1f\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8");
- assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8",
- "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8");
- // the separator overlapping the match is skipped
- assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\x1f\xe5\x87\xb1\xe5\x9c\xa8",
- "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8");
-
- { // check that resizing works
- UTF8SubstringSnippetModifier mod;
- EXPECT_EQUAL(mod.getModifiedBuf().getLength(), 32u);
- EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 0u);
- performSearch(mod, StringList().add("a"), StringFieldValue("aaaaaaaaaaaaaaaa"));
- EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 16u + 2 * 16u);
- EXPECT_TRUE(mod.getModifiedBuf().getLength() >= mod.getModifiedBuf().getPos());
- }
-}
-
-TEST("snippet modifier") {
- { // string field value
- SnippetModifierSetup sms(StringList().add("ab"));
- // multiple invokations
- assertSnippetModifier(sms, StringFieldValue("ab"), "\x1F""ab\x1F");
- assertSnippetModifier(sms, StringFieldValue("xxxxabxxxxabxxxx"), "xxxx\x1F""ab\x1Fxxxx\x1F""ab\x1Fxxxx");
- assertSnippetModifier(sms, StringFieldValue("xxabxx"), "xx\x1F""ab\x1Fxx");
- }
- { // collection field value
- SnippetModifierSetup sms(StringList().add("ab"));
- // multiple invokations
- assertSnippetModifier(sms, getFieldValue(StringList().add("ab")), "\x1F""ab\x1F");
- assertSnippetModifier(sms, getFieldValue(StringList().add("xxabxx")), "xx\x1F""ab\x1Fxx");
- assertSnippetModifier(sms, getFieldValue(StringList().add("ab").add("xxabxx").add("xxxxxx")),
- "\x1F""ab\x1F\x1E""xx\x1F""ab\x1F""xx\x1E""xxxxxx");
- assertSnippetModifier(sms, getFieldValue(StringList().add("cd").add("ef").add("gh")),
- "cd\x1E""ef\x1E""gh");
- }
- { // check that resizing works
- SnippetModifierSetup sms(StringList().add("a"));
- EXPECT_EQUAL(sms.modifier.getValueBuf().getLength(), 32u);
- EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 0u);
- sms.modifier.modify(StringFieldValue("aaaaaaaaaaaaaaaa"));
- EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 16u + 2 * 16u);
- EXPECT_TRUE(sms.modifier.getValueBuf().getLength() >= sms.modifier.getValueBuf().getPos());
- }
-}
-
-TEST("FieldSearchSpec constrution") {
- {
- FieldSearchSpec f;
- EXPECT_FALSE(f.valid());
- EXPECT_EQUAL(0u, f.id());
- EXPECT_EQUAL("", f.name());
- EXPECT_EQUAL(0x100000u, f.maxLength());
- }
- {
- FieldSearchSpec f(7, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 789);
- EXPECT_TRUE(f.valid());
- EXPECT_EQUAL(7u, f.id());
- EXPECT_EQUAL("f0", f.name());
- EXPECT_EQUAL(789u, f.maxLength());
- EXPECT_EQUAL(789u, f.searcher().maxFieldLength());
- }
-}
-
-TEST("snippet modifier manager") {
- FieldSearchSpecMapT specMap;
- specMap[0] = FieldSearchSpec(0, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 1000);
- specMap[1] = FieldSearchSpec(1, "f1", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "", 1000);
- IndexFieldMapT indexMap;
- indexMap["i0"].push_back(0);
- indexMap["i1"].push_back(1);
- indexMap["i2"].push_back(0);
- indexMap["i2"].push_back(1);
-
- {
- SnippetModifierManager man;
- Query query(StringList().add("i0:foo"));
- man.setup(query.qtl, specMap, indexMap);
- assertQueryTerms(man, 0, StringList().add("foo"));
- assertQueryTerms(man, 1, StringList());
- }
- {
- SnippetModifierManager man;
- Query query(StringList().add("i1:foo"));
- man.setup(query.qtl, specMap, indexMap);
- assertQueryTerms(man, 0, StringList());
- assertQueryTerms(man, 1, StringList());
- }
- {
- SnippetModifierManager man;
- Query query(StringList().add("i1:*foo*"));
- man.setup(query.qtl, specMap, indexMap);
- assertQueryTerms(man, 0, StringList());
- assertQueryTerms(man, 1, StringList().add("foo"));
- }
- {
- SnippetModifierManager man;
- Query query(StringList().add("i2:foo").add("i2:*bar*"));
- man.setup(query.qtl, specMap, indexMap);
- assertQueryTerms(man, 0, StringList().add("foo").add("bar"));
- assertQueryTerms(man, 1, StringList().add("bar"));
- }
- { // check buffer sizes
- SnippetModifierManager man;
- Query query(StringList().add("i2:foo").add("i2:*bar*"));
- man.setup(query.qtl, specMap, indexMap);
- {
- SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(0));
- UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get();
- EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u);
- EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u);
- }
- {
- SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(1));
- UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get();
- EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u);
- EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u);
- }
- }
-}
-
-TEST("Stripping of indexes")
-{
- EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f"));
- EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f[0]"));
- EXPECT_EQUAL("f[a]", FieldSearchSpecMap::stripNonFields("f[a]"));
-
- EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a}"));
- EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a0}"));
- EXPECT_EQUAL("f{a 0}", FieldSearchSpecMap::stripNonFields("f{a 0}"));
- EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{\"a 0\"}"));
-}
-
-TEST("counting of words") {
- EXPECT_TRUE(assertCountWords(0, ""));
- EXPECT_TRUE(assertCountWords(0, "?"));
- EXPECT_TRUE(assertCountWords(1, "foo"));
- EXPECT_TRUE(assertCountWords(2, "foo bar"));
- EXPECT_TRUE(assertCountWords(2, "? foo bar"));
- EXPECT_TRUE(assertCountWords(2, "foo bar ?"));
-
- // check that 'a' is counted as 1 word
- UTF8StrChrFieldSearcher fs(0);
- StringList field = StringList().add("a").add("aa bb cc");
- assertString(fs, "bb", field, Hits().add(2));
- assertString(fs, StringList().add("bb").add("not"), field, HitsList().add(Hits().add(2)).add(Hits()));
-}
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
index 01b625b6b3b..fdbd60ce30a 100644
--- a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
+++ b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt
@@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_searchvisitor_test_app TEST
SOURCES
searchvisitor_test.cpp
DEPENDS
- streamingvisitors
+ streamingvisitors_searchvisitor
)
vespa_add_test(NAME streamingvisitors_searchvisitor_test_app COMMAND streamingvisitors_searchvisitor_test_app)
diff --git a/streamingvisitors/src/tests/textutil/.gitignore b/streamingvisitors/src/tests/textutil/.gitignore
deleted file mode 100644
index 1103f79800a..00000000000
--- a/streamingvisitors/src/tests/textutil/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-textutil_test
-vsm_textutil_test_app
diff --git a/streamingvisitors/src/tests/textutil/CMakeLists.txt b/streamingvisitors/src/tests/textutil/CMakeLists.txt
deleted file mode 100644
index 59817d01137..00000000000
--- a/streamingvisitors/src/tests/textutil/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vsm_textutil_test_app TEST
- SOURCES
- textutil.cpp
- DEPENDS
- streamingvisitors
-)
-vespa_add_test(NAME vsm_textutil_test_app COMMAND vsm_textutil_test_app)
diff --git a/streamingvisitors/src/tests/textutil/textutil.cpp b/streamingvisitors/src/tests/textutil/textutil.cpp
deleted file mode 100644
index 2a1390eaa01..00000000000
--- a/streamingvisitors/src/tests/textutil/textutil.cpp
+++ /dev/null
@@ -1,285 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/fastlib/text/unicodeutil.h>
-#include <vespa/searchlib/query/base.h>
-#include <vespa/vsm/searcher/fold.h>
-#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h>
-#include <vespa/vsm/searcher/utf8stringfieldsearcherbase.h>
-
-using search::byte; // unsigned char
-
-namespace vsm {
-
-template <typename T>
-class Vector : public std::vector<T>
-{
-public:
- Vector() : std::vector<T>() {}
- Vector<T> & a(T v) { this->push_back(v); return *this; }
-};
-
-typedef Vector<ucs4_t> UCS4V;
-typedef Vector<size_t> SizeV;
-typedef UTF8StringFieldSearcherBase SFSB;
-typedef FUTF8StrChrFieldSearcher FSFS;
-
-class TextUtilTest : public vespalib::TestApp
-{
-private:
- ucs4_t getUTF8Char(const char * src);
- template <typename BW, bool OFF>
- void assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets);
- void assertAnsiFold(const std::string & toFold, const std::string & exp);
- void assertAnsiFold(char c, char exp);
-#ifdef __x86_64__
- void assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp);
- void assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded = 16);
-#endif
-
- template <typename BW, bool OFF>
- void testSkipSeparators();
- void testSkipSeparators();
- void testSeparatorCharacter();
- void testAnsiFold();
- void test_lfoldua();
-#ifdef __x86_64__
- void test_sse2_foldua();
-#endif
-
-public:
- int Main() override;
-};
-
-ucs4_t
-TextUtilTest::getUTF8Char(const char * src)
-{
- ucs4_t retval = Fast_UnicodeUtil::GetUTF8Char(src);
- ASSERT_TRUE(retval != Fast_UnicodeUtil::_BadUTF8Char);
- return retval;
-}
-
-template <typename BW, bool OFF>
-void
-TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets)
-{
- const byte * srcbuf = reinterpret_cast<const byte *>(input);
- auto dstbuf = std::make_unique<ucs4_t[]>(len + 1);
- auto offsets = std::make_unique<size_t[]>(len + 1);
- UTF8StrChrFieldSearcher fs;
- BW bw(dstbuf.get(), offsets.get());
- size_t dstlen = fs.skipSeparators(srcbuf, len, bw);
- EXPECT_EQUAL(dstlen, expdstbuf.size());
- ASSERT_TRUE(dstlen == expdstbuf.size());
- for (size_t i = 0; i < dstlen; ++i) {
- EXPECT_EQUAL(dstbuf[i], expdstbuf[i]);
- if (OFF) {
- EXPECT_EQUAL(offsets[i], expoffsets[i]);
- }
- }
-}
-
-void
-TextUtilTest::assertAnsiFold(const std::string & toFold, const std::string & exp)
-{
- char folded[256];
- EXPECT_TRUE(FSFS::ansiFold(toFold.c_str(), toFold.size(), folded));
- EXPECT_EQUAL(std::string(folded, toFold.size()), exp);
-}
-
-void
-TextUtilTest::assertAnsiFold(char c, char exp)
-{
- char folded;
- EXPECT_TRUE(FSFS::ansiFold(&c, 1, &folded));
- EXPECT_EQUAL((int32_t)folded, (int32_t)exp);
-}
-
-#ifdef __x86_64__
-void
-TextUtilTest::assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp)
-{
- char folded[256];
- size_t alignedStart = 0xF - (size_t(folded + 0xF) % 0x10);
- const unsigned char * toFoldOrg = reinterpret_cast<const unsigned char *>(toFold.c_str());
- const unsigned char * retval =
- sse2_foldua(toFoldOrg, toFold.size(), reinterpret_cast<unsigned char *>(folded + alignedStart));
- EXPECT_EQUAL((size_t)(retval - toFoldOrg), charFolded);
- EXPECT_EQUAL(std::string(folded + alignedStart, charFolded), exp);
-}
-
-void
-TextUtilTest::assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded)
-{
- unsigned char toFold[16];
- memset(toFold, c, 16);
- unsigned char folded[32];
- size_t alignedStart = 0xF - (size_t(folded + 0xF) % 0x10);
- const unsigned char * retval = sse2_foldua(toFold, 16, folded + alignedStart);
- EXPECT_EQUAL((size_t)(retval - toFold), charFolded);
- for (size_t i = 0; i < charFolded; ++i) {
- EXPECT_EQUAL((int32_t)folded[i + alignedStart], (int32_t)exp);
- }
-}
-#endif
-
-template <typename BW, bool OFF>
-void
-TextUtilTest::testSkipSeparators()
-{
- // ascii characters
- assertSkipSeparators<BW, OFF>("foo", 3, UCS4V().a('f').a('o').a('o'), SizeV().a(0).a(1).a(2));
- assertSkipSeparators<BW, OFF>("f\x1Fo", 3, UCS4V().a('f').a('o'), SizeV().a(0).a(2));
- assertSkipSeparators<BW, OFF>("f\no", 3, UCS4V().a('f').a('\n').a('o'), SizeV().a(0).a(1).a(2));
- assertSkipSeparators<BW, OFF>("f\to", 3, UCS4V().a('f').a('\t').a('o'), SizeV().a(0).a(1).a(2));
-
- // utf8 char
- assertSkipSeparators<BW, OFF>("\xC2\x80\x66", 3, UCS4V().a(getUTF8Char("\xC2\x80")).a('f'),
- SizeV().a(0).a(2));
- assertSkipSeparators<BW, OFF>("\xE0\xA0\x80\x66", 4, UCS4V().a(getUTF8Char("\xE0\xA0\x80")).a('f'),
- SizeV().a(0).a(3));
- assertSkipSeparators<BW, OFF>("\xF0\x90\x80\x80\x66", 5, UCS4V().a(getUTF8Char("\xF0\x90\x80\x80")).a('f'),
- SizeV().a(0).a(4));
-
- // replacement string (sharp s -> ss)
- assertSkipSeparators<BW, OFF>("\xC3\x9F\x66\xC3\x9F", 5, UCS4V().a('s').a('s').a('f').a('s').a('s'),
- SizeV().a(0).a(0).a(2).a(3).a(3));
-}
-
-void
-TextUtilTest::testSkipSeparators()
-{
- Fast_NormalizeWordFolder::Setup(Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION);
-
- testSkipSeparators<SFSB::BufferWrapper, false>();
- testSkipSeparators<SFSB::OffsetWrapper, true>();
-}
-
-void
-TextUtilTest::testSeparatorCharacter()
-{
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x00'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x01'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x02'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x03'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x04'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x05'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x06'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x07'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x08'));
- EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x09')); // '\t'
- EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x0a')); // '\n'
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0b'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0c'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0d'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0e'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0f'));
-
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x10'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x11'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x12'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x13'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x14'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x15'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x16'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x17'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x18'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x19'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1a'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1b'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1c'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1d'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1e'));
- EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1f'));
-
- EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x20')); // space
-}
-
-void
-TextUtilTest::testAnsiFold()
-{
- FieldSearcher::init();
- assertAnsiFold("", "");
- assertAnsiFold("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz");
- assertAnsiFold("abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz");
- assertAnsiFold("0123456789", "0123456789");
- for (int i = 0; i < 128; ++i) {
- if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) {
- assertAnsiFold(i, i);
- } else if (i >= 'A' && i <= 'Z') {
- assertAnsiFold(i, i + 32);
- } else {
- assertAnsiFold(i, 0);
- }
- }
-
- // non-ascii is ignored
- for (int i = 128; i < 256; ++i) {
- char toFold = i;
- char folded;
- EXPECT_TRUE(!FSFS::ansiFold(&toFold, 1, &folded));
- }
-}
-
-void
-TextUtilTest::test_lfoldua()
-{
- FieldSearcher::init();
- char folded[256];
- size_t alignedStart = 0;
- const char * toFold = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
- size_t len = strlen(toFold);
- EXPECT_TRUE(FSFS::lfoldua(toFold, len, folded, alignedStart));
- EXPECT_EQUAL(std::string(folded + alignedStart, len), "abcdefghijklmnopqrstuvwxyz");
-}
-
-#ifdef __x86_64__
-void
-TextUtilTest::test_sse2_foldua()
-{
- assert_sse2_foldua("", 0, "");
- assert_sse2_foldua("ABCD", 0, "");
- assert_sse2_foldua("ABCDEFGHIJKLMNO", 0, "");
- assert_sse2_foldua("ABCDEFGHIJKLMNOP", 16, "abcdefghijklmnop");
- assert_sse2_foldua("ABCDEFGHIJKLMNOPQ", 16, "abcdefghijklmnop");
- assert_sse2_foldua("KLMNOPQRSTUVWXYZ", 16, "klmnopqrstuvwxyz");
- assert_sse2_foldua("abcdefghijklmnop", 16, "abcdefghijklmnop");
- assert_sse2_foldua("klmnopqrstuvwxyz", 16, "klmnopqrstuvwxyz");
- assert_sse2_foldua("0123456789abcdef", 16, "0123456789abcdef");
-
- for (int i = 0; i < 128; ++i) {
- if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) {
- assert_sse2_foldua(i, i);
- } else if (i >= 'A' && i <= 'Z') {
- assert_sse2_foldua(i, i + 32);
- } else {
- assert_sse2_foldua(i, 0);
- }
- }
-
- // non-ascii is ignored
- for (int i = 128; i < 256; ++i) {
- assert_sse2_foldua(i, '?', 0);
- }
-}
-#endif
-
-int
-TextUtilTest::Main()
-{
- TEST_INIT("textutil_test");
-
- testSkipSeparators();
- testSeparatorCharacter();
- testAnsiFold();
- test_lfoldua();
-#ifdef __x86_64__
- test_sse2_foldua();
-#endif
-
- TEST_DONE();
-}
-
-}
-
-TEST_APPHOOK(vsm::TextUtilTest);
diff --git a/streamingvisitors/src/tests/utilapps/.gitignore b/streamingvisitors/src/tests/utilapps/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/streamingvisitors/src/tests/utilapps/.gitignore
+++ /dev/null