diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-05-15 00:40:43 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-15 00:40:43 +0200 |
commit | dacf557add1c6a3ffab036cdf2f7dfdf9750b22e (patch) | |
tree | 3a9dfff58b98898e2e28c0337925f4f04e5eaeb0 /streamingvisitors/src/tests | |
parent | 2722ce9d1d1ec12d57ebd3833ce37b0958afb752 (diff) |
Revert "Collapse vsm into streamingvisitors"
Diffstat (limited to 'streamingvisitors/src/tests')
23 files changed, 4 insertions, 2131 deletions
diff --git a/streamingvisitors/src/tests/charbuffer/.gitignore b/streamingvisitors/src/tests/charbuffer/.gitignore deleted file mode 100644 index 2c980038fb5..00000000000 --- a/streamingvisitors/src/tests/charbuffer/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -charbuffer_test -vsm_charbuffer_test_app diff --git a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt deleted file mode 100644 index 5d0c0068d37..00000000000 --- a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vsm_charbuffer_test_app TEST - SOURCES - charbuffer.cpp - DEPENDS - streamingvisitors -) -vespa_add_test(NAME vsm_charbuffer_test_app COMMAND vsm_charbuffer_test_app) diff --git a/streamingvisitors/src/tests/charbuffer/charbuffer.cpp b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp deleted file mode 100644 index 736d35459cb..00000000000 --- a/streamingvisitors/src/tests/charbuffer/charbuffer.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/vsm/common/charbuffer.h> - -namespace vsm { - -class CharBufferTest : public vespalib::TestApp -{ -private: - void test(); -public: - int Main() override; -}; - -void -CharBufferTest::test() -{ - { // empty - CharBuffer buf; - EXPECT_EQUAL(buf.getLength(), 0u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 0u); - } - { // explicit length - CharBuffer buf(8); - EXPECT_EQUAL(buf.getLength(), 8u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 8u); - } - { // resize - CharBuffer buf(8); - EXPECT_EQUAL(buf.getLength(), 8u); - buf.resize(16); - EXPECT_EQUAL(buf.getLength(), 16u); - buf.resize(8); - EXPECT_EQUAL(buf.getLength(), 16u); - } - { // put with triggered resize - CharBuffer buf(8); - buf.put("123456", 6); - EXPECT_EQUAL(buf.getLength(), 8u); - EXPECT_EQUAL(buf.getPos(), 6u); - EXPECT_EQUAL(buf.getRemaining(), 2u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456"); - buf.put("789", 3); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 9u); - EXPECT_EQUAL(buf.getRemaining(), 3u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789"); - buf.put('a'); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 10u); - EXPECT_EQUAL(buf.getRemaining(), 2u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789a"); - buf.reset(); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 12u); - buf.put("bcd", 3); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 3u); - EXPECT_EQUAL(buf.getRemaining(), 9u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "bcd"); - } -} - -int -CharBufferTest::Main() -{ - TEST_INIT("charbuffer_test"); - - test(); - - TEST_DONE(); -} - -} - -TEST_APPHOOK(vsm::CharBufferTest); diff --git a/streamingvisitors/src/tests/config/mail.cfg b/streamingvisitors/src/tests/config/mail.cfg deleted file mode 100644 index ce830beac23..00000000000 --- a/streamingvisitors/src/tests/config/mail.cfg +++ /dev/null @@ -1,116 +0,0 @@ -datatype[2] -datatype[0].id 1012 -datatype[0].arraytype[1] -datatype[0].arraytype[0].datatype 12 -datatype[1].id 1013 -datatype[1].arraytype[1] -datatype[1].arraytype[0].datatype 13 -documenttype[1] -documenttype[0].name mail -documenttype[0].version 0 -documenttype[0].inherits[0] -documenttype[0].field[26] -documenttype[0].field[0].name mailid -documenttype[0].field[0].id 2 -documenttype[0].field[0].header true -documenttype[0].field[0].datatype 2 -documenttype[0].field[1].name date -documenttype[0].field[1].id 3 -documenttype[0].field[1].header true -documenttype[0].field[1].datatype 0 -documenttype[0].field[2].name from -documenttype[0].field[2].id 4 -documenttype[0].field[2].header true -documenttype[0].field[2].datatype 12 -documenttype[0].field[3].name replyto -documenttype[0].field[3].id 5 -documenttype[0].field[3].header true -documenttype[0].field[3].datatype 12 -documenttype[0].field[4].name to -documenttype[0].field[4].id 6 -documenttype[0].field[4].header true -documenttype[0].field[4].datatype 12 -documenttype[0].field[5].name cc -documenttype[0].field[5].id 7 -documenttype[0].field[5].header true -documenttype[0].field[5].datatype 12 -documenttype[0].field[6].name bcc -documenttype[0].field[6].id 8 -documenttype[0].field[6].header true -documenttype[0].field[6].datatype 12 -documenttype[0].field[7].name subject -documenttype[0].field[7].id 9 -documenttype[0].field[7].header true -documenttype[0].field[7].datatype 12 -documenttype[0].field[8].name body -documenttype[0].field[8].id 10 -documenttype[0].field[8].header false -documenttype[0].field[8].datatype 12 -documenttype[0].field[9].name attachmentcount -documenttype[0].field[9].id 11 -documenttype[0].field[9].header false -documenttype[0].field[9].datatype 0 -documenttype[0].field[10].name attachmentpartids -documenttype[0].field[10].id 12 -documenttype[0].field[10].header false -documenttype[0].field[10].datatype 2 -documenttype[0].field[11].name attachmentsizes -documenttype[0].field[11].id 13 -documenttype[0].field[11].header false -documenttype[0].field[11].datatype 2 -documenttype[0].field[12].name attachmentnames -documenttype[0].field[12].id 14 -documenttype[0].field[12].header false -documenttype[0].field[12].datatype 2 -documenttype[0].field[13].name attachmenttypes -documenttype[0].field[13].id 15 -documenttype[0].field[13].header false -documenttype[0].field[13].datatype 2 -documenttype[0].field[14].name attachmentlanguages -documenttype[0].field[14].id 16 -documenttype[0].field[14].header false -documenttype[0].field[14].datatype 2 -documenttype[0].field[15].name attachmentcontent -documenttype[0].field[15].id 17 -documenttype[0].field[15].header false -documenttype[0].field[15].datatype 2 -documenttype[0].field[16].name bodylanguage -documenttype[0].field[16].id 18 -documenttype[0].field[16].header false -documenttype[0].field[16].datatype 2 -documenttype[0].field[17].name bodyencoding -documenttype[0].field[17].id 19 -documenttype[0].field[17].header false -documenttype[0].field[17].datatype 2 -documenttype[0].field[18].name collectionid -documenttype[0].field[18].id 20 -documenttype[0].field[18].header true -documenttype[0].field[18].datatype 4 -documenttype[0].field[19].name content -documenttype[0].field[19].id 21 -documenttype[0].field[19].header true -documenttype[0].field[19].datatype 12 -documenttype[0].field[20].name bodymeta -documenttype[0].field[20].id 50027053 -documenttype[0].field[20].header false -documenttype[0].field[20].datatype 13 -documenttype[0].field[21].name attachments -documenttype[0].field[21].id 1081629685 -documenttype[0].field[21].header false -documenttype[0].field[21].datatype 1012 -documenttype[0].field[22].name attachmentsmeta -documenttype[0].field[22].id 1203055625 -documenttype[0].field[22].header false -documenttype[0].field[22].datatype 1013 -documenttype[0].field[23].name tolist -documenttype[0].field[23].id 1084918181 -documenttype[0].field[23].header false -documenttype[0].field[23].datatype 1012 -documenttype[0].field[24].name cclist -documenttype[0].field[24].id 1733332403 -documenttype[0].field[24].header false -documenttype[0].field[24].datatype 1012 -documenttype[0].field[25].name bcclist -documenttype[0].field[25].id 410546306 -documenttype[0].field[25].header false -documenttype[0].field[25].datatype 1012 diff --git a/streamingvisitors/src/tests/config/vsm.cfg b/streamingvisitors/src/tests/config/vsm.cfg deleted file mode 100644 index dc50447f623..00000000000 --- a/streamingvisitors/src/tests/config/vsm.cfg +++ /dev/null @@ -1,3 +0,0 @@ -doctype file:../config/mail.cfg -storagecfg "" -vsmfields file:../config/vsmfields.cfg diff --git a/streamingvisitors/src/tests/config/vsmfields.cfg b/streamingvisitors/src/tests/config/vsmfields.cfg deleted file mode 100644 index 30f1c8ed8b1..00000000000 --- a/streamingvisitors/src/tests/config/vsmfields.cfg +++ /dev/null @@ -1,297 +0,0 @@ -threadsperquery 4 -documentverificationlevel=0 -searchall 1 -fieldspec[17] -fieldspec[0].name bcc -fieldspec[0].searchmethod AUTOUTF8 -fieldspec[0].arg1 "" -fieldspec[1].name cc -fieldspec[1].searchmethod AUTOUTF8 -fieldspec[1].arg1 "" -fieldspec[2].name from -fieldspec[2].searchmethod AUTOUTF8 -fieldspec[2].arg1 "" -fieldspec[3].name date -fieldspec[3].searchmethod INT32 -fieldspec[3].arg1 "" -fieldspec[4].name replyto -fieldspec[4].searchmethod AUTOUTF8 -fieldspec[4].arg1 "" -fieldspec[5].name subject -fieldspec[5].searchmethod AUTOUTF8 -fieldspec[5].arg1 "" -fieldspec[6].name to -fieldspec[6].searchmethod AUTOUTF8 -fieldspec[6].arg1 "" -fieldspec[7].name body -fieldspec[7].searchmethod AUTOUTF8 -fieldspec[7].arg1 "" -fieldspec[8].name bodymeta -fieldspec[8].searchmethod AUTOUTF8 -fieldspec[8].arg1 "" -fieldspec[9].name mailid -fieldspec[9].searchmethod AUTOUTF8 -fieldspec[9].arg1 "" -fieldspec[10].name attachmentcount -fieldspec[10].searchmethod INT32 -fieldspec[10].arg1 "" -fieldspec[11].name attachmentcontent -fieldspec[11].searchmethod AUTOUTF8 -fieldspec[11].arg1 "" -fieldspec[12].name attachmenttypes -fieldspec[12].searchmethod AUTOUTF8 -fieldspec[12].arg1 "" -fieldspec[13].name attachmentnames -fieldspec[13].searchmethod AUTOUTF8 -fieldspec[13].arg1 "" -fieldspec[14].name attachmentlanguages -fieldspec[14].searchmethod AUTOUTF8 -fieldspec[14].arg1 "" -fieldspec[15].name URI -fieldspec[15].searchmethod AUTOUTF8 -fieldspec[15].arg1 "" -fieldspec[16].name vsm_whichfieldmatched -fieldspec[16].searchmethod AUTOUTF8 -fieldspec[16].arg1 "" -index[26] -index[0].name default -index[0].field[10] -index[0].field[0].name from -index[0].field[1].name to -index[0].field[2].name cc -index[0].field[3].name bcc -index[0].field[4].name subject -index[0].field[5].name body -index[0].field[6].name attachmentcontent -index[0].field[7].name attachmentnames -index[0].field[8].name attachmenttypes -index[0].field[9].name date -index[1].name all -index[1].field[8] -index[1].field[0].name to -index[1].field[1].name cc -index[1].field[2].name bcc -index[1].field[3].name subject -index[1].field[4].name body -index[1].field[5].name attachmentcontent -index[1].field[6].name attachmentnames -index[1].field[7].name attachmenttypes -index[2].name header -index[2].field[6] -index[2].field[0].name from -index[2].field[1].name replyto -index[2].field[2].name to -index[2].field[3].name cc -index[2].field[4].name bcc -index[2].field[5].name subject -index[3].name senders -index[3].field[2] -index[3].field[0].name from -index[3].field[1].name replyto -index[4].name recipients -index[4].field[3] -index[4].field[0].name to -index[4].field[1].name cc -index[4].field[2].name bcc -index[5].name address -index[5].field[5] -index[5].field[0].name from -index[5].field[1].name replyto -index[5].field[2].name to -index[5].field[3].name cc -index[5].field[4].name bcc -index[6].name body -index[6].field[2] -index[6].field[0].name subject -index[6].field[1].name body -index[7].name meta -index[7].field[2] -index[7].field[0].name attachmentcontent -index[7].field[1].name attachmenttypes -index[8].name index1 -index[8].field[1] -index[8].field[0].name bcc -index[9].name index2 -index[9].field[2] -index[9].field[0].name bcc -index[9].field[1].name cc -index[10].name index3 -index[10].field[3] -index[10].field[0].name bcc -index[10].field[1].name cc -index[10].field[2].name from -index[11].name index4 -index[11].field[4] -index[11].field[0].name bcc -index[11].field[1].name cc -index[11].field[2].name from -index[11].field[3].name date -index[12].name index5 -index[12].field[5] -index[12].field[0].name bcc -index[12].field[1].name cc -index[12].field[2].name from -index[12].field[3].name date -index[12].field[4].name replyto -index[13].name index6 -index[13].field[6] -index[13].field[0].name bcc -index[13].field[1].name cc -index[13].field[2].name from -index[13].field[3].name date -index[13].field[4].name replyto -index[13].field[5].name subject -index[14].name index7 -index[14].field[7] -index[14].field[0].name bcc -index[14].field[1].name cc -index[14].field[2].name from -index[14].field[3].name date -index[14].field[4].name replyto -index[14].field[5].name subject -index[14].field[6].name to -index[15].name index8 -index[15].field[8] -index[15].field[0].name bcc -index[15].field[1].name cc -index[15].field[2].name from -index[15].field[3].name date -index[15].field[4].name replyto -index[15].field[5].name subject -index[15].field[6].name to -index[15].field[7].name body -index[16].name index9 -index[16].field[9] -index[16].field[0].name bcc -index[16].field[1].name cc -index[16].field[2].name from -index[16].field[3].name date -index[16].field[4].name replyto -index[16].field[5].name subject -index[16].field[6].name to -index[16].field[7].name body -index[16].field[8].name bodymeta -index[17].name index10 -index[17].field[10] -index[17].field[0].name bcc -index[17].field[1].name cc -index[17].field[2].name from -index[17].field[3].name date -index[17].field[4].name replyto -index[17].field[5].name subject -index[17].field[6].name to -index[17].field[7].name body -index[17].field[8].name bodymeta -index[17].field[9].name mailid -index[18].name index11 -index[18].field[11] -index[18].field[0].name bcc -index[18].field[1].name cc -index[18].field[2].name from -index[18].field[3].name date -index[18].field[4].name replyto -index[18].field[5].name subject -index[18].field[6].name to -index[18].field[7].name body -index[18].field[8].name bodymeta -index[18].field[9].name mailid -index[18].field[10].name attachmentcount -index[19].name index12 -index[19].field[12] -index[19].field[0].name bcc -index[19].field[1].name cc -index[19].field[2].name from -index[19].field[3].name date -index[19].field[4].name replyto -index[19].field[5].name subject -index[19].field[6].name to -index[19].field[7].name body -index[19].field[8].name bodymeta -index[19].field[9].name mailid -index[19].field[10].name attachmentcount -index[19].field[11].name attachmentcontent -index[20].name index13 -index[20].field[13] -index[20].field[0].name bcc -index[20].field[1].name cc -index[20].field[2].name from -index[20].field[3].name date -index[20].field[4].name replyto -index[20].field[5].name subject -index[20].field[6].name to -index[20].field[7].name body -index[20].field[8].name bodymeta -index[20].field[9].name mailid -index[20].field[10].name attachmentcount -index[20].field[11].name attachmentcontent -index[20].field[12].name attachmenttypes -index[21].name index14 -index[21].field[14] -index[21].field[0].name bcc -index[21].field[1].name cc -index[21].field[2].name from -index[21].field[3].name date -index[21].field[4].name replyto -index[21].field[5].name subject -index[21].field[6].name to -index[21].field[7].name body -index[21].field[8].name bodymeta -index[21].field[9].name mailid -index[21].field[10].name attachmentcount -index[21].field[11].name attachmentcontent -index[21].field[12].name attachmenttypes -index[21].field[13].name attachmentnames -index[22].name index15 -index[22].field[15] -index[22].field[0].name bcc -index[22].field[1].name cc -index[22].field[2].name from -index[22].field[3].name date -index[22].field[4].name replyto -index[22].field[5].name subject -index[22].field[6].name to -index[22].field[7].name body -index[22].field[8].name bodymeta -index[22].field[9].name mailid -index[22].field[10].name attachmentcount -index[22].field[11].name attachmentcontent -index[22].field[12].name attachmenttypes -index[22].field[13].name attachmentnames -index[22].field[14].name attachmentlanguages -index[23].name index16 -index[23].field[15] -index[23].field[0].name bcc -index[23].field[1].name cc -index[23].field[2].name from -index[23].field[3].name date -index[23].field[4].name replyto -index[23].field[5].name subject -index[23].field[6].name to -index[23].field[7].name body -index[23].field[8].name bodymeta -index[23].field[9].name mailid -index[23].field[10].name attachmentcount -index[23].field[11].name attachmentcontent -index[23].field[12].name attachmenttypes -index[23].field[13].name attachmentnames -index[23].field[14].name attachmentlanguages -index[24].name index17 -index[24].field[15] -index[24].field[0].name bcc -index[24].field[1].name cc -index[24].field[2].name from -index[24].field[3].name date -index[24].field[4].name replyto -index[24].field[5].name subject -index[24].field[6].name to -index[24].field[7].name body -index[24].field[8].name bodymeta -index[24].field[9].name mailid -index[24].field[10].name attachmentcount -index[24].field[11].name attachmentcontent -index[24].field[12].name attachmenttypes -index[24].field[13].name attachmentnames -index[24].field[14].name attachmentlanguages -index[25].name date -index[25].field[1] -index[25].field[0].name date diff --git a/streamingvisitors/src/tests/docsum/.gitignore b/streamingvisitors/src/tests/docsum/.gitignore deleted file mode 100644 index 9a697a94de8..00000000000 --- a/streamingvisitors/src/tests/docsum/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -docsum_test -vsm_docsum_test_app diff --git a/streamingvisitors/src/tests/docsum/CMakeLists.txt b/streamingvisitors/src/tests/docsum/CMakeLists.txt deleted file mode 100644 index 87c46409053..00000000000 --- a/streamingvisitors/src/tests/docsum/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vsm_docsum_test_app TEST - SOURCES - docsum.cpp - DEPENDS - streamingvisitors -) -vespa_add_test(NAME vsm_docsum_test_app COMMAND vsm_docsum_test_app) diff --git a/streamingvisitors/src/tests/docsum/docsum.cpp b/streamingvisitors/src/tests/docsum/docsum.cpp deleted file mode 100644 index 475489d2f5a..00000000000 --- a/streamingvisitors/src/tests/docsum/docsum.cpp +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/datatype/structdatatype.h> -#include <vespa/document/datatype/weightedsetdatatype.h> -#include <vespa/document/datatype/mapdatatype.h> -#include <vespa/vsm/common/docsum.h> -#include <vespa/vsm/vsm/flattendocsumwriter.h> -#include <vespa/vsm/vsm/slimefieldwriter.h> - -using namespace document; - -namespace vsm { - -template <typename T> -class Vector : public std::vector<T> -{ -public: - Vector<T> & add(T v) { this->push_back(v); return *this; } -}; - -typedef Vector<std::string> StringList; -typedef Vector<std::pair<std::string, int32_t> > WeightedStringList; - - -class TestDocument : public vsm::Document -{ -private: - std::vector<FieldValueContainer> _fields; - -public: - TestDocument(const search::DocumentIdT & docId, size_t numFields) : vsm::Document(docId, numFields), _fields(numFields) {} - virtual bool setField(FieldIdT fId, document::FieldValue::UP fv) override { - if (fId < _fields.size()) { - _fields[fId].reset(fv.release()); - return true; - } - return false; - } - virtual const document::FieldValue * getField(FieldIdT fId) const override { - if (fId < _fields.size()) { - return _fields[fId].get(); - } - return NULL; - } -}; - - -class DocsumTest : public vespalib::TestApp -{ -private: - ArrayFieldValue createFieldValue(const StringList & fv); - WeightedSetFieldValue createFieldValue(const WeightedStringList & fv); - - void assertFlattenDocsumWriter(const FieldValue & fv, const std::string & exp) { - FlattenDocsumWriter fdw; - assertFlattenDocsumWriter(fdw, fv, exp); - } - void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp); - void assertSlimeFieldWriter(const FieldValue & fv, const std::string & exp) { - SlimeFieldWriter sfw; - TEST_DO(assertSlimeFieldWriter(sfw, fv, exp)); - } - void assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp); - - void testFlattenDocsumWriter(); - void testSlimeFieldWriter(); - void requireThatSlimeFieldWriterHandlesMap(); - void testDocSumCache(); - -public: - int Main() override; -}; - -ArrayFieldValue -DocsumTest::createFieldValue(const StringList & fv) -{ - - static ArrayDataType type(*DataType::STRING); - ArrayFieldValue afv(type); - for (size_t i = 0; i < fv.size(); ++i) { - afv.add(StringFieldValue(fv[i])); - } - return afv; -} - -WeightedSetFieldValue -DocsumTest::createFieldValue(const WeightedStringList & fv) -{ - static WeightedSetDataType type(*DataType::STRING, false, false); - WeightedSetFieldValue wsfv(type); - for (size_t i = 0; i < fv.size(); ++i) { - wsfv.add(StringFieldValue(fv[i].first), fv[i].second); - } - return wsfv; -} - -void -DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp) -{ - FieldPath empty; - fv.iterateNested(empty.getFullRange(), fdw); - std::string actual(fdw.getResult().getBuffer(), fdw.getResult().getPos()); - EXPECT_EQUAL(actual, exp); -} - -void -DocsumTest::assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp) -{ - sfw.convert(fv); - - vespalib::Slime gotSlime; - vespalib::Memory serialized(sfw.out()); - size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, gotSlime); - ASSERT_EQUAL(decodeRes, serialized.size); - - vespalib::Slime expSlime; - size_t used = vespalib::slime::JsonFormat::decode(exp, expSlime); - EXPECT_TRUE(used > 0); - EXPECT_EQUAL(expSlime, gotSlime); -} - -void -DocsumTest::testFlattenDocsumWriter() -{ - { // basic tests - TEST_DO(assertFlattenDocsumWriter(StringFieldValue("foo bar"), "foo bar")); - TEST_DO(assertFlattenDocsumWriter(RawFieldValue("foo bar"), "foo bar")); - TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(true), "true")); - TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(false), "false")); - TEST_DO(assertFlattenDocsumWriter(LongFieldValue(123456789), "123456789")); - TEST_DO(assertFlattenDocsumWriter(createFieldValue(StringList().add("foo bar").add("baz").add(" qux ")), - "foo bar baz qux ")); - } - { // test mulitple invokations - FlattenDocsumWriter fdw("#"); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("foo"), "foo")); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("bar"), "foo#bar")); - fdw.clear(); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("baz"), "baz")); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("qux"), "baz qux")); - } - { // test resizing - FlattenDocsumWriter fdw("#"); - EXPECT_EQUAL(fdw.getResult().getPos(), 0u); - EXPECT_EQUAL(fdw.getResult().getLength(), 32u); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaabbbbccccddddeeeeffffgggghhhh"), - "aaaabbbbccccddddeeeeffffgggghhhh")); - EXPECT_EQUAL(fdw.getResult().getPos(), 32u); - EXPECT_EQUAL(fdw.getResult().getLength(), 32u); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaa"), "aaaabbbbccccddddeeeeffffgggghhhh#aaaa")); - EXPECT_EQUAL(fdw.getResult().getPos(), 37u); - EXPECT_TRUE(fdw.getResult().getLength() >= 37u); - fdw.clear(); - EXPECT_EQUAL(fdw.getResult().getPos(), 0u); - EXPECT_TRUE(fdw.getResult().getLength() >= 37u); - } -} - -void -DocsumTest::testSlimeFieldWriter() -{ - { // basic types - assertSlimeFieldWriter(LongFieldValue(123456789), "123456789"); - assertSlimeFieldWriter(BoolFieldValue(true), "true"); - assertSlimeFieldWriter(BoolFieldValue(false), "false"); - assertSlimeFieldWriter(DoubleFieldValue(12.34), "12.34"); - assertSlimeFieldWriter(StringFieldValue("foo bar"), "\"foo bar\""); - } - { // collection field values - assertSlimeFieldWriter(createFieldValue(StringList().add("foo").add("bar").add("baz")), - "[\"foo\",\"bar\",\"baz\"]"); - assertSlimeFieldWriter(createFieldValue(WeightedStringList().add(std::make_pair("bar", 20)). - add(std::make_pair("baz", 30)). - add(std::make_pair("foo", 10))), - "[{item:\"bar\",weight:20},{item:\"baz\",weight:30},{item:\"foo\",weight:10}]"); - } - { // struct field value - StructDataType subType("substruct"); - Field fd("d", 0, *DataType::STRING); - Field fe("e", 1, *DataType::STRING); - subType.addField(fd); - subType.addField(fe); - StructFieldValue subValue(subType); - subValue.setValue(fd, StringFieldValue("baz")); - subValue.setValue(fe, StringFieldValue("qux")); - - StructDataType type("struct"); - Field fa("a", 0, *DataType::STRING); - Field fb("b", 1, *DataType::STRING); - Field fc("c", 2, subType); - type.addField(fa); - type.addField(fb); - type.addField(fc); - StructFieldValue value(type); - value.setValue(fa, StringFieldValue("foo")); - value.setValue(fb, StringFieldValue("bar")); - value.setValue(fc, subValue); - - - { // select a subset and then all - SlimeFieldWriter sfw; - DocsumFieldSpec::FieldIdentifierVector fields; - { - FieldPath path; - type.buildFieldPath(path, "a"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); - } - { - FieldPath path; - type.buildFieldPath(path, "c.e"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); - } - sfw.setInputFields(fields); - TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}")); - sfw.clear(); - TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"b\":\"bar\",\"c\":{\"d\":\"baz\",\"e\":\"qux\"}}")); - } - - { // multiple invocations - SlimeFieldWriter sfw; - TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("foo"), "\"foo\"")); - sfw.clear(); - TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("bar"), "\"bar\"")); - sfw.clear(); - TEST_DO(assertSlimeFieldWriter(sfw, StringFieldValue("baz"), "\"baz\"")); - } - - } -} - -void -DocsumTest::requireThatSlimeFieldWriterHandlesMap() -{ - { // map<string, string> - MapDataType mapType(*DataType::STRING, *DataType::STRING); - MapFieldValue mapfv(mapType); - EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), StringFieldValue("v1"))); - EXPECT_TRUE(mapfv.put(StringFieldValue("k2"), StringFieldValue("v2"))); - assertSlimeFieldWriter(mapfv, "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]"); - } - { // map<string, struct> - StructDataType structType("struct"); - Field fa("a", 0, *DataType::STRING); - Field fb("b", 1, *DataType::STRING); - structType.addField(fa); - structType.addField(fb); - StructFieldValue structValue(structType); - structValue.setValue(fa, StringFieldValue("foo")); - structValue.setValue(fb, StringFieldValue("bar")); - MapDataType mapType(*DataType::STRING, structType); - MapFieldValue mapfv(mapType); - EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), structValue)); - { // select a subset and then all - SlimeFieldWriter sfw; - DocsumFieldSpec::FieldIdentifierVector fields; - { - FieldPath path; - mapType.buildFieldPath(path, "value.b"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); - } - sfw.setInputFields(fields); - TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]")); - { - FieldPath path; - mapType.buildFieldPath(path, "{k1}.a"); - fields[0] = DocsumFieldSpec::FieldIdentifier(0, std::move(path)); - } - sfw.clear(); - sfw.setInputFields(fields); - TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\"}}]")); - sfw.clear(); // all fields implicit - TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\",\"b\":\"bar\"}}]")); - } - } -} - -int -DocsumTest::Main() -{ - TEST_INIT("docsum_test"); - - TEST_DO(testFlattenDocsumWriter()); - TEST_DO(testSlimeFieldWriter()); - TEST_DO(requireThatSlimeFieldWriterHandlesMap()); - - TEST_DONE(); -} - -} - -TEST_APPHOOK(vsm::DocsumTest); - diff --git a/streamingvisitors/src/tests/document/.gitignore b/streamingvisitors/src/tests/document/.gitignore deleted file mode 100644 index d47781eff63..00000000000 --- a/streamingvisitors/src/tests/document/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -document_test -vsm_document_test_app diff --git a/streamingvisitors/src/tests/document/CMakeLists.txt b/streamingvisitors/src/tests/document/CMakeLists.txt deleted file mode 100644 index 5ea12dc5e2d..00000000000 --- a/streamingvisitors/src/tests/document/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vsm_document_test_app TEST - SOURCES - document.cpp - DEPENDS - streamingvisitors -) -vespa_add_test(NAME vsm_document_test_app COMMAND vsm_document_test_app) diff --git a/streamingvisitors/src/tests/document/document.cpp b/streamingvisitors/src/tests/document/document.cpp deleted file mode 100644 index 1e97d232a64..00000000000 --- a/streamingvisitors/src/tests/document/document.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/vsm/common/storagedocument.h> -#include <vespa/vespalib/stllike/asciistream.h> - -using namespace document; - -namespace vsm { - -class DocumentTest : public vespalib::TestApp -{ -private: - void testStorageDocument(); - void testStringFieldIdTMap(); -public: - int Main() override; -}; - -void -DocumentTest::testStorageDocument() -{ - DocumentType dt("testdoc", 0); - - Field fa("a", 0, *DataType::STRING); - Field fb("b", 1, *DataType::STRING); - dt.addField(fa); - dt.addField(fb); - - document::Document::UP doc(new document::Document(dt, DocumentId())); - doc->setValue(fa, StringFieldValue("foo")); - doc->setValue(fb, StringFieldValue("bar")); - - SharedFieldPathMap fpmap(new FieldPathMapT()); - fpmap->emplace_back(); - dt.buildFieldPath(fpmap->back(),"a"); - fpmap->emplace_back(); - dt.buildFieldPath(fpmap->back(), "b"); - fpmap->emplace_back(); - ASSERT_TRUE((*fpmap)[0].size() == 1); - ASSERT_TRUE((*fpmap)[1].size() == 1); - ASSERT_TRUE((*fpmap)[2].size() == 0); - - StorageDocument sdoc(std::move(doc), fpmap, 3); - ASSERT_TRUE(sdoc.valid()); - - EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); - EXPECT_TRUE(sdoc.getField(2) == nullptr); - // test caching - EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); - EXPECT_TRUE(sdoc.getField(2) == nullptr); - - // set new values - EXPECT_TRUE(sdoc.setField(0, FieldValue::UP(new StringFieldValue("baz")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); - EXPECT_TRUE(sdoc.getField(2) == nullptr); - EXPECT_TRUE(sdoc.setField(1, FieldValue::UP(new StringFieldValue("qux")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString()); - EXPECT_TRUE(sdoc.getField(2) == nullptr); - EXPECT_TRUE(sdoc.setField(2, FieldValue::UP(new StringFieldValue("quux")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString()); - EXPECT_EQUAL(std::string("quux"), sdoc.getField(2)->getAsString()); - - EXPECT_TRUE(!sdoc.setField(3, FieldValue::UP(new StringFieldValue("thud")))); - - SharedFieldPathMap fim; - StorageDocument s2(std::make_unique<document::Document>(), fim, 0); - EXPECT_EQUAL(IdString().toString(), s2.docDoc().getId().toString()); -} - -void DocumentTest::testStringFieldIdTMap() -{ - StringFieldIdTMap m; - EXPECT_EQUAL(0u, m.highestFieldNo()); - EXPECT_TRUE(StringFieldIdTMap::npos == m.fieldNo("unknown")); - m.add("f1"); - EXPECT_EQUAL(0u, m.fieldNo("f1")); - EXPECT_EQUAL(1u, m.highestFieldNo()); - m.add("f1"); - EXPECT_EQUAL(0u, m.fieldNo("f1")); - EXPECT_EQUAL(1u, m.highestFieldNo()); - m.add("f2"); - EXPECT_EQUAL(1u, m.fieldNo("f2")); - EXPECT_EQUAL(2u, m.highestFieldNo()); - m.add("f3", 7); - EXPECT_EQUAL(7u, m.fieldNo("f3")); - EXPECT_EQUAL(8u, m.highestFieldNo()); - m.add("f3"); - EXPECT_EQUAL(7u, m.fieldNo("f3")); - EXPECT_EQUAL(8u, m.highestFieldNo()); - m.add("f2", 13); - EXPECT_EQUAL(13u, m.fieldNo("f2")); - EXPECT_EQUAL(14u, m.highestFieldNo()); - m.add("f4"); - EXPECT_EQUAL(3u, m.fieldNo("f4")); - EXPECT_EQUAL(14u, m.highestFieldNo()); - { - vespalib::asciistream os; - StringFieldIdTMap t; - t.add("b"); - t.add("a"); - os << t; - EXPECT_EQUAL(vespalib::string("a = 1\nb = 0\n"), os.str()); - } - -} - -int -DocumentTest::Main() -{ - TEST_INIT("document_test"); - - testStorageDocument(); - testStringFieldIdTMap(); - - TEST_DONE(); -} - -} - -TEST_APPHOOK(vsm::DocumentTest); - diff --git a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt index dbec820a462..f25ab348265 100644 --- a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt +++ b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_hitcollector_test_app TEST SOURCES hitcollector_test.cpp DEPENDS - streamingvisitors + streamingvisitors_searchvisitor ) vespa_add_test(NAME streamingvisitors_hitcollector_test_app COMMAND streamingvisitors_hitcollector_test_app) diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt index 5cc2977b3c3..ef93d551912 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt +++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST SOURCES matching_elements_filler_test.cpp DEPENDS - streamingvisitors + streamingvisitors_searchvisitor GTest::GTest ) vespa_add_test(NAME streamingvisitors_matching_elements_filler_test_app COMMAND streamingvisitors_matching_elements_filler_test_app) diff --git a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt index e0131d0c6cc..7cae60e6a11 100644 --- a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt +++ b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_querywrapper_test_app TEST SOURCES querywrapper_test.cpp DEPENDS - streamingvisitors + streamingvisitors_searchvisitor ) vespa_add_test(NAME streamingvisitors_querywrapper_test_app COMMAND streamingvisitors_querywrapper_test_app) diff --git a/streamingvisitors/src/tests/searcher/.gitignore b/streamingvisitors/src/tests/searcher/.gitignore deleted file mode 100644 index 52a56dff405..00000000000 --- a/streamingvisitors/src/tests/searcher/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -searcher_test -vsm_searcher_test_app diff --git a/streamingvisitors/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt deleted file mode 100644 index 2277f5ef55f..00000000000 --- a/streamingvisitors/src/tests/searcher/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vsm_searcher_test_app TEST - SOURCES - searcher_test.cpp - DEPENDS - streamingvisitors -) -vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app) diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp deleted file mode 100644 index 34fa66eaa90..00000000000 --- a/streamingvisitors/src/tests/searcher/searcher_test.cpp +++ /dev/null @@ -1,864 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/vsm/searcher/fieldsearcher.h> -#include <vespa/vsm/searcher/floatfieldsearcher.h> -#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> -#include <vespa/vsm/searcher/intfieldsearcher.h> -#include <vespa/vsm/searcher/boolfieldsearcher.h> -#include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h> -#include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h> -#include <vespa/vsm/searcher/utf8substringsearcher.h> -#include <vespa/vsm/searcher/utf8substringsnippetmodifier.h> -#include <vespa/vsm/searcher/utf8suffixstringfieldsearcher.h> -#include <vespa/vsm/vsm/snippetmodifier.h> -#include <vespa/searchlib/query/streaming/queryterm.h> -#include <vespa/document/fieldvalue/fieldvalues.h> - -using namespace document; -using search::streaming::HitList; -using search::streaming::QueryNodeResultFactory; -using search::streaming::QueryTerm; -using search::streaming::QueryTermList; -using TermType = QueryTerm::Type; -using namespace vsm; - -template <typename T> -class Vector : public std::vector<T> -{ -public: - Vector() : std::vector<T>() {} - Vector<T> & add(T v) { this->push_back(v); return *this; } -}; - -typedef Vector<size_t> Hits; -typedef Vector<std::string> StringList; -typedef Vector<Hits> HitsList; -typedef Vector<bool> BoolList; -typedef Vector<int64_t> LongList; -typedef Vector<float> FloatList; -typedef QueryTerm::FieldInfo QTFieldInfo; -typedef Vector<QTFieldInfo> FieldInfoList; - -class String -{ -private: - const std::string & _str; -public: - String(const std::string & str) : _str(str) {} - bool operator==(const String & rhs) const { - return _str == rhs._str; - } -}; - -class Query -{ -private: - void setupQuery(const StringList & terms) { - for (size_t i = 0; i < terms.size(); ++i) { - ParsedQueryTerm pqt = parseQueryTerm(terms[i]); - ParsedTerm pt = parseTerm(pqt.second); - qtv.push_back(std::make_unique<QueryTerm>(eqnr.create(), pt.first, pqt.first.empty() ? "index" : pqt.first, pt.second)); - } - for (size_t i = 0; i < qtv.size(); ++i) { - qtl.push_back(qtv[i].get()); - } - } -public: - typedef std::pair<std::string, std::string> ParsedQueryTerm; - typedef std::pair<std::string, TermType> ParsedTerm; - QueryNodeResultFactory eqnr; - std::vector<QueryTerm::UP> qtv; - QueryTermList qtl; - Query(const StringList & terms); - ~Query(); - static ParsedQueryTerm parseQueryTerm(const std::string & queryTerm) { - size_t i = queryTerm.find(':'); - if (i != std::string::npos) { - return ParsedQueryTerm(queryTerm.substr(0, i), queryTerm.substr(i + 1)); - } - return ParsedQueryTerm(std::string(), queryTerm); - } - static ParsedTerm parseTerm(const std::string & term) { - if (term[0] == '*' && term[term.size() - 1] == '*') { - return std::make_pair(term.substr(1, term.size() - 2), TermType::SUBSTRINGTERM); - } else if (term[0] == '*') { - return std::make_pair(term.substr(1, term.size() - 1), TermType::SUFFIXTERM); - } else if (term[term.size() - 1] == '*') { - return std::make_pair(term.substr(0, term.size() - 1), TermType::PREFIXTERM); - } else { - return std::make_pair(term, TermType::WORD); - } - } -}; - -Query::Query(const StringList & terms) : eqnr(), qtv(), qtl() { - setupQuery(terms); -} -Query::~Query() = default; - -struct SnippetModifierSetup -{ - Query query; - UTF8SubstringSnippetModifier::SP searcher; - SharedSearcherBuf buf; - SnippetModifier modifier; - explicit SnippetModifierSetup(const StringList & terms); - ~SnippetModifierSetup(); -}; - -SnippetModifierSetup::SnippetModifierSetup(const StringList & terms) - : query(terms), - searcher(new UTF8SubstringSnippetModifier()), - buf(new SearcherBuf(8)), - modifier(searcher) -{ - searcher->prepare(query.qtl, buf); -} -SnippetModifierSetup::~SnippetModifierSetup() = default; - -// helper functions -ArrayFieldValue getFieldValue(const StringList &fv); -ArrayFieldValue getFieldValue(const LongList &fv); -ArrayFieldValue getFieldValue(const FloatList &fv); - -bool assertMatchTermSuffix(const std::string &term, const std::string &word); -void assertSnippetModifier(const StringList &query, const std::string &fv, const std::string &exp); -void assertSnippetModifier(SnippetModifierSetup &setup, const FieldValue &fv, const std::string &exp); -void assertQueryTerms(const SnippetModifierManager &man, FieldIdT fId, const StringList &terms); -void assertNumeric(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const BoolList &exp); -std::vector<QueryTerm::UP> performSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv); -void assertSearch(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const HitsList &exp); -bool assertCountWords(size_t numWords, const std::string &field); -bool assertFieldInfo(FieldSearcher &fs, const StringList &query, const FieldValue &fv, const FieldInfoList &exp); - -void assertString(StrChrFieldSearcher &fs, const StringList &query, const std::string &field, const HitsList &exp) { - assertSearch(fs, query, StringFieldValue(field), exp); -} - -void assertString(StrChrFieldSearcher &fs, const StringList &query, const StringList &field, const HitsList &exp) { - assertSearch(fs, query, getFieldValue(field), exp); -} - -void assertString(StrChrFieldSearcher &fs, const std::string &term, const std::string &field, const Hits &exp) { - assertString(fs, StringList().add(term), field, HitsList().add(exp)); -} -void assertString(StrChrFieldSearcher &fs, const std::string &term, const StringList &field, const Hits &exp) { - assertString(fs, StringList().add(term), field, HitsList().add(exp)); -} - -void assertInt(IntFieldSearcher & fs, const StringList &query, int64_t field, const BoolList &exp) { - assertNumeric(fs, query, LongFieldValue(field), exp); -} - -void assertInt(IntFieldSearcher & fs, const std::string &term, int64_t field, bool exp) { - assertInt(fs, StringList().add(term), field, BoolList().add(exp)); -} - -void assertBool(BoolFieldSearcher & fs, const StringList &query, bool field, const BoolList &exp) { - assertNumeric(fs, query, BoolFieldValue(field), exp); -} -void assertBool(BoolFieldSearcher & fs, const std::string &term, bool field, bool exp) { - assertBool(fs, StringList().add(term), field, BoolList().add(exp)); -} - -void assertInt(IntFieldSearcher & fs, const StringList &query, const LongList &field, const HitsList &exp) { - assertSearch(fs, query, getFieldValue(field), exp); -} - -void assertInt(IntFieldSearcher & fs, const std::string &term, const LongList &field, const Hits &exp) { - assertInt(fs, StringList().add(term), field, HitsList().add(exp)); -} - -void assertFloat(FloatFieldSearcher & fs, const StringList &query, float field, const BoolList &exp) { - assertNumeric(fs, query, FloatFieldValue(field), exp); -} - -void assertFloat(FloatFieldSearcher & fs, const std::string &term, float field, bool exp) { - assertFloat(fs, StringList().add(term), field, BoolList().add(exp)); -} - -void assertFloat(FloatFieldSearcher & fs, const StringList &query, const FloatList &field, const HitsList &exp) { - assertSearch(fs, query, getFieldValue(field), exp); -} - -void assertFloat(FloatFieldSearcher & fs, const std::string &term, const FloatList &field, const Hits &exp) { - assertFloat(fs, StringList().add(term), field, HitsList().add(exp)); -} - -bool -assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const std::string &fv, const FieldInfoList &exp) { - return assertFieldInfo(fs, query, StringFieldValue(fv), exp); -} - -bool -assertFieldInfo(StrChrFieldSearcher &fs, const StringList &query, const StringList &fv, const FieldInfoList &exp) { - return assertFieldInfo(fs, query, getFieldValue(fv), exp); -} -bool -assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const StringList &fv, const QTFieldInfo &exp) { - return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - -bool -assertFieldInfo(StrChrFieldSearcher &fs, const std::string &term, const std::string &fv, const QTFieldInfo &exp) { - return assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - -void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, int64_t fv, const FieldInfoList &exp) { - assertFieldInfo(fs, query, LongFieldValue(fv), exp); -} - -void assertFieldInfo(IntFieldSearcher & fs, const StringList &query, const LongList &fv, const FieldInfoList &exp) { - assertFieldInfo(fs, query, getFieldValue(fv), exp); -} - -void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, int64_t fv, const QTFieldInfo &exp) { - assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - -void assertFieldInfo(IntFieldSearcher & fs, const std::string &term, const LongList &fv, const QTFieldInfo &exp) { - assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - -void assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, float fv, const FieldInfoList &exp) { - assertFieldInfo(fs, query, FloatFieldValue(fv), exp); -} - -void -assertFieldInfo(FloatFieldSearcher & fs, const StringList &query, const FloatList &fv, const FieldInfoList &exp) { - assertFieldInfo(fs, query, getFieldValue(fv), exp); -} - -/** float field searcher **/ -void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, float fv, const QTFieldInfo &exp) { - assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - -void assertFieldInfo(FloatFieldSearcher & fs, const std::string &term, const FloatList &fv, const QTFieldInfo &exp) { - assertFieldInfo(fs, StringList().add(term), fv, FieldInfoList().add(exp)); -} - - -/** snippet modifer searcher **/ -void assertSnippetModifier(const std::string &term, const std::string &fv, const std::string &exp) { - assertSnippetModifier(StringList().add(term), fv, exp); -} - - -ArrayFieldValue -getFieldValue(const StringList & fv) -{ - - static ArrayDataType type(*DataType::STRING); - ArrayFieldValue afv(type); - for (size_t i = 0; i < fv.size(); ++i) { - afv.add(StringFieldValue(fv[i])); - } - return afv; -} - -ArrayFieldValue -getFieldValue(const LongList & fv) -{ - static ArrayDataType type(*DataType::LONG); - ArrayFieldValue afv(type); - for (size_t i = 0; i < fv.size(); ++i) { - afv.add(LongFieldValue(fv[i])); - } - return afv; -} - -ArrayFieldValue -getFieldValue(const FloatList & fv) -{ - static ArrayDataType type(*DataType::FLOAT); - ArrayFieldValue afv(type); - for (size_t i = 0; i < fv.size(); ++i) { - afv.add(FloatFieldValue(fv[i])); - } - return afv; -} - -bool -assertMatchTermSuffix(const std::string & term, const std::string & word) -{ - QueryNodeResultFactory eqnr; - QueryTerm qa(eqnr.create(), term, "index", TermType::WORD); - QueryTerm qb(eqnr.create(), word, "index", TermType::WORD); - const ucs4_t * a; - size_t alen = qa.term(a); - const ucs4_t * b; - size_t blen = qb.term(b); - return UTF8StringFieldSearcherBase::matchTermSuffix(a, alen, b, blen); -} - -void -assertNumeric(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const BoolList & exp) -{ - HitsList hl; - for (size_t i = 0; i < exp.size(); ++i) { - hl.push_back(exp[i] ? Hits().add(0) : Hits()); - } - assertSearch(fs, query, fv, hl); -} - -std::vector<QueryTerm::UP> -performSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv) -{ - Query q(query); - - // prepare field searcher - SharedSearcherBuf ssb = SharedSearcherBuf(new SearcherBuf()); - fs.prepare(q.qtl, ssb); - - // setup document - SharedFieldPathMap sfim(new FieldPathMapT()); - sfim->push_back(FieldPath()); - StorageDocument doc(std::make_unique<document::Document>(), sfim, 1); - doc.setField(0, document::FieldValue::UP(fv.clone())); - - fs.search(doc); - return std::move(q.qtv); -} - -void -assertSearch(FieldSearcher & fs, const StringList & query, const FieldValue & fv, const HitsList & exp) -{ - auto qtv = performSearch(fs, query, fv); - EXPECT_EQUAL(qtv.size(), exp.size()); - ASSERT_TRUE(qtv.size() == exp.size()); - for (size_t i = 0; i < qtv.size(); ++i) { - const HitList & hl = qtv[i]->getHitList(); - EXPECT_EQUAL(hl.size(), exp[i].size()); - ASSERT_TRUE(hl.size() == exp[i].size()); - for (size_t j = 0; j < hl.size(); ++j) { - EXPECT_EQUAL((size_t)hl[j].pos(), exp[i][j]); - } - } -} - -bool -assertFieldInfo(FieldSearcher & fs, const StringList & query, - const FieldValue & fv, const FieldInfoList & exp) -{ - auto qtv = performSearch(fs, query, fv); - if (!EXPECT_EQUAL(qtv.size(), exp.size())) return false; - bool retval = true; - for (size_t i = 0; i < qtv.size(); ++i) { - if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitOffset(), exp[i].getHitOffset())) retval = false; - if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getHitCount(), exp[i].getHitCount())) retval = false; - if (!EXPECT_EQUAL(qtv[i]->getFieldInfo(0).getFieldLength(), exp[i].getFieldLength())) retval = false; - } - return retval; -} - -void -assertSnippetModifier(const StringList & query, const std::string & fv, const std::string & exp) -{ - UTF8SubstringSnippetModifier mod; - performSearch(mod, query, StringFieldValue(fv)); - EXPECT_EQUAL(mod.getModifiedBuf().getPos(), exp.size()); - std::string actual(mod.getModifiedBuf().getBuffer(), mod.getModifiedBuf().getPos()); - EXPECT_EQUAL(actual.size(), exp.size()); - EXPECT_EQUAL(actual, exp); -} - -void assertSnippetModifier(SnippetModifierSetup & setup, const FieldValue & fv, const std::string & exp) -{ - FieldValue::UP mfv = setup.modifier.modify(fv); - const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(*mfv.get()); - const std::string & actual = lfv.getValue(); - EXPECT_EQUAL(actual.size(), exp.size()); - EXPECT_EQUAL(actual, exp); -} - -void assertQueryTerms(const SnippetModifierManager & man, FieldIdT fId, const StringList & terms) -{ - if (terms.size() == 0) { - ASSERT_TRUE(man.getModifiers().getModifier(fId) == NULL); - return; - } - ASSERT_TRUE(man.getModifiers().getModifier(fId) != NULL); - UTF8SubstringSnippetModifier * searcher = - (static_cast<SnippetModifier *>(man.getModifiers().getModifier(fId)))->getSearcher().get(); - EXPECT_EQUAL(searcher->getQueryTerms().size(), terms.size()); - ASSERT_TRUE(searcher->getQueryTerms().size() == terms.size()); - for (size_t i = 0; i < terms.size(); ++i) { - EXPECT_EQUAL(std::string(searcher->getQueryTerms()[i]->getTerm()), terms[i]); - } -} - -bool assertCountWords(size_t numWords, const std::string & field) -{ - FieldRef ref(field.c_str(), field.size()); - return EXPECT_EQUAL(numWords, FieldSearcher::countWords(ref)); -} - -bool -testStringFieldInfo(StrChrFieldSearcher & fs) -{ - assertString(fs, "foo", StringList().add("foo bar baz").add("foo bar").add("baz foo"), Hits().add(0).add(3).add(6)); - assertString(fs, StringList().add("foo").add("bar"), StringList().add("foo bar baz").add("foo bar").add("baz foo"), - HitsList().add(Hits().add(0).add(3).add(6)).add(Hits().add(1).add(4))); - - bool retval = true; - if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo", QTFieldInfo(0, 1, 1)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo", QTFieldInfo(0, 0, 1)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "bar", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "baz", "foo bar baz", QTFieldInfo(0, 1, 3)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "qux", "foo bar baz", QTFieldInfo(0, 0, 3)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", "foo foo foo", QTFieldInfo(0, 3, 3)))) retval = false; - // query term size > last term size - if (!EXPECT_TRUE(assertFieldInfo(fs, "runner", "Road Runner Disco", QTFieldInfo(0, 1, 3)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("roadrun").add("runner"), "Road Runner Disco", - FieldInfoList().add(QTFieldInfo(0, 0, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false; - // multiple terms - if (!EXPECT_TRUE(assertFieldInfo(fs, "foo", StringList().add("foo bar baz").add("foo bar"), - QTFieldInfo(0, 2, 5)))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), "foo bar baz", - FieldInfoList().add(QTFieldInfo(0, 1, 3)).add(QTFieldInfo(0, 1, 3))))) retval = false; - if (!EXPECT_TRUE(assertFieldInfo(fs, StringList().add("foo").add("baz"), StringList().add("foo bar baz").add("foo bar"), - FieldInfoList().add(QTFieldInfo(0, 2, 5)).add(QTFieldInfo(0, 1, 5))))) retval = false; - return retval; -} -bool -testStrChrFieldSearcher(StrChrFieldSearcher & fs) -{ - std::string field = "operators and operator overloading with utf8 char oe = \xc3\x98"; - assertString(fs, "oper", field, Hits()); - assertString(fs, "tor", field, Hits()); - assertString(fs, "oper*", field, Hits().add(0).add(2)); - assertString(fs, "and", field, Hits().add(1)); - - assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits()).add(Hits())); - assertString(fs, StringList().add("and").add("overloading"), field, HitsList().add(Hits().add(1)).add(Hits().add(3))); - - fs.setMatchType(FieldSearcher::PREFIX); - assertString(fs, "oper", field, Hits().add(0).add(2)); - assertString(fs, StringList().add("oper").add("tor"), field, HitsList().add(Hits().add(0).add(2)).add(Hits())); - - fs.setMatchType(FieldSearcher::REGULAR); - if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false; - - { // test handling of several underscores - StringList query = StringList().add("foo").add("bar"); - HitsList exp = HitsList().add(Hits().add(0)).add(Hits().add(1)); - assertString(fs, query, "foo_bar", exp); - assertString(fs, query, "foo__bar", exp); - assertString(fs, query, "foo___bar", exp); - assertString(fs, query, "foo________bar", exp); - assertString(fs, query, "foo____________________bar", exp); - assertString(fs, query, "________________________________________foo________________________________________bar________________________________________", exp); - query = StringList().add("foo").add("thisisaveryveryverylongword"); - assertString(fs, query, "foo____________________thisisaveryveryverylongword", exp); - - assertString(fs, "bar", "foo bar", Hits().add(1)); - assertString(fs, "bar", "foo____________________bar", Hits().add(1)); - assertString(fs, "bar", "foo____________________thisisaveryveryverylongword____________________bar", Hits().add(2)); - } - return true; -} - - TEST("verify correct term parsing") { - ASSERT_TRUE(Query::parseQueryTerm("index:term").first == "index"); - ASSERT_TRUE(Query::parseQueryTerm("index:term").second == "term"); - ASSERT_TRUE(Query::parseQueryTerm("term").first == ""); - ASSERT_TRUE(Query::parseQueryTerm("term").second == "term"); - ASSERT_TRUE(Query::parseTerm("*substr*").first == "substr"); - ASSERT_TRUE(Query::parseTerm("*substr*").second == TermType::SUBSTRINGTERM); - ASSERT_TRUE(Query::parseTerm("*suffix").first == "suffix"); - ASSERT_TRUE(Query::parseTerm("*suffix").second == TermType::SUFFIXTERM); - ASSERT_TRUE(Query::parseTerm("prefix*").first == "prefix"); - ASSERT_TRUE(Query::parseTerm("prefix*").second == TermType::PREFIXTERM); - ASSERT_TRUE(Query::parseTerm("term").first == "term"); - ASSERT_TRUE(Query::parseTerm("term").second == TermType::WORD); - } - - TEST("suffix matching") { - EXPECT_EQUAL(assertMatchTermSuffix("a", "vespa"), true); - EXPECT_EQUAL(assertMatchTermSuffix("spa", "vespa"), true); - EXPECT_EQUAL(assertMatchTermSuffix("vespa", "vespa"), true); - EXPECT_EQUAL(assertMatchTermSuffix("vvespa", "vespa"), false); - EXPECT_EQUAL(assertMatchTermSuffix("fspa", "vespa"), false); - EXPECT_EQUAL(assertMatchTermSuffix("v", "vespa"), false); - } - -TEST("Test basic strchrfield searchers") { - { - UTF8StrChrFieldSearcher fs(0); - EXPECT_TRUE(testStrChrFieldSearcher(fs)); - } - { - FUTF8StrChrFieldSearcher fs(0); - EXPECT_TRUE(testStrChrFieldSearcher(fs)); - } -} - -bool -testUTF8SubStringFieldSearcher(StrChrFieldSearcher & fs) -{ - std::string field = "operators and operator overloading"; - assertString(fs, "rsand", field, Hits()); - assertString(fs, "ove", field, Hits().add(3)); - assertString(fs, "ing", field, Hits().add(3)); - assertString(fs, "era", field, Hits().add(0).add(2)); - assertString(fs, "a", field, Hits().add(0).add(1).add(2).add(3)); - - assertString(fs, StringList().add("dn").add("gn"), field, HitsList().add(Hits()).add(Hits())); - assertString(fs, StringList().add("ato").add("load"), field, HitsList().add(Hits().add(0).add(2)).add(Hits().add(3))); - - assertString(fs, StringList().add("aa").add("ab"), "aaaab", - HitsList().add(Hits().add(0).add(0).add(0)).add(Hits().add(0))); - - if (!EXPECT_TRUE(testStringFieldInfo(fs))) return false; - return true; -} - -TEST("utf8 substring search") { - { - UTF8SubStringFieldSearcher fs(0); - EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs)); - assertString(fs, "aa", "aaaa", Hits().add(0).add(0)); - } - { - UTF8SubStringFieldSearcher fs(0); - EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs)); - assertString(fs, "abc", "abc bcd abc", Hits().add(0).add(2)); - fs.maxFieldLength(4); - assertString(fs, "abc", "abc bcd abc", Hits().add(0)); - } - { - UTF8SubstringSnippetModifier fs(0); - EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs)); - // we don't have 1 term optimization - assertString(fs, "aa", "aaaa", Hits().add(0).add(0).add(0)); - } -} - -TEST("utf8 substring search with empty term") -{ - UTF8SubStringFieldSearcher fs(0); - EXPECT_TRUE(testUTF8SubStringFieldSearcher(fs)); - assertString(fs, "", "abc", Hits()); - assertFieldInfo(fs, "", "abc", QTFieldInfo().setFieldLength(0)); -} - -TEST("utf8 suffix search") { - UTF8SuffixStringFieldSearcher fs(0); - std::string field = "operators and operator overloading"; - assertString(fs, "rsand", field, Hits()); - assertString(fs, "tor", field, Hits().add(2)); - assertString(fs, "tors", field, Hits().add(0)); - - assertString(fs, StringList().add("an").add("din"), field, HitsList().add(Hits()).add(Hits())); - assertString(fs, StringList().add("nd").add("g"), field, HitsList().add(Hits().add(1)).add(Hits().add(3))); - - EXPECT_TRUE(testStringFieldInfo(fs)); -} - -TEST("utf8 exact match") { - UTF8ExactStringFieldSearcher fs(0); - // regular - TEST_DO(assertString(fs, "vespa", "vespa", Hits().add(0))); - TEST_DO(assertString(fs, "vespar", "vespa", Hits())); - TEST_DO(assertString(fs, "vespa", "vespar", Hits())); - TEST_DO(assertString(fs, "vespa", "vespa vespa", Hits())); - TEST_DO(assertString(fs, "vesp", "vespa", Hits())); - TEST_DO(assertString(fs, "vesp*", "vespa", Hits().add(0))); - TEST_DO(assertString(fs, "hutte", "hutte", Hits().add(0))); - TEST_DO(assertString(fs, "hütte", "hütte", Hits().add(0))); - TEST_DO(assertString(fs, "hutte", "hütte", Hits())); - TEST_DO(assertString(fs, "hütte", "hutte", Hits())); - TEST_DO(assertString(fs, "hütter", "hütte", Hits())); - TEST_DO(assertString(fs, "hütte", "hütter", Hits())); -} - -TEST("utf8 flexible searcher"){ - UTF8FlexibleStringFieldSearcher fs(0); - // regular - assertString(fs, "vespa", "vespa", Hits().add(0)); - assertString(fs, "vesp", "vespa", Hits()); - assertString(fs, "esp", "vespa", Hits()); - assertString(fs, "espa", "vespa", Hits()); - - // prefix - assertString(fs, "vesp*", "vespa", Hits().add(0)); - fs.setMatchType(FieldSearcher::PREFIX); - assertString(fs, "vesp", "vespa", Hits().add(0)); - - // substring - fs.setMatchType(FieldSearcher::REGULAR); - assertString(fs, "*esp*", "vespa", Hits().add(0)); - fs.setMatchType(FieldSearcher::SUBSTRING); - assertString(fs, "esp", "vespa", Hits().add(0)); - - // suffix - fs.setMatchType(FieldSearcher::REGULAR); - assertString(fs, "*espa", "vespa", Hits().add(0)); - fs.setMatchType(FieldSearcher::SUFFIX); - assertString(fs, "espa", "vespa", Hits().add(0)); - - fs.setMatchType(FieldSearcher::REGULAR); - EXPECT_TRUE(testStringFieldInfo(fs)); -} - -TEST("bool search") { - BoolFieldSearcher fs(0); - TEST_DO(assertBool(fs, "true", true, true)); - TEST_DO(assertBool(fs, "true", false, false)); - TEST_DO(assertBool(fs, "1", true, true)); - TEST_DO(assertBool(fs, "1", false, false)); - TEST_DO(assertBool(fs, "false", true, false)); - TEST_DO(assertBool(fs, "false", false, true)); - TEST_DO(assertBool(fs, "0", true, false)); - TEST_DO(assertBool(fs, "0", false, true)); - TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"), true, BoolList().add(true).add(false).add(true))); - TEST_DO(assertBool(fs, StringList().add("true").add("false").add("true"), false, BoolList().add(false).add(true).add(false))); -} - -TEST("integer search") -{ - IntFieldSearcher fs(0); - TEST_DO(assertInt(fs, "10", 10, true)); - TEST_DO(assertInt(fs, "9", 10, false)); - TEST_DO(assertInt(fs, ">9", 10, true)); - TEST_DO(assertInt(fs, ">9", 9, false)); - TEST_DO(assertInt(fs, "<11", 10, true)); - TEST_DO(assertInt(fs, "<11", 11, false)); - TEST_DO(assertInt(fs, "-10", -10, true)); - TEST_DO(assertInt(fs, "-9", -10, false)); - TEST_DO(assertInt(fs, "a", 10, false)); - TEST_DO(assertInt(fs, "[-5;5]", -5, true)); - TEST_DO(assertInt(fs, "[-5;5]", 0, true)); - TEST_DO(assertInt(fs, "[-5;5]", 5, true)); - TEST_DO(assertInt(fs, "[-5;5]", -6, false)); - TEST_DO(assertInt(fs, "[-5;5]", 6, false)); - - TEST_DO(assertInt(fs, StringList().add("9").add("11"), 10, BoolList().add(false).add(false))); - TEST_DO(assertInt(fs, StringList().add("9").add("10"), 10, BoolList().add(false).add(true))); - TEST_DO(assertInt(fs, StringList().add("10").add(">9"), 10, BoolList().add(true).add(true))); - - TEST_DO(assertInt(fs, "10", LongList().add(10).add(20).add(10).add(30), Hits().add(0).add(2))); - TEST_DO(assertInt(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30), - HitsList().add(Hits().add(0).add(2)).add(Hits().add(1)))); - - TEST_DO(assertFieldInfo(fs, "10", 10, QTFieldInfo(0, 1, 1))); - TEST_DO(assertFieldInfo(fs, "10", LongList().add(10).add(20).add(10).add(30), QTFieldInfo(0, 2, 4))); - TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), 10, - FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1)))); - TEST_DO(assertFieldInfo(fs, StringList().add("10").add("20"), LongList().add(10).add(20).add(10).add(30), - FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4)))); -} - -TEST("floating point search") -{ - FloatFieldSearcher fs; - TEST_DO(assertFloat(fs, "10", 10, true)); - TEST_DO(assertFloat(fs, "10.5", 10.5, true)); - TEST_DO(assertFloat(fs, "-10.5", -10.5, true)); - TEST_DO(assertFloat(fs, ">10.5", 10.6, true)); - TEST_DO(assertFloat(fs, ">10.5", 10.5, false)); - TEST_DO(assertFloat(fs, "<10.5", 10.4, true)); - TEST_DO(assertFloat(fs, "<10.5", 10.5, false)); - TEST_DO(assertFloat(fs, "10.4", 10.5, false)); - TEST_DO(assertFloat(fs, "-10.4", -10.5, false)); - TEST_DO(assertFloat(fs, "a", 10.5, false)); - TEST_DO(assertFloat(fs, "[-5.5;5.5]", -5.5, true)); - TEST_DO(assertFloat(fs, "[-5.5;5.5]", 0, true)); - TEST_DO(assertFloat(fs, "[-5.5;5.5]", 5.5, true)); - TEST_DO(assertFloat(fs, "[-5.5;5.5]", -5.6, false)); - TEST_DO(assertFloat(fs, "[-5.5;5.5]", 5.6, false)); - - TEST_DO(assertFloat(fs, StringList().add("10").add("11"), 10.5, BoolList().add(false).add(false))); - TEST_DO(assertFloat(fs, StringList().add("10").add("10.5"), 10.5, BoolList().add(false).add(true))); - TEST_DO(assertFloat(fs, StringList().add(">10.4").add("10.5"), 10.5, BoolList().add(true).add(true))); - - TEST_DO(assertFloat(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), Hits().add(0).add(2))); - TEST_DO(assertFloat(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5), - HitsList().add(Hits().add(0).add(2)).add(Hits().add(1)))); - - TEST_DO(assertFieldInfo(fs, "10.5", 10.5, QTFieldInfo(0, 1, 1))); - TEST_DO(assertFieldInfo(fs, "10.5", FloatList().add(10.5).add(20.5).add(10.5).add(30.5), QTFieldInfo(0, 2, 4))); - TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), 10.5, - FieldInfoList().add(QTFieldInfo(0, 1, 1)).add(QTFieldInfo(0, 0, 1)))); - TEST_DO(assertFieldInfo(fs, StringList().add("10.5").add("20.5"), FloatList().add(10.5).add(20.5).add(10.5).add(30.5), - FieldInfoList().add(QTFieldInfo(0, 2, 4)).add(QTFieldInfo(0, 1, 4)))); -} - -TEST("Snippet modifier search") { - // ascii - assertSnippetModifier("f", "foo", "\x1F""f\x1Foo"); - assertSnippetModifier("o", "foo", "f\x1Fo\x1F\x1Fo\x1F"); - assertSnippetModifier("r", "bar", "ba\x1Fr\x1F"); - assertSnippetModifier("foo", "foo foo", "\x1F""foo\x1F \x1F""foo\x1F"); - assertSnippetModifier("aa", "aaaaaa", "\x1F""aa\x1F\x1F""aa\x1F\x1F""aa\x1F"); - assertSnippetModifier("ab", "abcd\x1F""efgh", "\x1F""ab\x1F""cd\x1F""efgh"); - assertSnippetModifier("ef", "abcd\x1F""efgh", "abcd\x1F\x1F""ef\x1Fgh"); - assertSnippetModifier("fg", "abcd\x1F""efgh", "abcd\x1F""e\x1F""fg\x1Fh"); - // the separator overlapping the match is skipped - assertSnippetModifier("cdef", "abcd\x1F""efgh", "ab\x1F""cdef\x1F""gh"); - // no hits - assertSnippetModifier("bb", "aaaaaa", "aaaaaa"); - - - // multiple query terms - assertSnippetModifier(StringList().add("ab").add("cd"), "abcd", "\x1F""ab\x1F\x1F""cd\x1F"); - // when we have overlap we only get the first match - assertSnippetModifier(StringList().add("ab").add("bc"), "abcd", "\x1F""ab\x1F""cd"); - assertSnippetModifier(StringList().add("bc").add("ab"), "abcd", "\x1F""ab\x1F""cd"); - // the separator overlapping the match is skipped - assertSnippetModifier(StringList().add("de").add("ef"), "abcd\x1F""efgh", "abc\x1F""de\x1F""fgh"); - - // cjk - assertSnippetModifier("\xe7\x9f\xb3", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8", - "\x1f\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8"); - assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\xe5\x87\xb1\xe5\x9c\xa8", - "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8"); - // the separator overlapping the match is skipped - assertSnippetModifier("\xe6\x98\x8e\xe5\x87\xb1", "\xe7\x9f\xb3\xe6\x98\x8e\x1f\xe5\x87\xb1\xe5\x9c\xa8", - "\xe7\x9f\xb3\x1f\xe6\x98\x8e\xe5\x87\xb1\x1f\xe5\x9c\xa8"); - - { // check that resizing works - UTF8SubstringSnippetModifier mod; - EXPECT_EQUAL(mod.getModifiedBuf().getLength(), 32u); - EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 0u); - performSearch(mod, StringList().add("a"), StringFieldValue("aaaaaaaaaaaaaaaa")); - EXPECT_EQUAL(mod.getModifiedBuf().getPos(), 16u + 2 * 16u); - EXPECT_TRUE(mod.getModifiedBuf().getLength() >= mod.getModifiedBuf().getPos()); - } -} - -TEST("snippet modifier") { - { // string field value - SnippetModifierSetup sms(StringList().add("ab")); - // multiple invokations - assertSnippetModifier(sms, StringFieldValue("ab"), "\x1F""ab\x1F"); - assertSnippetModifier(sms, StringFieldValue("xxxxabxxxxabxxxx"), "xxxx\x1F""ab\x1Fxxxx\x1F""ab\x1Fxxxx"); - assertSnippetModifier(sms, StringFieldValue("xxabxx"), "xx\x1F""ab\x1Fxx"); - } - { // collection field value - SnippetModifierSetup sms(StringList().add("ab")); - // multiple invokations - assertSnippetModifier(sms, getFieldValue(StringList().add("ab")), "\x1F""ab\x1F"); - assertSnippetModifier(sms, getFieldValue(StringList().add("xxabxx")), "xx\x1F""ab\x1Fxx"); - assertSnippetModifier(sms, getFieldValue(StringList().add("ab").add("xxabxx").add("xxxxxx")), - "\x1F""ab\x1F\x1E""xx\x1F""ab\x1F""xx\x1E""xxxxxx"); - assertSnippetModifier(sms, getFieldValue(StringList().add("cd").add("ef").add("gh")), - "cd\x1E""ef\x1E""gh"); - } - { // check that resizing works - SnippetModifierSetup sms(StringList().add("a")); - EXPECT_EQUAL(sms.modifier.getValueBuf().getLength(), 32u); - EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 0u); - sms.modifier.modify(StringFieldValue("aaaaaaaaaaaaaaaa")); - EXPECT_EQUAL(sms.modifier.getValueBuf().getPos(), 16u + 2 * 16u); - EXPECT_TRUE(sms.modifier.getValueBuf().getLength() >= sms.modifier.getValueBuf().getPos()); - } -} - -TEST("FieldSearchSpec constrution") { - { - FieldSearchSpec f; - EXPECT_FALSE(f.valid()); - EXPECT_EQUAL(0u, f.id()); - EXPECT_EQUAL("", f.name()); - EXPECT_EQUAL(0x100000u, f.maxLength()); - } - { - FieldSearchSpec f(7, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 789); - EXPECT_TRUE(f.valid()); - EXPECT_EQUAL(7u, f.id()); - EXPECT_EQUAL("f0", f.name()); - EXPECT_EQUAL(789u, f.maxLength()); - EXPECT_EQUAL(789u, f.searcher().maxFieldLength()); - } -} - -TEST("snippet modifier manager") { - FieldSearchSpecMapT specMap; - specMap[0] = FieldSearchSpec(0, "f0", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "substring", 1000); - specMap[1] = FieldSearchSpec(1, "f1", VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8, "", 1000); - IndexFieldMapT indexMap; - indexMap["i0"].push_back(0); - indexMap["i1"].push_back(1); - indexMap["i2"].push_back(0); - indexMap["i2"].push_back(1); - - { - SnippetModifierManager man; - Query query(StringList().add("i0:foo")); - man.setup(query.qtl, specMap, indexMap); - assertQueryTerms(man, 0, StringList().add("foo")); - assertQueryTerms(man, 1, StringList()); - } - { - SnippetModifierManager man; - Query query(StringList().add("i1:foo")); - man.setup(query.qtl, specMap, indexMap); - assertQueryTerms(man, 0, StringList()); - assertQueryTerms(man, 1, StringList()); - } - { - SnippetModifierManager man; - Query query(StringList().add("i1:*foo*")); - man.setup(query.qtl, specMap, indexMap); - assertQueryTerms(man, 0, StringList()); - assertQueryTerms(man, 1, StringList().add("foo")); - } - { - SnippetModifierManager man; - Query query(StringList().add("i2:foo").add("i2:*bar*")); - man.setup(query.qtl, specMap, indexMap); - assertQueryTerms(man, 0, StringList().add("foo").add("bar")); - assertQueryTerms(man, 1, StringList().add("bar")); - } - { // check buffer sizes - SnippetModifierManager man; - Query query(StringList().add("i2:foo").add("i2:*bar*")); - man.setup(query.qtl, specMap, indexMap); - { - SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(0)); - UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get(); - EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u); - EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u); - } - { - SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(1)); - UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get(); - EXPECT_EQUAL(sm->getValueBuf().getLength(), 128u); - EXPECT_EQUAL(searcher->getModifiedBuf().getLength(), 64u); - } - } -} - -TEST("Stripping of indexes") -{ - EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f")); - EXPECT_EQUAL("f", FieldSearchSpecMap::stripNonFields("f[0]")); - EXPECT_EQUAL("f[a]", FieldSearchSpecMap::stripNonFields("f[a]")); - - EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a}")); - EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{a0}")); - EXPECT_EQUAL("f{a 0}", FieldSearchSpecMap::stripNonFields("f{a 0}")); - EXPECT_EQUAL("f.value", FieldSearchSpecMap::stripNonFields("f{\"a 0\"}")); -} - -TEST("counting of words") { - EXPECT_TRUE(assertCountWords(0, "")); - EXPECT_TRUE(assertCountWords(0, "?")); - EXPECT_TRUE(assertCountWords(1, "foo")); - EXPECT_TRUE(assertCountWords(2, "foo bar")); - EXPECT_TRUE(assertCountWords(2, "? foo bar")); - EXPECT_TRUE(assertCountWords(2, "foo bar ?")); - - // check that 'a' is counted as 1 word - UTF8StrChrFieldSearcher fs(0); - StringList field = StringList().add("a").add("aa bb cc"); - assertString(fs, "bb", field, Hits().add(2)); - assertString(fs, StringList().add("bb").add("not"), field, HitsList().add(Hits().add(2)).add(Hits())); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt index 01b625b6b3b..fdbd60ce30a 100644 --- a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt +++ b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_searchvisitor_test_app TEST SOURCES searchvisitor_test.cpp DEPENDS - streamingvisitors + streamingvisitors_searchvisitor ) vespa_add_test(NAME streamingvisitors_searchvisitor_test_app COMMAND streamingvisitors_searchvisitor_test_app) diff --git a/streamingvisitors/src/tests/textutil/.gitignore b/streamingvisitors/src/tests/textutil/.gitignore deleted file mode 100644 index 1103f79800a..00000000000 --- a/streamingvisitors/src/tests/textutil/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -textutil_test -vsm_textutil_test_app diff --git a/streamingvisitors/src/tests/textutil/CMakeLists.txt b/streamingvisitors/src/tests/textutil/CMakeLists.txt deleted file mode 100644 index 59817d01137..00000000000 --- a/streamingvisitors/src/tests/textutil/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vsm_textutil_test_app TEST - SOURCES - textutil.cpp - DEPENDS - streamingvisitors -) -vespa_add_test(NAME vsm_textutil_test_app COMMAND vsm_textutil_test_app) diff --git a/streamingvisitors/src/tests/textutil/textutil.cpp b/streamingvisitors/src/tests/textutil/textutil.cpp deleted file mode 100644 index 2a1390eaa01..00000000000 --- a/streamingvisitors/src/tests/textutil/textutil.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/fastlib/text/unicodeutil.h> -#include <vespa/searchlib/query/base.h> -#include <vespa/vsm/searcher/fold.h> -#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> -#include <vespa/vsm/searcher/utf8stringfieldsearcherbase.h> - -using search::byte; // unsigned char - -namespace vsm { - -template <typename T> -class Vector : public std::vector<T> -{ -public: - Vector() : std::vector<T>() {} - Vector<T> & a(T v) { this->push_back(v); return *this; } -}; - -typedef Vector<ucs4_t> UCS4V; -typedef Vector<size_t> SizeV; -typedef UTF8StringFieldSearcherBase SFSB; -typedef FUTF8StrChrFieldSearcher FSFS; - -class TextUtilTest : public vespalib::TestApp -{ -private: - ucs4_t getUTF8Char(const char * src); - template <typename BW, bool OFF> - void assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets); - void assertAnsiFold(const std::string & toFold, const std::string & exp); - void assertAnsiFold(char c, char exp); -#ifdef __x86_64__ - void assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp); - void assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded = 16); -#endif - - template <typename BW, bool OFF> - void testSkipSeparators(); - void testSkipSeparators(); - void testSeparatorCharacter(); - void testAnsiFold(); - void test_lfoldua(); -#ifdef __x86_64__ - void test_sse2_foldua(); -#endif - -public: - int Main() override; -}; - -ucs4_t -TextUtilTest::getUTF8Char(const char * src) -{ - ucs4_t retval = Fast_UnicodeUtil::GetUTF8Char(src); - ASSERT_TRUE(retval != Fast_UnicodeUtil::_BadUTF8Char); - return retval; -} - -template <typename BW, bool OFF> -void -TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets) -{ - const byte * srcbuf = reinterpret_cast<const byte *>(input); - auto dstbuf = std::make_unique<ucs4_t[]>(len + 1); - auto offsets = std::make_unique<size_t[]>(len + 1); - UTF8StrChrFieldSearcher fs; - BW bw(dstbuf.get(), offsets.get()); - size_t dstlen = fs.skipSeparators(srcbuf, len, bw); - EXPECT_EQUAL(dstlen, expdstbuf.size()); - ASSERT_TRUE(dstlen == expdstbuf.size()); - for (size_t i = 0; i < dstlen; ++i) { - EXPECT_EQUAL(dstbuf[i], expdstbuf[i]); - if (OFF) { - EXPECT_EQUAL(offsets[i], expoffsets[i]); - } - } -} - -void -TextUtilTest::assertAnsiFold(const std::string & toFold, const std::string & exp) -{ - char folded[256]; - EXPECT_TRUE(FSFS::ansiFold(toFold.c_str(), toFold.size(), folded)); - EXPECT_EQUAL(std::string(folded, toFold.size()), exp); -} - -void -TextUtilTest::assertAnsiFold(char c, char exp) -{ - char folded; - EXPECT_TRUE(FSFS::ansiFold(&c, 1, &folded)); - EXPECT_EQUAL((int32_t)folded, (int32_t)exp); -} - -#ifdef __x86_64__ -void -TextUtilTest::assert_sse2_foldua(const std::string & toFold, size_t charFolded, const std::string & exp) -{ - char folded[256]; - size_t alignedStart = 0xF - (size_t(folded + 0xF) % 0x10); - const unsigned char * toFoldOrg = reinterpret_cast<const unsigned char *>(toFold.c_str()); - const unsigned char * retval = - sse2_foldua(toFoldOrg, toFold.size(), reinterpret_cast<unsigned char *>(folded + alignedStart)); - EXPECT_EQUAL((size_t)(retval - toFoldOrg), charFolded); - EXPECT_EQUAL(std::string(folded + alignedStart, charFolded), exp); -} - -void -TextUtilTest::assert_sse2_foldua(unsigned char c, unsigned char exp, size_t charFolded) -{ - unsigned char toFold[16]; - memset(toFold, c, 16); - unsigned char folded[32]; - size_t alignedStart = 0xF - (size_t(folded + 0xF) % 0x10); - const unsigned char * retval = sse2_foldua(toFold, 16, folded + alignedStart); - EXPECT_EQUAL((size_t)(retval - toFold), charFolded); - for (size_t i = 0; i < charFolded; ++i) { - EXPECT_EQUAL((int32_t)folded[i + alignedStart], (int32_t)exp); - } -} -#endif - -template <typename BW, bool OFF> -void -TextUtilTest::testSkipSeparators() -{ - // ascii characters - assertSkipSeparators<BW, OFF>("foo", 3, UCS4V().a('f').a('o').a('o'), SizeV().a(0).a(1).a(2)); - assertSkipSeparators<BW, OFF>("f\x1Fo", 3, UCS4V().a('f').a('o'), SizeV().a(0).a(2)); - assertSkipSeparators<BW, OFF>("f\no", 3, UCS4V().a('f').a('\n').a('o'), SizeV().a(0).a(1).a(2)); - assertSkipSeparators<BW, OFF>("f\to", 3, UCS4V().a('f').a('\t').a('o'), SizeV().a(0).a(1).a(2)); - - // utf8 char - assertSkipSeparators<BW, OFF>("\xC2\x80\x66", 3, UCS4V().a(getUTF8Char("\xC2\x80")).a('f'), - SizeV().a(0).a(2)); - assertSkipSeparators<BW, OFF>("\xE0\xA0\x80\x66", 4, UCS4V().a(getUTF8Char("\xE0\xA0\x80")).a('f'), - SizeV().a(0).a(3)); - assertSkipSeparators<BW, OFF>("\xF0\x90\x80\x80\x66", 5, UCS4V().a(getUTF8Char("\xF0\x90\x80\x80")).a('f'), - SizeV().a(0).a(4)); - - // replacement string (sharp s -> ss) - assertSkipSeparators<BW, OFF>("\xC3\x9F\x66\xC3\x9F", 5, UCS4V().a('s').a('s').a('f').a('s').a('s'), - SizeV().a(0).a(0).a(2).a(3).a(3)); -} - -void -TextUtilTest::testSkipSeparators() -{ - Fast_NormalizeWordFolder::Setup(Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION); - - testSkipSeparators<SFSB::BufferWrapper, false>(); - testSkipSeparators<SFSB::OffsetWrapper, true>(); -} - -void -TextUtilTest::testSeparatorCharacter() -{ - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x00')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x01')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x02')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x03')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x04')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x05')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x06')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x07')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x08')); - EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x09')); // '\t' - EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x0a')); // '\n' - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0b')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0c')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0d')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0e')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x0f')); - - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x10')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x11')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x12')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x13')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x14')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x15')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x16')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x17')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x18')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x19')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1a')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1b')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1c')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1d')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1e')); - EXPECT_TRUE(SFSB::isSeparatorCharacter('\x1f')); - - EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x20')); // space -} - -void -TextUtilTest::testAnsiFold() -{ - FieldSearcher::init(); - assertAnsiFold("", ""); - assertAnsiFold("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"); - assertAnsiFold("abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"); - assertAnsiFold("0123456789", "0123456789"); - for (int i = 0; i < 128; ++i) { - if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) { - assertAnsiFold(i, i); - } else if (i >= 'A' && i <= 'Z') { - assertAnsiFold(i, i + 32); - } else { - assertAnsiFold(i, 0); - } - } - - // non-ascii is ignored - for (int i = 128; i < 256; ++i) { - char toFold = i; - char folded; - EXPECT_TRUE(!FSFS::ansiFold(&toFold, 1, &folded)); - } -} - -void -TextUtilTest::test_lfoldua() -{ - FieldSearcher::init(); - char folded[256]; - size_t alignedStart = 0; - const char * toFold = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - size_t len = strlen(toFold); - EXPECT_TRUE(FSFS::lfoldua(toFold, len, folded, alignedStart)); - EXPECT_EQUAL(std::string(folded + alignedStart, len), "abcdefghijklmnopqrstuvwxyz"); -} - -#ifdef __x86_64__ -void -TextUtilTest::test_sse2_foldua() -{ - assert_sse2_foldua("", 0, ""); - assert_sse2_foldua("ABCD", 0, ""); - assert_sse2_foldua("ABCDEFGHIJKLMNO", 0, ""); - assert_sse2_foldua("ABCDEFGHIJKLMNOP", 16, "abcdefghijklmnop"); - assert_sse2_foldua("ABCDEFGHIJKLMNOPQ", 16, "abcdefghijklmnop"); - assert_sse2_foldua("KLMNOPQRSTUVWXYZ", 16, "klmnopqrstuvwxyz"); - assert_sse2_foldua("abcdefghijklmnop", 16, "abcdefghijklmnop"); - assert_sse2_foldua("klmnopqrstuvwxyz", 16, "klmnopqrstuvwxyz"); - assert_sse2_foldua("0123456789abcdef", 16, "0123456789abcdef"); - - for (int i = 0; i < 128; ++i) { - if ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) { - assert_sse2_foldua(i, i); - } else if (i >= 'A' && i <= 'Z') { - assert_sse2_foldua(i, i + 32); - } else { - assert_sse2_foldua(i, 0); - } - } - - // non-ascii is ignored - for (int i = 128; i < 256; ++i) { - assert_sse2_foldua(i, '?', 0); - } -} -#endif - -int -TextUtilTest::Main() -{ - TEST_INIT("textutil_test"); - - testSkipSeparators(); - testSeparatorCharacter(); - testAnsiFold(); - test_lfoldua(); -#ifdef __x86_64__ - test_sse2_foldua(); -#endif - - TEST_DONE(); -} - -} - -TEST_APPHOOK(vsm::TextUtilTest); diff --git a/streamingvisitors/src/tests/utilapps/.gitignore b/streamingvisitors/src/tests/utilapps/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/utilapps/.gitignore +++ /dev/null |