diff options
author | Tor Egge <Tor.Egge@online.no> | 2022-10-27 13:51:28 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2022-10-27 13:51:28 +0200 |
commit | a2a67715b01218bfeef547c91c3026aa2b06b9ee (patch) | |
tree | fd2fd74bba3bdc65d1c241ee15515857a8baf892 /searchlib | |
parent | 15719d50be8f37fdf78f4e6855af26c7d47d0ad8 (diff) |
Rewrite diskindex unit test to use gtest.
Diffstat (limited to 'searchlib')
-rw-r--r-- | searchlib/src/tests/diskindex/diskindex/CMakeLists.txt | 1 | ||||
-rw-r--r-- | searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp | 344 |
2 files changed, 225 insertions, 120 deletions
diff --git a/searchlib/src/tests/diskindex/diskindex/CMakeLists.txt b/searchlib/src/tests/diskindex/diskindex/CMakeLists.txt index 9a19c1bf1d5..7dff06507f4 100644 --- a/searchlib/src/tests/diskindex/diskindex/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/diskindex/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(searchlib_diskindex_test_app TEST SOURCES diskindex_test.cpp DEPENDS - searchlib searchlib_test ) vespa_add_test(NAME searchlib_diskindex_test_app COMMAND searchlib_diskindex_test_app) diff --git a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp index 2e8617b4dcb..df168fa3bc6 100644 --- a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp +++ b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/diskindex/disktermblueprint.h> #include <vespa/searchlib/test/diskindex/testdiskindex.h> +#define ENABLE_GTEST_MIGRATION #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/diskindex/zcposocciterators.h> @@ -14,23 +14,39 @@ #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <filesystem> -#include <iostream> #include <set> +using search::BitVector; using search::BitVectorIterator; -using namespace search::fef; -using namespace search::index; -using namespace search::query; -using namespace search::queryeval; -using namespace search::queryeval::blueprint; +using search::diskindex::DiskIndex; +using search::diskindex::DiskTermBlueprint; +using search::diskindex::TestDiskIndex; +using search::diskindex::ZcRareWordPosOccIterator; +using search::fef::TermFieldMatchDataArray; +using search::index::DummyFileHeaderContext; +using search::index::PostingListHandle; +using search::index::Schema; +using search::query::SimpleStringTerm; +using search::queryeval::Blueprint; +using search::queryeval::BooleanMatchIteratorWrapper; +using search::queryeval::EmptyBlueprint; +using search::queryeval::EmptySearch; +using search::queryeval::ExecuteInfo; +using search::queryeval::FakeRequestContext; +using search::queryeval::FieldSpec; +using search::queryeval::LeafBlueprint; +using search::queryeval::SearchIterator; using search::test::SearchIteratorVerifier; -using namespace search::fakedata; +using search::fakedata::FPFactory; +using search::fakedata::FakePosting; +using search::fakedata::FakeWord; +using search::fakedata::getFPFactory; +using LookupResult = DiskIndex::LookupResult; -namespace search { -namespace diskindex { - -typedef DiskIndex::LookupResult LookupResult; +namespace { std::string toString(SearchIterator & sb) @@ -51,23 +67,6 @@ makeTerm(const std::string & term) return SimpleStringTerm(term, "field", 0, search::query::Weight(0)); } -class Test : public vespalib::TestApp, public TestDiskIndex { -private: - FakeRequestContext _requestContext; - - void requireThatLookupIsWorking(bool fieldEmpty, bool docEmpty, bool wordEmpty); - void requireThatWeCanReadPostingList(); - void require_that_we_can_get_field_length_info(); - void requireThatWeCanReadBitVector(); - void requireThatBlueprintIsCreated(); - void requireThatBlueprintCanCreateSearchIterators(); - void requireThatSearchIteratorsConforms(); -public: - Test(); - ~Test(); - int Main() override; -}; - class Verifier : public SearchIteratorVerifier { public: Verifier(FakePosting::SP fp); @@ -96,14 +95,67 @@ Verifier::Verifier(FakePosting::SP fp) Verifier::~Verifier() = default; +struct EmptySettings +{ + bool _empty_field; + bool _empty_doc; + bool _empty_word; + EmptySettings() + : _empty_field(false), + _empty_doc(false), + _empty_word(false) + { + } + EmptySettings empty_field() && { _empty_field = true; return *this; } + EmptySettings empty_doc() && { _empty_doc = true; return *this; } + EmptySettings empty_word() && { _empty_word = true; return *this; } +}; + +struct IOSettings +{ + bool _use_directio; + bool _use_mmap; + IOSettings() + : _use_directio(false), + _use_mmap(false) + { + } + IOSettings use_directio() && { _use_directio = true; return *this; } + IOSettings use_mmap() && { _use_mmap = true; return *this; } +}; + +class DiskIndexTest : public ::testing::Test, public TestDiskIndex { +private: + FakeRequestContext _requestContext; + +protected: + void requireThatLookupIsWorking(const EmptySettings& empty_settings); + void requireThatWeCanReadPostingList(); + void require_that_we_can_get_field_length_info(); + void requireThatWeCanReadBitVector(); + void requireThatBlueprintIsCreated(); + void requireThatBlueprintCanCreateSearchIterators(); + void requireThatSearchIteratorsConforms(); + void build_index(const IOSettings& io_settings, const EmptySettings& empty_settings); + void test_empty_settings(const EmptySettings& empty_settings); + void test_io_settings(const IOSettings& io_settings); +public: + DiskIndexTest(); + ~DiskIndexTest(); +}; + +DiskIndexTest::DiskIndexTest() = default; + +DiskIndexTest::~DiskIndexTest() = default; + void -Test::requireThatSearchIteratorsConforms() +DiskIndexTest::requireThatSearchIteratorsConforms() { FakePosting::SP tmp; Verifier verTmp(tmp); Schema schema; schema.addIndexField(Schema::IndexField("a", Schema::DataType::STRING)); - bitcompression::PosOccFieldsParams params; + search::bitcompression::PosOccFieldsParams params; params.setSchemaParams(schema, 0); search::fakedata::FakeWord fw(verTmp.getDocIdLimit(), verTmp.getExpectedDocIds(), "a", params, 0); TermFieldMatchData md; @@ -118,22 +170,23 @@ Test::requireThatSearchIteratorsConforms() "EGCompr64FilterOcc", "EGCompr64LEFilterOcc", "EGCompr64NoSkipFilterOcc", "EGCompr64SkipFilterOcc" }; for (auto postingType : search::fakedata::getPostingTypes()) { + SCOPED_TRACE(postingType); if (ignored.find(postingType) == ignored.end()) { - std::cerr << "Verifying " << postingType << std::endl; std::unique_ptr<FPFactory> ff(getFPFactory(postingType, schema)); ff->setup(v); FakePosting::SP f(ff->make(fw)); Verifier verifier(f); - TEST_DO(verifier.verify()); + verifier.verify(); } } } void -Test::requireThatLookupIsWorking(bool fieldEmpty, - bool docEmpty, - bool wordEmpty) +DiskIndexTest::requireThatLookupIsWorking(const EmptySettings& empty_settings) { + auto fieldEmpty = empty_settings._empty_field; + auto docEmpty = empty_settings._empty_doc; + auto wordEmpty = empty_settings._empty_word; uint32_t f1(_schema.getIndexFieldId("f1")); uint32_t f2(_schema.getIndexFieldId("f2")); uint32_t f3(_schema.getIndexFieldId("f3")); @@ -149,8 +202,8 @@ Test::requireThatLookupIsWorking(bool fieldEmpty, if (wordEmpty || fieldEmpty || docEmpty) { EXPECT_TRUE(!r || r->counts._numDocs == 0); } else { - EXPECT_EQUAL(1u, r->wordNum); - EXPECT_EQUAL(2u, r->counts._numDocs); + EXPECT_EQ(1u, r->wordNum); + EXPECT_EQ(2u, r->counts._numDocs); } r = _index->lookup(f1, "w2"); EXPECT_TRUE(!r || r->counts._numDocs == 0); @@ -160,15 +213,15 @@ Test::requireThatLookupIsWorking(bool fieldEmpty, if (wordEmpty || fieldEmpty || docEmpty) { EXPECT_TRUE(!r || r->counts._numDocs == 0); } else { - EXPECT_EQUAL(1u, r->wordNum); - EXPECT_EQUAL(3u, r->counts._numDocs); + EXPECT_EQ(1u, r->wordNum); + EXPECT_EQ(3u, r->counts._numDocs); } r = _index->lookup(f2, "w2"); if (wordEmpty || fieldEmpty || docEmpty) { EXPECT_TRUE(!r || r->counts._numDocs == 0); } else { - EXPECT_EQUAL(2u, r->wordNum); - EXPECT_EQUAL(17u, r->counts._numDocs); + EXPECT_EQ(2u, r->wordNum); + EXPECT_EQ(17u, r->counts._numDocs); } } { // field 'f3' doesn't exist @@ -180,7 +233,7 @@ Test::requireThatLookupIsWorking(bool fieldEmpty, } void -Test::requireThatWeCanReadPostingList() +DiskIndexTest::requireThatWeCanReadPostingList() { TermFieldMatchDataArray mda; { // field 'f1' @@ -188,27 +241,27 @@ Test::requireThatWeCanReadPostingList() PostingListHandle::UP h = _index->readPostingList(*r); SearchIterator * sb = h->createIterator(r->counts, mda); sb->initFullRange(); - EXPECT_EQUAL("1,3", toString(*sb)); + EXPECT_EQ("1,3", toString(*sb)); delete sb; } } void -Test::require_that_we_can_get_field_length_info() +DiskIndexTest::require_that_we_can_get_field_length_info() { auto info = _index->get_field_length_info("f1"); - EXPECT_EQUAL(3.5, info.get_average_field_length()); - EXPECT_EQUAL(21u, info.get_num_samples()); + EXPECT_EQ(3.5, info.get_average_field_length()); + EXPECT_EQ(21u, info.get_num_samples()); info = _index->get_field_length_info("f2"); - EXPECT_EQUAL(4.0, info.get_average_field_length()); - EXPECT_EQUAL(23u, info.get_num_samples()); + EXPECT_EQ(4.0, info.get_average_field_length()); + EXPECT_EQ(23u, info.get_num_samples()); info = _index->get_field_length_info("f3"); - EXPECT_EQUAL(0.0, info.get_average_field_length()); - EXPECT_EQUAL(0u, info.get_num_samples()); + EXPECT_EQ(0.0, info.get_average_field_length()); + EXPECT_EQ(0u, info.get_num_samples()); } void -Test::requireThatWeCanReadBitVector() +DiskIndexTest::requireThatWeCanReadBitVector() { { // word 'w1' LookupResult::UP r = _index->lookup(1, "w1"); @@ -229,7 +282,7 @@ Test::requireThatWeCanReadBitVector() } void -Test::requireThatBlueprintIsCreated() +DiskIndexTest::requireThatBlueprintIsCreated() { { // unknown field Blueprint::UP b = @@ -245,7 +298,7 @@ Test::requireThatBlueprintIsCreated() Blueprint::UP b = _index->createBlueprint(_requestContext, FieldSpec("f1", 0, 0), makeTerm("w1")); EXPECT_TRUE(dynamic_cast<DiskTermBlueprint *>(b.get()) != NULL); - EXPECT_EQUAL(2u, b->getState().estimate().estHits); + EXPECT_EQ(2u, b->getState().estimate().estHits); EXPECT_TRUE(!b->getState().estimate().empty); } { // known field & word without hits @@ -254,13 +307,13 @@ Test::requireThatBlueprintIsCreated() // std::cerr << "BP = " << typeid(*b).name() << std::endl; EXPECT_TRUE((dynamic_cast<DiskTermBlueprint *>(b.get()) != NULL) || (dynamic_cast<EmptyBlueprint *>(b.get()) != NULL)); - EXPECT_EQUAL(0u, b->getState().estimate().estHits); + EXPECT_EQ(0u, b->getState().estimate().estHits); EXPECT_TRUE(b->getState().estimate().empty); } } void -Test::requireThatBlueprintCanCreateSearchIterators() +DiskIndexTest::requireThatBlueprintCanCreateSearchIterators() { TermFieldMatchData md; TermFieldMatchDataArray mda; @@ -269,13 +322,13 @@ Test::requireThatBlueprintCanCreateSearchIterators() SearchIterator::UP s; { // bit vector due to isFilter b = _index->createBlueprint(_requestContext, FieldSpec("f2", 0, 0, true), makeTerm("w2")); - b->fetchPostings(queryeval::ExecuteInfo::TRUE); + b->fetchPostings(search::queryeval::ExecuteInfo::TRUE); s = (dynamic_cast<LeafBlueprint *>(b.get()))->createLeafSearch(mda, true); EXPECT_TRUE(dynamic_cast<BitVectorIterator *>(s.get()) != NULL); } { // bit vector due to no ranking needed b = _index->createBlueprint(_requestContext, FieldSpec("f2", 0, 0, false), makeTerm("w2")); - b->fetchPostings(queryeval::ExecuteInfo::TRUE); + b->fetchPostings(ExecuteInfo::TRUE); s = (dynamic_cast<LeafBlueprint *>(b.get()))->createLeafSearch(mda, true); EXPECT_FALSE(dynamic_cast<BitVectorIterator *>(s.get()) != NULL); TermFieldMatchData md2; @@ -289,7 +342,7 @@ Test::requireThatBlueprintCanCreateSearchIterators() { // fake bit vector b = _index->createBlueprint(_requestContext, FieldSpec("f1", 0, 0, true), makeTerm("w2")); // std::cerr << "BP = " << typeid(*b).name() << std::endl; - b->fetchPostings(queryeval::ExecuteInfo::TRUE); + b->fetchPostings(ExecuteInfo::TRUE); s = (dynamic_cast<LeafBlueprint *>(b.get()))->createLeafSearch(mda, true); // std::cerr << "SI = " << typeid(*s).name() << std::endl; EXPECT_TRUE((dynamic_cast<BooleanMatchIteratorWrapper *>(s.get()) != NULL) || @@ -297,78 +350,131 @@ Test::requireThatBlueprintCanCreateSearchIterators() } { // posting list iterator b = _index->createBlueprint(_requestContext, FieldSpec("f1", 0, 0), makeTerm("w1")); - b->fetchPostings(queryeval::ExecuteInfo::TRUE); + b->fetchPostings(ExecuteInfo::TRUE); s = (dynamic_cast<LeafBlueprint *>(b.get()))->createLeafSearch(mda, true); ASSERT_TRUE((dynamic_cast<ZcRareWordPosOccIterator<true, false> *>(s.get()) != NULL)); } } -Test::Test() = default; +void +DiskIndexTest::build_index(const IOSettings& io_settings, const EmptySettings& empty_settings) +{ + vespalib::asciistream name; + int io_settings_num = 1; + if (io_settings._use_directio) { + io_settings_num += 1; + } + if (io_settings._use_mmap) { + io_settings_num += 2; + } + name << "index/" << io_settings_num; + if (empty_settings._empty_field) { + name << "fe"; + } else { + buildSchema(); + } + if (empty_settings._empty_doc) { + name << "de"; + } + if (empty_settings._empty_word) { + name << "we"; + } + openIndex(name.str(), io_settings._use_directio, io_settings._use_mmap, empty_settings._empty_field, empty_settings._empty_doc, empty_settings._empty_word); +} + +void +DiskIndexTest::test_empty_settings(const EmptySettings& empty_settings) +{ + build_index(IOSettings(), empty_settings); + requireThatLookupIsWorking(empty_settings); +} -Test::~Test() = default; +void +DiskIndexTest::test_io_settings(const IOSettings& io_settings) +{ + EmptySettings empty_settings; + build_index(io_settings, empty_settings); + requireThatLookupIsWorking(empty_settings); + requireThatWeCanReadPostingList(); + require_that_we_can_get_field_length_info(); + requireThatWeCanReadBitVector(); + requireThatBlueprintIsCreated(); + requireThatBlueprintCanCreateSearchIterators(); +} -int -Test::Main() +TEST_F(DiskIndexTest, empty_settings_empty_field_empty_doc_empty_word) { - TEST_INIT("diskindex_test"); + test_empty_settings(EmptySettings().empty_field().empty_doc().empty_word()); +} - if (_argc > 0) { - DummyFileHeaderContext::setCreator(_argv[0]); - } +TEST_F(DiskIndexTest, empty_settings_empty_field_empty_doc) +{ + test_empty_settings(EmptySettings().empty_field().empty_doc()); +} + +TEST_F(DiskIndexTest, empty_settings_empty_field_empty_word) +{ + test_empty_settings(EmptySettings().empty_field().empty_word()); +} - std::filesystem::create_directory(std::filesystem::path("index")); - TEST_DO(openIndex("index/1fedewe", false, false, true, true, true)); - TEST_DO(requireThatLookupIsWorking(true, true, true)); - TEST_DO(openIndex("index/1fede", false, false, true, true, false)); - TEST_DO(requireThatLookupIsWorking(true, true, false)); - TEST_DO(openIndex("index/1fewe", false, false, true, false, true)); - TEST_DO(requireThatLookupIsWorking(true, false, true)); - TEST_DO(openIndex("index/1fe", false, false, true, false, false)); - TEST_DO(requireThatLookupIsWorking(true, false, false)); - buildSchema(); - TEST_DO(openIndex("index/1dewe", false, false, false, true, true)); - TEST_DO(requireThatLookupIsWorking(false, true, true)); - TEST_DO(openIndex("index/1de", false, false, false, true, false)); - TEST_DO(requireThatLookupIsWorking(false, true, false)); - TEST_DO(openIndex("index/1we", false, false, false, false, true)); - TEST_DO(requireThatLookupIsWorking(false, false, true)); - TEST_DO(openIndex("index/1", false, false, false, false, false)); - TEST_DO(requireThatLookupIsWorking(false, false, false)); - TEST_DO(requireThatWeCanReadPostingList()); - TEST_DO(require_that_we_can_get_field_length_info()); - TEST_DO(requireThatWeCanReadBitVector()); - TEST_DO(requireThatBlueprintIsCreated()); - TEST_DO(requireThatBlueprintCanCreateSearchIterators()); - - TEST_DO(openIndex("index/2", true, false, false, false, false)); - TEST_DO(requireThatLookupIsWorking(false, false, false)); - TEST_DO(requireThatWeCanReadPostingList()); - TEST_DO(require_that_we_can_get_field_length_info()); - TEST_DO(requireThatWeCanReadBitVector()); - TEST_DO(requireThatBlueprintIsCreated()); - TEST_DO(requireThatBlueprintCanCreateSearchIterators()); - - TEST_DO(openIndex("index/3", false, true, false, false, false)); - TEST_DO(requireThatLookupIsWorking(false, false, false)); - TEST_DO(requireThatWeCanReadPostingList()); - TEST_DO(require_that_we_can_get_field_length_info()); - TEST_DO(requireThatWeCanReadBitVector()); - TEST_DO(requireThatBlueprintIsCreated()); - TEST_DO(requireThatBlueprintCanCreateSearchIterators()); - - TEST_DO(openIndex("index/4", true, true, false, false, false)); - TEST_DO(requireThatLookupIsWorking(false, false, false)); - TEST_DO(requireThatWeCanReadPostingList()); - TEST_DO(require_that_we_can_get_field_length_info()); - TEST_DO(requireThatWeCanReadBitVector()); - TEST_DO(requireThatBlueprintIsCreated()); - TEST_DO(requireThatBlueprintCanCreateSearchIterators()); - TEST_DO(requireThatSearchIteratorsConforms()); - - TEST_DONE(); +TEST_F(DiskIndexTest, empty_settings_empty_field) +{ + test_empty_settings(EmptySettings().empty_field()); } +TEST_F(DiskIndexTest, empty_settings_empty_doc_empty_word) +{ + test_empty_settings(EmptySettings().empty_doc().empty_word()); } + +TEST_F(DiskIndexTest, empty_settings_empty_doc) +{ + test_empty_settings(EmptySettings().empty_doc()); +} + +TEST_F(DiskIndexTest, empty_settings_empty_word) +{ + test_empty_settings(EmptySettings().empty_word()); } -TEST_APPHOOK(search::diskindex::Test); +TEST_F(DiskIndexTest, io_settings_normal) +{ + test_io_settings(IOSettings()); +} + +TEST_F(DiskIndexTest, io_settings_directio) +{ + test_io_settings(IOSettings().use_directio()); +} + +TEST_F(DiskIndexTest, io_settings_mmap) +{ + test_io_settings(IOSettings().use_mmap()); +} + +TEST_F(DiskIndexTest, io_settings_directio_mmap) +{ + test_io_settings(IOSettings().use_directio().use_mmap()); +} + +TEST_F(DiskIndexTest, search_iterators_conformance) +{ + requireThatSearchIteratorsConforms(); +} + +} + +int +main(int argc, char* argv[]) +{ + if (argc > 0) { + DummyFileHeaderContext::setCreator(argv[0]); + } + ::testing::InitGoogleTest(&argc, argv); + std::filesystem::path index_path("index"); + std::filesystem::remove_all(index_path); + std::filesystem::create_directory(index_path); + auto rval = RUN_ALL_TESTS(); + std::filesystem::remove_all(index_path); + return rval; +} |