diff options
Diffstat (limited to 'searchlib/src/tests/document_store')
9 files changed, 545 insertions, 0 deletions
diff --git a/searchlib/src/tests/document_store/.gitignore b/searchlib/src/tests/document_store/.gitignore new file mode 100644 index 00000000000..bc9b97decab --- /dev/null +++ b/searchlib/src/tests/document_store/.gitignore @@ -0,0 +1 @@ +searchlib_document_store_test_app diff --git a/searchlib/src/tests/document_store/CMakeLists.txt b/searchlib/src/tests/document_store/CMakeLists.txt new file mode 100644 index 00000000000..18b9e408fae --- /dev/null +++ b/searchlib/src/tests/document_store/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_store_test_app + SOURCES + document_store_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_document_store_test_app COMMAND searchlib_document_store_test_app) diff --git a/searchlib/src/tests/document_store/FILES b/searchlib/src/tests/document_store/FILES new file mode 100644 index 00000000000..b1dd2b610d0 --- /dev/null +++ b/searchlib/src/tests/document_store/FILES @@ -0,0 +1 @@ +document_store_test.cpp diff --git a/searchlib/src/tests/document_store/document_store_test.cpp b/searchlib/src/tests/document_store/document_store_test.cpp new file mode 100644 index 00000000000..e6a3d9b5c3d --- /dev/null +++ b/searchlib/src/tests/document_store/document_store_test.cpp @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/docstore/documentstore.h> +#include <vespa/searchlib/docstore/cachestats.h> +#include <vespa/document/repo/documenttyperepo.h> + +using namespace search; + +document::DocumentTypeRepo repo; + +struct NullDataStore : IDataStore { + NullDataStore() : IDataStore("") {} + ssize_t read(uint32_t, vespalib::DataBuffer &) const override { return 0; } + void read(const LidVector &, IBufferVisitor &) const override { } + void write(uint64_t, uint32_t, const void *, size_t) override {} + void remove(uint64_t, uint32_t) override {} + void flush(uint64_t) override {} + + uint64_t initFlush(uint64_t syncToken) override { return syncToken; } + + size_t memoryUsed() const override { return 0; } + size_t memoryMeta() const override { return 0; } + size_t getDiskFootprint() const override { return 0; } + size_t getDiskBloat() const override { return 0; } + uint64_t lastSyncToken() const override { return 0; } + uint64_t tentativeLastSyncToken() const override { return 0; } + fastos::TimeStamp getLastFlushTime() const override { return fastos::TimeStamp(); } + void accept(IDataStoreVisitor &, IDataStoreVisitorProgress &, bool) override { } + double getVisitCost() const override { return 1.0; } + virtual DataStoreStorageStats getStorageStats() const override { + return DataStoreStorageStats(0, 0, 0.0, 0, 0); + } + virtual std::vector<DataStoreFileChunkStats> + getFileChunkStats() const override { + std::vector<DataStoreFileChunkStats> result; + return result; + } +}; + +TEST_FFF("require that uncache docstore lookups are counted", + DocumentStore::Config(document::CompressionConfig::NONE, 0, 0), + NullDataStore(), DocumentStore(f1, f2)) +{ + EXPECT_EQUAL(0u, f3.getCacheStats().misses); + f3.read(1, repo); + EXPECT_EQUAL(1u, f3.getCacheStats().misses); +} + +TEST_FFF("require that cached docstore lookups are counted", + DocumentStore::Config(document::CompressionConfig::NONE, 100000, 100), + NullDataStore(), DocumentStore(f1, f2)) +{ + EXPECT_EQUAL(0u, f3.getCacheStats().misses); + f3.read(1, repo); + EXPECT_EQUAL(1u, f3.getCacheStats().misses); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/document_store/visitor/.gitignore b/searchlib/src/tests/document_store/visitor/.gitignore new file mode 100644 index 00000000000..c97186f86d7 --- /dev/null +++ b/searchlib/src/tests/document_store/visitor/.gitignore @@ -0,0 +1 @@ +searchlib_document_store_visitor_test_app diff --git a/searchlib/src/tests/document_store/visitor/CMakeLists.txt b/searchlib/src/tests/document_store/visitor/CMakeLists.txt new file mode 100644 index 00000000000..976463bdfe8 --- /dev/null +++ b/searchlib/src/tests/document_store/visitor/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_store_visitor_test_app + SOURCES + document_store_visitor_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_document_store_visitor_test_app COMMAND searchlib_document_store_visitor_test_app) diff --git a/searchlib/src/tests/document_store/visitor/DESC b/searchlib/src/tests/document_store/visitor/DESC new file mode 100644 index 00000000000..03e9c6681ad --- /dev/null +++ b/searchlib/src/tests/document_store/visitor/DESC @@ -0,0 +1 @@ +Document store visiting test. diff --git a/searchlib/src/tests/document_store/visitor/FILES b/searchlib/src/tests/document_store/visitor/FILES new file mode 100644 index 00000000000..412f9879bb5 --- /dev/null +++ b/searchlib/src/tests/document_store/visitor/FILES @@ -0,0 +1 @@ +document_store_visitor_test.cpp diff --git a/searchlib/src/tests/document_store/visitor/document_store_visitor_test.cpp b/searchlib/src/tests/document_store/visitor/document_store_visitor_test.cpp new file mode 100644 index 00000000000..1898fa35a29 --- /dev/null +++ b/searchlib/src/tests/document_store/visitor/document_store_visitor_test.cpp @@ -0,0 +1,466 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("document_store_visitor_test"); + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/docstore/documentstore.h> +#include <vespa/searchlib/docstore/logdocumentstore.h> +#include <vespa/searchlib/docstore/cachestats.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/vespalib/io/fileutil.h> + +using namespace search; + +using vespalib::string; +using document::DataType; +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using vespalib::asciistream; +using index::DummyFileHeaderContext; + +namespace +{ + +const string doc_type_name = "test"; +const string header_name = doc_type_name + ".header"; +const string body_name = doc_type_name + ".body"; + +document::DocumenttypesConfig +makeDocTypeRepoConfig(void) +{ + const int32_t doc_type_id = 787121340; + document::config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(doc_type_id, + doc_type_name, + document::config_builder::Struct(header_name), + document::config_builder::Struct(body_name). + addField("main", DataType::T_STRING). + addField("extra", DataType::T_STRING)); + return builder.config(); +} + + +Document::UP +makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool before) +{ + asciistream idstr; + idstr << "id:test:test:: " << i; + DocumentId id(idstr.str()); + const DocumentType *docType = repo.getDocumentType(doc_type_name); + Document::UP doc(new Document(*docType, id)); + ASSERT_TRUE(doc.get()); + asciistream mainstr; + mainstr << "static text" << i << " body something"; + for (uint32_t j = 0; j < 10; ++j) { + mainstr << (j + i * 1000) << " "; + } + mainstr << " and end field"; + doc->set("main", mainstr.c_str()); + if (!before) { + doc->set("extra", "foo"); + } + + return doc; +} + +} + +class MyTlSyncer : public transactionlog::SyncProxy +{ + SerialNum _syncedTo; + +public: + MyTlSyncer(void) + : _syncedTo(0) + { + } + + void + sync(SerialNum syncTo) + { + _syncedTo = syncTo; + } +}; + + +class MyVisitorBase +{ +public: + DocumentTypeRepo &_repo; + uint32_t _visitCount; + uint32_t _visitRmCount; + uint32_t _docIdLimit; + BitVector::UP _valid; + bool _before; + + MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool before); +}; + +MyVisitorBase::MyVisitorBase(DocumentTypeRepo &repo, + uint32_t docIdLimit, + bool before) + : _repo(repo), + _visitCount(0u), + _visitRmCount(0u), + _docIdLimit(docIdLimit), + _valid(BitVector::create(docIdLimit)), + _before(before) +{ +} + + +class MyVisitor : public MyVisitorBase, + public IDocumentStoreReadVisitor +{ +public: + using MyVisitorBase::MyVisitorBase; + + virtual void + visit(uint32_t lid, const Document &doc); + + virtual void + visit(uint32_t lid); +}; + + +void +MyVisitor::visit(uint32_t lid, const Document &doc) +{ + ++_visitCount; + assert(lid < _docIdLimit); + Document::UP expDoc(makeDoc(_repo, lid, _before)); + EXPECT_TRUE(*expDoc == doc); + _valid->slowSetBit(lid); +} + + +void +MyVisitor::visit(uint32_t lid) +{ + ++_visitRmCount; + assert(lid < _docIdLimit); + _valid->slowClearBit(lid); +} + + +class MyRewriteVisitor : public MyVisitorBase, + public IDocumentStoreRewriteVisitor +{ +public: + using MyVisitorBase::MyVisitorBase; + + virtual void + visit(uint32_t lid, Document &doc); +}; + + +void +MyRewriteVisitor::visit(uint32_t lid, Document &doc) +{ + ++_visitCount; + assert(lid < _docIdLimit); + Document::UP expDoc(makeDoc(_repo, lid, _before)); + EXPECT_TRUE(*expDoc == doc); + _valid->slowSetBit(lid); + doc.set("extra", "foo"); +} + + +class MyVisitorProgress : public IDocumentStoreVisitorProgress +{ +public: + double _progress; + uint32_t _updates; + + MyVisitorProgress(); + + virtual void + updateProgress(double progress); + + virtual double + getProgress() const; +}; + + +MyVisitorProgress::MyVisitorProgress() + : _progress(0.0), + _updates(0) +{ +} + + +void +MyVisitorProgress::updateProgress(double progress) +{ + EXPECT_TRUE(progress >= _progress); + _progress = progress; + ++_updates; + LOG(info, + "updateProgress(%6.2f), %u updates", + progress, _updates); +} + + +double +MyVisitorProgress::getProgress() const +{ + return _progress; +} + + +struct Fixture +{ + string _baseDir; + DocumentTypeRepo _repo; + LogDocumentStore::Config _storeConfig; + vespalib::ThreadStackExecutor _executor; + DummyFileHeaderContext _fileHeaderContext; + MyTlSyncer _tlSyncer; + std::unique_ptr<LogDocumentStore> _store; + uint64_t _syncToken; + uint32_t _docIdLimit; + BitVector::UP _valid; + + Fixture(); + + ~Fixture(); + + Document::UP + makeDoc(uint32_t i); + + void + resetDocStore(); + + void + mkdir(); + + void + rmdir(); + + void + setDocIdLimit(uint32_t docIdLimit); + + void + put(const Document &doc, uint32_t lid); + + void + remove(uint32_t lid); + + void + flush(); + + void + populate(uint32_t low, uint32_t high, uint32_t docIdLimit); + + void + applyRemoves(uint32_t rmDocs); + + void + checkRemovePostCond(uint32_t numDocs, + uint32_t docIdLimit, + uint32_t rmDocs, + bool before); +}; + +Fixture::Fixture() + : _baseDir("visitor"), + _repo(makeDocTypeRepoConfig()), + _storeConfig(DocumentStore:: + Config(document::CompressionConfig::NONE, 0, 0), + LogDataStore:: + Config(50000, 0.2, 3.0, 0.2, 1, true, + WriteableFileChunk::Config( + document::CompressionConfig(), + 16384, + 64))), + _executor(_storeConfig.getLogConfig().getNumThreads(), 128 * 1024), + _fileHeaderContext(), + _tlSyncer(), + _store(), + _syncToken(0u), + _docIdLimit(0u), + _valid(BitVector::create(0u)) +{ + rmdir(); + mkdir(); + resetDocStore(); +} + + +Fixture::~Fixture() +{ + _store.reset(); + rmdir(); +} + +Document::UP +Fixture::makeDoc(uint32_t i) +{ + return ::makeDoc(_repo, i, true); +} + +void +Fixture::resetDocStore() +{ + _store.reset(new LogDocumentStore(_executor, + _baseDir, + _storeConfig, + GrowStrategy(), + TuneFileSummary(), + _fileHeaderContext, + _tlSyncer, + NULL)); +} + + +void +Fixture::rmdir() +{ + vespalib::rmdir(_baseDir, true); +} + +void +Fixture::mkdir() +{ + vespalib::mkdir(_baseDir, false); +} + + +void +Fixture::setDocIdLimit(uint32_t docIdLimit) +{ + _docIdLimit = docIdLimit; + _valid->resize(_docIdLimit); +} + +void +Fixture::put(const Document &doc, uint32_t lid) +{ + ++_syncToken; + assert(lid < _docIdLimit); + _store->write(_syncToken, doc, lid); + _valid->slowSetBit(lid); +} + + +void +Fixture::remove(uint32_t lid) +{ + ++_syncToken; + assert(lid < _docIdLimit); + _store->remove(_syncToken, lid); + _valid->slowClearBit(lid); +} + + +void +Fixture::flush() +{ + _store->initFlush(_syncToken); + _store->flush(_syncToken); +} + + +void +Fixture::populate(uint32_t low, uint32_t high, uint32_t docIdLimit) +{ + setDocIdLimit(docIdLimit); + for (uint32_t lid = low; lid < high; ++lid) { + Document::UP doc = makeDoc(lid); + put(*doc, lid); + } +} + + +void +Fixture::applyRemoves(uint32_t rmDocs) +{ + for (uint32_t lid = 20; lid < 20 + rmDocs; ++lid) { + remove(lid); + } + put(*makeDoc(25), 25); + remove(25); + put(*makeDoc(25), 25); +} + + +void +Fixture::checkRemovePostCond(uint32_t numDocs, + uint32_t docIdLimit, + uint32_t rmDocs, + bool before) +{ + MyVisitor visitor(_repo, docIdLimit, before); + MyVisitorProgress visitorProgress; + EXPECT_EQUAL(0.0, visitorProgress.getProgress()); + EXPECT_EQUAL(0u, visitorProgress._updates); + _store->accept(visitor, visitorProgress, _repo); + EXPECT_EQUAL(numDocs - rmDocs + 1, visitor._visitCount); + EXPECT_EQUAL(rmDocs - 1, visitor._visitRmCount); + EXPECT_EQUAL(1.0, visitorProgress.getProgress()); + EXPECT_NOT_EQUAL(0u, visitorProgress._updates); + EXPECT_TRUE(*_valid == *visitor._valid); +} + + +TEST_F("require that basic visit works", Fixture()) +{ + uint32_t numDocs = 3000; + uint32_t docIdLimit = numDocs + 1; + f.populate(1, docIdLimit, docIdLimit); + f.flush(); + MyVisitor visitor(f._repo, docIdLimit, true); + MyVisitorProgress visitorProgress; + EXPECT_EQUAL(0.0, visitorProgress.getProgress()); + EXPECT_EQUAL(0u, visitorProgress._updates); + f._store->accept(visitor, visitorProgress, f._repo); + EXPECT_EQUAL(numDocs, visitor._visitCount); + EXPECT_EQUAL(0u, visitor._visitRmCount); + EXPECT_EQUAL(1.0, visitorProgress.getProgress()); + EXPECT_NOT_EQUAL(0u, visitorProgress._updates); + EXPECT_TRUE(*f._valid == *visitor._valid); +} + + +TEST_F("require that visit with remove works", Fixture()) +{ + uint32_t numDocs = 1000; + uint32_t docIdLimit = numDocs + 1; + f.populate(1, docIdLimit, docIdLimit); + uint32_t rmDocs = 20; + f.applyRemoves(rmDocs); + f.flush(); + f.checkRemovePostCond(numDocs, docIdLimit, rmDocs, true); +} + +TEST_F("require that visit with rewrite and remove works", Fixture()) +{ + uint32_t numDocs = 1000; + uint32_t docIdLimit = numDocs + 1; + f.populate(1, docIdLimit, docIdLimit); + uint32_t rmDocs = 20; + f.applyRemoves(rmDocs); + f.flush(); + f.checkRemovePostCond(numDocs, docIdLimit, rmDocs, true); + { + MyRewriteVisitor visitor(f._repo, docIdLimit, true); + MyVisitorProgress visitorProgress; + EXPECT_EQUAL(0.0, visitorProgress.getProgress()); + EXPECT_EQUAL(0u, visitorProgress._updates); + f._store->accept(visitor, visitorProgress, f._repo); + EXPECT_EQUAL(numDocs - rmDocs + 1, visitor._visitCount); + EXPECT_EQUAL(1.0, visitorProgress.getProgress()); + EXPECT_NOT_EQUAL(0u, visitorProgress._updates); + EXPECT_TRUE(*f._valid == *visitor._valid); + f.flush(); + } + f.checkRemovePostCond(numDocs, docIdLimit, rmDocs, false); +} + +TEST_MAIN() { TEST_RUN_ALL(); } |