diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-02-04 12:44:06 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-02-04 12:46:01 +0000 |
commit | 8224095b228cfa64c61baf3d7c152f9817d7e399 (patch) | |
tree | dc127aa2c832b8ca90b03a725b24e5da87f9053a /searchcore | |
parent | 20369863ee98ff9ca07078bf7465f229573663b0 (diff) |
Refactor test for easier reuse.
Diffstat (limited to 'searchcore')
5 files changed, 565 insertions, 508 deletions
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt index 1c11ed745a6..e1111343555 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt @@ -1,18 +1,36 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(searchcore_bucketmover_test STATIC + SOURCES + bucketmover_common.cpp +) + vespa_add_executable(searchcore_documentbucketmover_test_app TEST SOURCES documentbucketmover_test.cpp DEPENDS + searchcore_bucketmover_test searchcore_test searchcore_server - searchcore_persistenceengine searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_grouping - searchcore_fconfig + #searchcore_persistenceengine + #searchcore_matching + #searchcore_attribute + #searchcore_documentmetastore + #searchcore_bucketdb + #searchcore_pcommon + #searchcore_grouping + #searchcore_fconfig ) vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app) + +vespa_add_executable(searchcore_scaniterator_test_app TEST + SOURCES + scaniterator_test.cpp + DEPENDS + searchcore_bucketmover_test + searchcore_server + searchcore_test + searchcore_feedoperation +) +vespa_add_test(NAME searchcore_scaniterator_test_app COMMAND searchcore_scaniterator_test_app) diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.cpp new file mode 100644 index 00000000000..76ff3eb74b6 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.cpp @@ -0,0 +1,81 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucketmover_common.h" +#include <vespa/vespalib/testkit/test_macros.h> + +using vespalib::IDestructorCallback; + +namespace proton::move::test { + +void +MyBucketModifiedHandler::notifyBucketModified(const BucketId &bucket) { + auto itr = std::find(_modified.begin(), _modified.end(), bucket); + ASSERT_TRUE(itr == _modified.end()); + _modified.push_back(bucket); +} + +MyMoveHandler::MyMoveHandler(BucketDBOwner &bucketDb, bool storeMoveDoneContext) + : _bucketDb(bucketDb), + _moves(), + _numCachedBuckets(), + _storeMoveDoneContexts(storeMoveDoneContext), + _moveDoneContexts() +{} + +MyMoveHandler::~MyMoveHandler() = default; + +void +MyMoveHandler::handleMove(MoveOperation &op, IDestructorCallback::SP moveDoneCtx) { + _moves.push_back(op); + if (_bucketDb.takeGuard()->isCachedBucket(op.getBucketId())) { + ++_numCachedBuckets; + } + if (_storeMoveDoneContexts) { + _moveDoneContexts.push_back(std::move(moveDoneCtx)); + } +} + +MySubDb::MySubDb(const std::shared_ptr<const DocumentTypeRepo> &repo, std::shared_ptr<BucketDBOwner> bucketDB, + uint32_t subDbId, SubDbType subDbType) + : _metaStoreSP(std::make_shared<DocumentMetaStore>(bucketDB, DocumentMetaStore::getFixedName(), + search::GrowStrategy(), subDbType)), + _metaStore(*_metaStoreSP), + _realRetriever(std::make_shared<MyDocumentRetriever>(repo)), + _retriever(_realRetriever), + _subDb("my_sub_db", subDbId, _metaStoreSP, _retriever, IFeedView::SP(), nullptr), + _docs(), + _bucketDBHandler(*bucketDB) { + _bucketDBHandler.addDocumentMetaStore(_metaStoreSP.get(), 0); +} + +MySubDb::~MySubDb() = default; + +void +MySubDb::insertDocs(const UserDocuments &docs_) { + for (const auto & entry : docs_) { + const auto & bucketDocs = entry.second; + for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { + const auto & testDoc = bucketDocs.getDocs()[i]; + _metaStore.put(testDoc.getGid(), testDoc.getBucket(), + testDoc.getTimestamp(), testDoc.getDocSize(), testDoc.getLid(), 0u); + _realRetriever->_docs.push_back(testDoc.getDoc()); + ASSERT_EQUAL(testDoc.getLid() + 1, _realRetriever->_docs.size()); + } + } + _docs.merge(docs_); +} + +bool +assertEqual(const document::BucketId &bucket, const proton::test::Document &doc, + uint32_t sourceSubDbId, uint32_t targetSubDbId, const MoveOperation &op) { + if (!EXPECT_EQUAL(bucket, op.getBucketId())) return false; + if (!EXPECT_EQUAL(doc.getTimestamp(), op.getTimestamp())) return false; + if (!EXPECT_EQUAL(doc.getDocId(), op.getDocument()->getId())) return false; + if (!EXPECT_EQUAL(doc.getLid(), op.getSourceDbdId().getLid())) return false; + if (!EXPECT_EQUAL(sourceSubDbId, op.getSourceDbdId().getSubDbId())) return false; + if (!EXPECT_EQUAL(0u, op.getTargetDbdId().getLid())) return false; + if (!EXPECT_EQUAL(targetSubDbId, op.getTargetDbdId().getSubDbId())) return false; + return true; +} + +} diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.h b/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.h new file mode 100644 index 00000000000..2702d603078 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.h @@ -0,0 +1,131 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> +#include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> +#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/server/bucketmovejob.h> +#include <vespa/searchcore/proton/server/documentbucketmover.h> +#include <vespa/searchcore/proton/server/i_move_operation_limiter.h> +#include <vespa/searchcore/proton/server/idocumentmovehandler.h> +#include <vespa/searchcore/proton/server/imaintenancejobrunner.h> +#include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> +#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h> +#include <vespa/searchcore/proton/test/buckethandler.h> +#include <vespa/searchcore/proton/test/clusterstatehandler.h> +#include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> +#include <vespa/searchcore/proton/test/test.h> +#include <vespa/document/test/make_bucket_space.h> + +namespace proton::move::test { + +struct MyMoveOperationLimiter : public IMoveOperationLimiter { + uint32_t beginOpCount; + + MyMoveOperationLimiter() : beginOpCount(0) {} + + vespalib::IDestructorCallback::SP beginOperation() override { + ++beginOpCount; + return {}; + } +}; + +struct MyMoveHandler : public IDocumentMoveHandler { + using MoveOperationVector = std::vector<MoveOperation>; + BucketDBOwner &_bucketDb; + MoveOperationVector _moves; + size_t _numCachedBuckets; + bool _storeMoveDoneContexts; + std::vector<vespalib::IDestructorCallback::SP> _moveDoneContexts; + + MyMoveHandler(BucketDBOwner &bucketDb, bool storeMoveDoneContext = false); + ~MyMoveHandler() override; + void handleMove(MoveOperation &op, vespalib::IDestructorCallback::SP moveDoneCtx) override; + + void reset() { + _moves.clear(); + _numCachedBuckets = 0; + } + + void clearMoveDoneContexts() { + _moveDoneContexts.clear(); + } +}; + +struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { + using DocumentTypeRepo = document::DocumentTypeRepo; + using DocumentMetaData = search::DocumentMetaData; + using Document = document::Document; + using DocumentId = document::DocumentId; + using DocumentIdT = search::DocumentIdT; + using DocumentVector = std::vector<Document::SP>; + std::shared_ptr<const DocumentTypeRepo> _repo; + DocumentVector _docs; + + MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo) : _repo(std::move(repo)), _docs() { + _docs.push_back(Document::UP()); // lid 0 invalid + } + + const DocumentTypeRepo &getDocumentTypeRepo() const override { return *_repo; } + + void getBucketMetaData(const storage::spi::Bucket &, DocumentMetaData::Vector &) const override {} + + DocumentMetaData getDocumentMetaData(const DocumentId &) const override { return DocumentMetaData(); } + + Document::UP getFullDocument(DocumentIdT lid) const override { + return Document::UP(_docs[lid]->clone()); + } + + CachedSelect::SP parseSelect(const vespalib::string &) const override { + return {}; + } +}; + +struct MyBucketModifiedHandler : public IBucketModifiedHandler { + using BucketId = document::BucketId; + BucketId::List _modified; + + void notifyBucketModified(const BucketId &bucket) override; + + void reset() { _modified.clear(); } +}; + +struct MySubDb { + using BucketId = document::BucketId; + using Document = document::Document; + using DocumentTypeRepo = document::DocumentTypeRepo; + using DocumentVector = proton::test::DocumentVector; + using UserDocuments = proton::test::UserDocuments; + DocumentMetaStore::SP _metaStoreSP; + DocumentMetaStore &_metaStore; + std::shared_ptr<MyDocumentRetriever> _realRetriever; + std::shared_ptr<IDocumentRetriever> _retriever; + MaintenanceDocumentSubDB _subDb; + UserDocuments _docs; + bucketdb::BucketDBHandler _bucketDBHandler; + + MySubDb(const std::shared_ptr<const DocumentTypeRepo> &repo, std::shared_ptr<BucketDBOwner> bucketDB, + uint32_t subDbId, SubDbType subDbType); + + ~MySubDb(); + + void insertDocs(const UserDocuments &docs_); + + BucketId bucket(uint32_t userId) const { + return _docs.getBucket(userId); + } + + DocumentVector docs(uint32_t userId) { + return _docs.getGidOrderDocs(userId); + } + + void setBucketState(const BucketId &bucketId, bool active) { + _metaStore.setBucketState(bucketId, active); + } +}; + +bool +assertEqual(const document::BucketId &bucket, const proton::test::Document &doc, + uint32_t sourceSubDbId, uint32_t targetSubDbId, const MoveOperation &op); + +}
\ No newline at end of file diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index a952efdecdc..fbdaf38930b 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -1,512 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> -#include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> -#include <vespa/searchcore/proton/test/bucketfactory.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> -#include <vespa/searchcore/proton/server/bucketmovejob.h> -#include <vespa/searchcore/proton/server/documentbucketmover.h> -#include <vespa/searchcore/proton/server/i_move_operation_limiter.h> -#include <vespa/searchcore/proton/server/idocumentmovehandler.h> -#include <vespa/searchcore/proton/server/imaintenancejobrunner.h> -#include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> -#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h> -#include <vespa/searchcore/proton/test/buckethandler.h> -#include <vespa/searchcore/proton/test/clusterstatehandler.h> -#include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> -#include <vespa/searchcore/proton/test/test.h> -#include <vespa/searchlib/index/docbuilder.h> -#include <vespa/document/test/make_bucket_space.h> +#include "bucketmover_common.h" #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> LOG_SETUP("document_bucket_mover_test"); using namespace proton; +using namespace proton::move::test; using document::BucketId; -using document::Document; -using document::DocumentId; -using document::DocumentTypeRepo; -using document::GlobalId; using document::test::makeBucketSpace; using proton::bucketdb::BucketCreateNotifier; -using search::DocumentIdT; -using search::DocumentMetaData; -using vespalib::IDestructorCallback; -using search::index::DocBuilder; -using search::index::Schema; using storage::spi::BucketInfo; -using storage::spi::Timestamp; -using vespalib::make_string; - using BlockedReason = IBlockableMaintenanceJob::BlockedReason; -using BucketIdSet = std::set<BucketId>; -using BucketIdVector = BucketId::List; -using DocumentVector = std::vector<Document::SP>; using MoveOperationVector = std::vector<MoveOperation>; -using ScanItr = bucketdb::ScanIterator; -using ScanPass = ScanItr::Pass; - -struct MyMoveOperationLimiter : public IMoveOperationLimiter { - uint32_t beginOpCount; - MyMoveOperationLimiter() : beginOpCount(0) {} - IDestructorCallback::SP beginOperation() override { - ++beginOpCount; - return IDestructorCallback::SP(); - } -}; - -struct MyMoveHandler : public IDocumentMoveHandler -{ - BucketDBOwner &_bucketDb; - MoveOperationVector _moves; - size_t _numCachedBuckets; - bool _storeMoveDoneContexts; - std::vector<IDestructorCallback::SP> _moveDoneContexts; - MyMoveHandler(BucketDBOwner &bucketDb, bool storeMoveDoneContext = false) - : _bucketDb(bucketDb), - _moves(), - _numCachedBuckets(), - _storeMoveDoneContexts(storeMoveDoneContext), - _moveDoneContexts() - {} - void handleMove(MoveOperation &op, IDestructorCallback::SP moveDoneCtx) override { - _moves.push_back(op); - if (_bucketDb.takeGuard()->isCachedBucket(op.getBucketId())) { - ++_numCachedBuckets; - } - if (_storeMoveDoneContexts) { - _moveDoneContexts.push_back(std::move(moveDoneCtx)); - } - } - void reset() { - _moves.clear(); - _numCachedBuckets = 0; - } - void clearMoveDoneContexts() { - _moveDoneContexts.clear(); - } -}; - -struct MyDocumentRetriever : public DocumentRetrieverBaseForTest -{ - std::shared_ptr<const DocumentTypeRepo> _repo; - DocumentVector _docs; - MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo) : _repo(std::move(repo)), _docs() { - _docs.push_back(Document::SP()); // lid 0 invalid - } - const document::DocumentTypeRepo &getDocumentTypeRepo() const override { return *_repo; } - void getBucketMetaData(const storage::spi::Bucket &, DocumentMetaData::Vector &) const override {} - DocumentMetaData getDocumentMetaData(const DocumentId &) const override { return DocumentMetaData(); } - Document::UP getFullDocument(DocumentIdT lid) const override { - return Document::UP(_docs[lid]->clone()); - } - - CachedSelect::SP parseSelect(const vespalib::string &) const override { - return CachedSelect::SP(); - } -}; - -struct MyBucketModifiedHandler : public IBucketModifiedHandler -{ - BucketIdVector _modified; - void notifyBucketModified(const BucketId &bucket) override { - BucketIdVector::const_iterator itr = std::find(_modified.begin(), _modified.end(), bucket); - ASSERT_TRUE(itr == _modified.end()); - _modified.push_back(bucket); - } - void reset() { _modified.clear(); } -}; - -struct MySubDb -{ - DocumentMetaStore::SP _metaStoreSP; - DocumentMetaStore & _metaStore; - std::shared_ptr<MyDocumentRetriever> _realRetriever; - std::shared_ptr<IDocumentRetriever> _retriever; - MaintenanceDocumentSubDB _subDb; - test::UserDocuments _docs; - bucketdb::BucketDBHandler _bucketDBHandler; - MySubDb(const std::shared_ptr<const DocumentTypeRepo> &repo, std::shared_ptr<BucketDBOwner> bucketDB, - uint32_t subDbId, SubDbType subDbType); - ~MySubDb(); - void insertDocs(const test::UserDocuments &docs_) { - for (test::UserDocuments::Iterator itr = docs_.begin(); itr != docs_.end(); ++itr) { - const test::BucketDocuments &bucketDocs = itr->second; - for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { - const test::Document &testDoc = bucketDocs.getDocs()[i]; - _metaStore.put(testDoc.getGid(), testDoc.getBucket(), - testDoc.getTimestamp(), testDoc.getDocSize(), testDoc.getLid(), 0u); - _realRetriever->_docs.push_back(testDoc.getDoc()); - ASSERT_EQUAL(testDoc.getLid() + 1, - _realRetriever->_docs.size()); - } - } - _docs.merge(docs_); - } - - BucketId bucket(uint32_t userId) const { - return _docs.getBucket(userId); - } - - test::DocumentVector docs(uint32_t userId) { - return _docs.getGidOrderDocs(userId); - } - - void setBucketState(const BucketId &bucketId, bool active) { - _metaStore.setBucketState(bucketId, active); - } - - void removeBucket(uint32_t userId) { - const test::DocumentVector &userDocs = _docs.getDocs(userId); - for (size_t i = 0; i < userDocs.size(); ++i) { - _metaStore.remove(userDocs[i].getLid(), 0u); - if (_metaStore.getFreeListActive()) { - _metaStore.removeComplete(userDocs[i].getLid()); - } - } - BucketId b(bucket(userId)); - EXPECT_EQUAL(0u, _metaStore.getBucketDB().takeGuard()->get(b).getEntryCount()); - _bucketDBHandler.handleDeleteBucket(b); - } - -}; - -MySubDb::MySubDb(const std::shared_ptr<const DocumentTypeRepo> &repo, std::shared_ptr<BucketDBOwner> bucketDB, - uint32_t subDbId, SubDbType subDbType) - : _metaStoreSP(std::make_shared<DocumentMetaStore>(bucketDB, - DocumentMetaStore::getFixedName(), - search::GrowStrategy(), - subDbType)), - _metaStore(*_metaStoreSP), - _realRetriever(std::make_shared<MyDocumentRetriever>(repo)), - _retriever(_realRetriever), - _subDb("my_sub_db", subDbId, _metaStoreSP, _retriever, IFeedView::SP(), nullptr), - _docs(), - _bucketDBHandler(*bucketDB) -{ - _bucketDBHandler.addDocumentMetaStore(_metaStoreSP.get(), 0); -} -MySubDb::~MySubDb() {} - -struct MySubDbTwoBuckets : public MySubDb -{ - MySubDbTwoBuckets(test::UserDocumentsBuilder &builder, - std::shared_ptr<BucketDBOwner> bucketDB, - uint32_t subDbId, - SubDbType subDbType) - : MySubDb(builder.getRepo(), bucketDB, subDbId, subDbType) - { - builder.createDocs(1, 1, 6); - builder.createDocs(2, 6, 9); - insertDocs(builder.getDocs()); - ASSERT_NOT_EQUAL(bucket(1), bucket(2)); - ASSERT_EQUAL(5u, docs(1).size()); - ASSERT_EQUAL(3u, docs(2).size()); - ASSERT_EQUAL(9u, _realRetriever->_docs.size()); - } -}; - -struct MoveFixture -{ - test::UserDocumentsBuilder _builder; - std::shared_ptr<BucketDBOwner> _bucketDB; - MyMoveOperationLimiter _limiter; - DocumentBucketMover _mover; - MySubDbTwoBuckets _source; - BucketDBOwner _bucketDb; - MyMoveHandler _handler; - PendingLidTracker _pendingLidsForCommit; - MoveFixture() - : _builder(), - _bucketDB(std::make_shared<BucketDBOwner>()), - _limiter(), - _mover(_limiter), - _source(_builder, _bucketDB, 0u, SubDbType::READY), - _bucketDb(), - _handler(_bucketDb) - { - } - void setupForBucket(const BucketId &bucket, - uint32_t sourceSubDbId, - uint32_t targetSubDbId) { - _source._subDb = MaintenanceDocumentSubDB(_source._subDb.name(), - sourceSubDbId, - _source._subDb.meta_store(), - _source._subDb.retriever(), - _source._subDb.feed_view(), - &_pendingLidsForCommit); - _mover.setupForBucket(bucket, &_source._subDb, targetSubDbId, _handler, _bucketDb); - } - bool moveDocuments(size_t maxDocsToMove) { - return _mover.moveDocuments(maxDocsToMove); - } -}; - -TEST("require that initial bucket mover is done") -{ - MyMoveOperationLimiter limiter; - DocumentBucketMover mover(limiter); - EXPECT_TRUE(mover.bucketDone()); - mover.moveDocuments(2); - EXPECT_TRUE(mover.bucketDone()); -} - -bool -assertEqual(const BucketId &bucket, const test::Document &doc, - uint32_t sourceSubDbId, uint32_t targetSubDbId, const MoveOperation &op) -{ - if (!EXPECT_EQUAL(bucket, op.getBucketId())) return false; - if (!EXPECT_EQUAL(doc.getTimestamp(), op.getTimestamp())) return false; - if (!EXPECT_EQUAL(doc.getDocId(), op.getDocument()->getId())) return false; - if (!EXPECT_EQUAL(doc.getLid(), op.getSourceDbdId().getLid())) return false; - if (!EXPECT_EQUAL(sourceSubDbId, op.getSourceDbdId().getSubDbId())) return false; - if (!EXPECT_EQUAL(0u, op.getTargetDbdId().getLid())) return false; - if (!EXPECT_EQUAL(targetSubDbId, op.getTargetDbdId().getSubDbId())) return false; - return true; -} - -TEST_F("require that we can move all documents", MoveFixture) -{ - f.setupForBucket(f._source.bucket(1), 6, 9); - EXPECT_TRUE(f.moveDocuments(5)); - EXPECT_TRUE(f._mover.bucketDone()); - EXPECT_EQUAL(5u, f._handler._moves.size()); - EXPECT_EQUAL(5u, f._limiter.beginOpCount); - for (size_t i = 0; i < 5u; ++i) { - assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); - } -} - -TEST_F("require that move is stalled if document is pending commit", MoveFixture) -{ - f.setupForBucket(f._source.bucket(1), 6, 9); - { - IPendingLidTracker::Token token = f._pendingLidsForCommit.produce(1); - EXPECT_FALSE(f.moveDocuments(5)); - EXPECT_FALSE(f._mover.bucketDone()); - } - EXPECT_TRUE(f.moveDocuments(5)); - EXPECT_TRUE(f._mover.bucketDone()); - EXPECT_EQUAL(5u, f._handler._moves.size()); - EXPECT_EQUAL(5u, f._limiter.beginOpCount); - for (size_t i = 0; i < 5u; ++i) { - assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); - } -} - -TEST_F("require that bucket is cached when IDocumentMoveHandler handles move operation", MoveFixture) -{ - f.setupForBucket(f._source.bucket(1), 6, 9); - EXPECT_TRUE(f.moveDocuments(5)); - EXPECT_TRUE(f._mover.bucketDone()); - EXPECT_EQUAL(5u, f._handler._moves.size()); - EXPECT_EQUAL(5u, f._handler._numCachedBuckets); - EXPECT_FALSE(f._bucketDb.takeGuard()->isCachedBucket(f._source.bucket(1))); -} - -TEST_F("require that we can move documents in several steps", MoveFixture) -{ - f.setupForBucket(f._source.bucket(1), 6, 9); - f.moveDocuments(2); - EXPECT_FALSE(f._mover.bucketDone()); - EXPECT_EQUAL(2u, f._handler._moves.size()); - assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); - assertEqual(f._source.bucket(1), f._source.docs(1)[1], 6, 9, f._handler._moves[1]); - EXPECT_TRUE(f.moveDocuments(2)); - EXPECT_FALSE(f._mover.bucketDone()); - EXPECT_EQUAL(4u, f._handler._moves.size()); - assertEqual(f._source.bucket(1), f._source.docs(1)[2], 6, 9, f._handler._moves[2]); - assertEqual(f._source.bucket(1), f._source.docs(1)[3], 6, 9, f._handler._moves[3]); - EXPECT_TRUE(f.moveDocuments(2)); - EXPECT_TRUE(f._mover.bucketDone()); - EXPECT_EQUAL(5u, f._handler._moves.size()); - assertEqual(f._source.bucket(1), f._source.docs(1)[4], 6, 9, f._handler._moves[4]); - EXPECT_TRUE(f.moveDocuments(2)); - EXPECT_TRUE(f._mover.bucketDone()); - EXPECT_EQUAL(5u, f._handler._moves.size()); -} - -struct ScanFixtureBase -{ - test::UserDocumentsBuilder _builder; - std::shared_ptr<BucketDBOwner> _bucketDB; - MySubDb _ready; - MySubDb _notReady; - ScanFixtureBase(); - ~ScanFixtureBase(); - - ScanItr getItr() { - return ScanItr(_bucketDB->takeGuard(), BucketId()); - } - - ScanItr getItr(BucketId bucket, BucketId endBucket = BucketId(), ScanPass pass = ScanPass::FIRST) { - return ScanItr(_bucketDB->takeGuard(), pass, bucket, endBucket); - } -}; - -ScanFixtureBase::ScanFixtureBase() - : _builder(), - _bucketDB(std::make_shared<BucketDBOwner>()), - _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY), - _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY) -{} -ScanFixtureBase::~ScanFixtureBase() {} - -struct ScanFixture : public ScanFixtureBase -{ - ScanFixture() : ScanFixtureBase() - { - _builder.createDocs(6, 1, 2); - _builder.createDocs(8, 2, 3); - _ready.insertDocs(_builder.getDocs()); - _builder.clearDocs(); - _builder.createDocs(2, 1, 2); - _builder.createDocs(4, 2, 3); - _notReady.insertDocs(_builder.getDocs()); - _builder.clearDocs(); - } -}; - -struct OnlyNotReadyScanFixture : public ScanFixtureBase -{ - OnlyNotReadyScanFixture() : ScanFixtureBase() - { - _builder.createDocs(2, 1, 2); - _builder.createDocs(4, 2, 3); - _notReady.insertDocs(_builder.getDocs()); - } -}; - -struct OnlyReadyScanFixture : public ScanFixtureBase -{ - OnlyReadyScanFixture() : ScanFixtureBase() - { - _builder.createDocs(6, 1, 2); - _builder.createDocs(8, 2, 3); - _ready.insertDocs(_builder.getDocs()); - } -}; - -struct BucketVector : public BucketId::List -{ - BucketVector() : BucketId::List() {} - BucketVector &add(const BucketId &bucket) { - push_back(bucket); - return *this; - } -}; - -void -advanceToFirstBucketWithDocs(ScanItr &itr, SubDbType subDbType) -{ - while (itr.valid()) { - if (subDbType == SubDbType::READY) { - if (itr.hasReadyBucketDocs()) - return; - } else { - if (itr.hasNotReadyBucketDocs()) - return; - } - ++itr; - } -} - -void assertEquals(const BucketVector &exp, ScanItr &itr, SubDbType subDbType) -{ - for (size_t i = 0; i < exp.size(); ++i) { - advanceToFirstBucketWithDocs(itr, subDbType); - EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(exp[i], itr.getBucket()); - ++itr; - } - advanceToFirstBucketWithDocs(itr, subDbType); - EXPECT_FALSE(itr.valid()); -} - -TEST_F("require that we can iterate all buckets from start to end", ScanFixture) -{ - { - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._notReady.bucket(2)). - add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); - } - { - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._ready.bucket(6)). - add(f._ready.bucket(8)), itr, SubDbType::READY); - } -} - -TEST_F("require that we can iterate from the middle of not ready buckets", ScanFixture) -{ - BucketId bucket = f._notReady.bucket(2); - { - ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); - assertEquals(BucketVector(). - add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); - } - { - ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); - assertEquals(BucketVector(). - add(f._notReady.bucket(2)), itr, SubDbType::NOTREADY); - } - { - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._ready.bucket(6)). - add(f._ready.bucket(8)), itr, SubDbType::READY); - } -} - -TEST_F("require that we can iterate from the middle of ready buckets", ScanFixture) -{ - BucketId bucket = f._ready.bucket(6); - { - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._notReady.bucket(2)). - add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); - } - { - ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); - assertEquals(BucketVector(). - add(f._ready.bucket(8)), itr, SubDbType::READY); - } - { - ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); - assertEquals(BucketVector(). - add(f._ready.bucket(6)), itr, SubDbType::READY); - } -} - -TEST_F("require that we can iterate only not ready buckets", OnlyNotReadyScanFixture) -{ - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._notReady.bucket(2)). - add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); -} - -TEST_F("require that we can iterate only ready buckets", OnlyReadyScanFixture) -{ - ScanItr itr = f.getItr(); - assertEquals(BucketVector(). - add(f._ready.bucket(6)). - add(f._ready.bucket(8)), itr, SubDbType::READY); -} - -TEST_F("require that we can iterate zero buckets", ScanFixtureBase) -{ - ScanItr itr = f.getItr(); - EXPECT_FALSE(itr.valid()); -} struct MyFrozenBucketHandler : public IFrozenBucketHandler { - BucketIdSet _frozen; + std::set<BucketId> _frozen; std::set<IBucketFreezeListener *> _listeners; MyFrozenBucketHandler() @@ -597,25 +108,21 @@ struct ControllerFixtureBase } ControllerFixtureBase &activateBucket(const BucketId &bucket) { _ready.setBucketState(bucket, true); - _bucketHandler.notifyBucketStateChanged(bucket, - BucketInfo::ActiveState:: - ACTIVE); + _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::ACTIVE); return *this; } ControllerFixtureBase &deactivateBucket(const BucketId &bucket) { _ready.setBucketState(bucket, false); - _bucketHandler.notifyBucketStateChanged(bucket, - BucketInfo::ActiveState:: - NOT_ACTIVE); + _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::NOT_ACTIVE); return *this; } const MoveOperationVector &docsMoved() const { return _moveHandler._moves; } - const BucketIdVector &bucketsModified() const { + const BucketId::List &bucketsModified() const { return _modifiedHandler._modified; } - const BucketIdVector &calcAsked() const { + const BucketId::List &calcAsked() const { return _calc->asked(); } void runLoop() { diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/scaniterator_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/scaniterator_test.cpp new file mode 100644 index 00000000000..15c951e4fee --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/scaniterator_test.cpp @@ -0,0 +1,320 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucketmover_common.h" +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/log/log.h> +LOG_SETUP("document_bucket_mover_test"); + +using namespace proton; +using namespace proton::move::test; +using document::BucketId; + +using ScanItr = bucketdb::ScanIterator; +using ScanPass = ScanItr::Pass; + +struct MySubDbTwoBuckets : public MySubDb +{ + MySubDbTwoBuckets(test::UserDocumentsBuilder &builder, + std::shared_ptr<BucketDBOwner> bucketDB, + uint32_t subDbId, + SubDbType subDbType) + : MySubDb(builder.getRepo(), bucketDB, subDbId, subDbType) + { + builder.createDocs(1, 1, 6); + builder.createDocs(2, 6, 9); + insertDocs(builder.getDocs()); + ASSERT_NOT_EQUAL(bucket(1), bucket(2)); + ASSERT_EQUAL(5u, docs(1).size()); + ASSERT_EQUAL(3u, docs(2).size()); + ASSERT_EQUAL(9u, _realRetriever->_docs.size()); + } +}; + +struct MoveFixture +{ + test::UserDocumentsBuilder _builder; + std::shared_ptr<BucketDBOwner> _bucketDB; + MyMoveOperationLimiter _limiter; + DocumentBucketMover _mover; + MySubDbTwoBuckets _source; + BucketDBOwner _bucketDb; + MyMoveHandler _handler; + PendingLidTracker _pendingLidsForCommit; + MoveFixture() + : _builder(), + _bucketDB(std::make_shared<BucketDBOwner>()), + _limiter(), + _mover(_limiter), + _source(_builder, _bucketDB, 0u, SubDbType::READY), + _bucketDb(), + _handler(_bucketDb) + { + } + void setupForBucket(const BucketId &bucket, + uint32_t sourceSubDbId, + uint32_t targetSubDbId) { + _source._subDb = MaintenanceDocumentSubDB(_source._subDb.name(), + sourceSubDbId, + _source._subDb.meta_store(), + _source._subDb.retriever(), + _source._subDb.feed_view(), + &_pendingLidsForCommit); + _mover.setupForBucket(bucket, &_source._subDb, targetSubDbId, _handler, _bucketDb); + } + bool moveDocuments(size_t maxDocsToMove) { + return _mover.moveDocuments(maxDocsToMove); + } +}; + +TEST("require that initial bucket mover is done") +{ + MyMoveOperationLimiter limiter; + DocumentBucketMover mover(limiter); + EXPECT_TRUE(mover.bucketDone()); + mover.moveDocuments(2); + EXPECT_TRUE(mover.bucketDone()); +} + +TEST_F("require that we can move all documents", MoveFixture) +{ + f.setupForBucket(f._source.bucket(1), 6, 9); + EXPECT_TRUE(f.moveDocuments(5)); + EXPECT_TRUE(f._mover.bucketDone()); + EXPECT_EQUAL(5u, f._handler._moves.size()); + EXPECT_EQUAL(5u, f._limiter.beginOpCount); + for (size_t i = 0; i < 5u; ++i) { + assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); + } +} + +TEST_F("require that move is stalled if document is pending commit", MoveFixture) +{ + f.setupForBucket(f._source.bucket(1), 6, 9); + { + IPendingLidTracker::Token token = f._pendingLidsForCommit.produce(1); + EXPECT_FALSE(f.moveDocuments(5)); + EXPECT_FALSE(f._mover.bucketDone()); + } + EXPECT_TRUE(f.moveDocuments(5)); + EXPECT_TRUE(f._mover.bucketDone()); + EXPECT_EQUAL(5u, f._handler._moves.size()); + EXPECT_EQUAL(5u, f._limiter.beginOpCount); + for (size_t i = 0; i < 5u; ++i) { + assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); + } +} + +TEST_F("require that bucket is cached when IDocumentMoveHandler handles move operation", MoveFixture) +{ + f.setupForBucket(f._source.bucket(1), 6, 9); + EXPECT_TRUE(f.moveDocuments(5)); + EXPECT_TRUE(f._mover.bucketDone()); + EXPECT_EQUAL(5u, f._handler._moves.size()); + EXPECT_EQUAL(5u, f._handler._numCachedBuckets); + EXPECT_FALSE(f._bucketDb.takeGuard()->isCachedBucket(f._source.bucket(1))); +} + +TEST_F("require that we can move documents in several steps", MoveFixture) +{ + f.setupForBucket(f._source.bucket(1), 6, 9); + f.moveDocuments(2); + EXPECT_FALSE(f._mover.bucketDone()); + EXPECT_EQUAL(2u, f._handler._moves.size()); + assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]); + assertEqual(f._source.bucket(1), f._source.docs(1)[1], 6, 9, f._handler._moves[1]); + EXPECT_TRUE(f.moveDocuments(2)); + EXPECT_FALSE(f._mover.bucketDone()); + EXPECT_EQUAL(4u, f._handler._moves.size()); + assertEqual(f._source.bucket(1), f._source.docs(1)[2], 6, 9, f._handler._moves[2]); + assertEqual(f._source.bucket(1), f._source.docs(1)[3], 6, 9, f._handler._moves[3]); + EXPECT_TRUE(f.moveDocuments(2)); + EXPECT_TRUE(f._mover.bucketDone()); + EXPECT_EQUAL(5u, f._handler._moves.size()); + assertEqual(f._source.bucket(1), f._source.docs(1)[4], 6, 9, f._handler._moves[4]); + EXPECT_TRUE(f.moveDocuments(2)); + EXPECT_TRUE(f._mover.bucketDone()); + EXPECT_EQUAL(5u, f._handler._moves.size()); +} + +struct ScanFixtureBase +{ + test::UserDocumentsBuilder _builder; + std::shared_ptr<BucketDBOwner> _bucketDB; + MySubDb _ready; + MySubDb _notReady; + ScanFixtureBase(); + ~ScanFixtureBase(); + + ScanItr getItr() { + return ScanItr(_bucketDB->takeGuard(), BucketId()); + } + + ScanItr getItr(BucketId bucket, BucketId endBucket = BucketId(), ScanPass pass = ScanPass::FIRST) { + return ScanItr(_bucketDB->takeGuard(), pass, bucket, endBucket); + } +}; + +ScanFixtureBase::ScanFixtureBase() + : _builder(), + _bucketDB(std::make_shared<BucketDBOwner>()), + _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY), + _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY) +{} +ScanFixtureBase::~ScanFixtureBase() = default; + +struct ScanFixture : public ScanFixtureBase +{ + ScanFixture() : ScanFixtureBase() + { + _builder.createDocs(6, 1, 2); + _builder.createDocs(8, 2, 3); + _ready.insertDocs(_builder.getDocs()); + _builder.clearDocs(); + _builder.createDocs(2, 1, 2); + _builder.createDocs(4, 2, 3); + _notReady.insertDocs(_builder.getDocs()); + _builder.clearDocs(); + } +}; + +struct OnlyNotReadyScanFixture : public ScanFixtureBase +{ + OnlyNotReadyScanFixture() : ScanFixtureBase() + { + _builder.createDocs(2, 1, 2); + _builder.createDocs(4, 2, 3); + _notReady.insertDocs(_builder.getDocs()); + } +}; + +struct OnlyReadyScanFixture : public ScanFixtureBase +{ + OnlyReadyScanFixture() : ScanFixtureBase() + { + _builder.createDocs(6, 1, 2); + _builder.createDocs(8, 2, 3); + _ready.insertDocs(_builder.getDocs()); + } +}; + +struct BucketVector : public BucketId::List +{ + BucketVector() : BucketId::List() {} + BucketVector &add(const BucketId &bucket) { + push_back(bucket); + return *this; + } +}; + +void +advanceToFirstBucketWithDocs(ScanItr &itr, SubDbType subDbType) +{ + while (itr.valid()) { + if (subDbType == SubDbType::READY) { + if (itr.hasReadyBucketDocs()) + return; + } else { + if (itr.hasNotReadyBucketDocs()) + return; + } + ++itr; + } +} + +void assertEquals(const BucketVector &exp, ScanItr &itr, SubDbType subDbType) +{ + for (size_t i = 0; i < exp.size(); ++i) { + advanceToFirstBucketWithDocs(itr, subDbType); + EXPECT_TRUE(itr.valid()); + EXPECT_EQUAL(exp[i], itr.getBucket()); + ++itr; + } + advanceToFirstBucketWithDocs(itr, subDbType); + EXPECT_FALSE(itr.valid()); +} + +TEST_F("require that we can iterate all buckets from start to end", ScanFixture) +{ + { + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._notReady.bucket(2)). + add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); + } + { + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._ready.bucket(6)). + add(f._ready.bucket(8)), itr, SubDbType::READY); + } +} + +TEST_F("require that we can iterate from the middle of not ready buckets", ScanFixture) +{ + BucketId bucket = f._notReady.bucket(2); + { + ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); + assertEquals(BucketVector(). + add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); + } + { + ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); + assertEquals(BucketVector(). + add(f._notReady.bucket(2)), itr, SubDbType::NOTREADY); + } + { + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._ready.bucket(6)). + add(f._ready.bucket(8)), itr, SubDbType::READY); + } +} + +TEST_F("require that we can iterate from the middle of ready buckets", ScanFixture) +{ + BucketId bucket = f._ready.bucket(6); + { + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._notReady.bucket(2)). + add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); + } + { + ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); + assertEquals(BucketVector(). + add(f._ready.bucket(8)), itr, SubDbType::READY); + } + { + ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); + assertEquals(BucketVector(). + add(f._ready.bucket(6)), itr, SubDbType::READY); + } +} + +TEST_F("require that we can iterate only not ready buckets", OnlyNotReadyScanFixture) +{ + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._notReady.bucket(2)). + add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); +} + +TEST_F("require that we can iterate only ready buckets", OnlyReadyScanFixture) +{ + ScanItr itr = f.getItr(); + assertEquals(BucketVector(). + add(f._ready.bucket(6)). + add(f._ready.bucket(8)), itr, SubDbType::READY); +} + +TEST_F("require that we can iterate zero buckets", ScanFixtureBase) +{ + ScanItr itr = f.getItr(); + EXPECT_FALSE(itr.valid()); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} |