diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-01-22 18:49:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-22 18:49:13 +0100 |
commit | 696ee4a7d60370b93613bfe2d3e786e3be5a1ac5 (patch) | |
tree | 54ec1e2251cf23d1c9bc84942e73901c56f61d84 | |
parent | 724c57cb13aadf085fc98971009086f7d08792e6 (diff) | |
parent | 7008026d0058282fc67000e8f66e0eedcc337372 (diff) |
Merge pull request #16164 from vespa-engine/balder/split-test
- Splitt common test code.
8 files changed, 828 insertions, 557 deletions
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp index 8730e25e52b..9aa647fcc65 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp @@ -24,14 +24,14 @@ DummyBucketExecutor::~DummyBucketExecutor() { std::unique_ptr<BucketTask> DummyBucketExecutor::execute(const Bucket & bucket, std::unique_ptr<BucketTask> task) { - { - std::unique_lock guard(_lock); - while (_inFlight.contains(bucket.getBucket())) { - _cond.wait(guard); - } - _inFlight.insert(bucket.getBucket()); - } _executor->execute(makeLambdaTask([this, bucket, bucketTask=std::move(task)]() { + { + std::unique_lock guard(_lock); + while (_inFlight.contains(bucket.getBucket())) { + _cond.wait(guard); + } + _inFlight.insert(bucket.getBucket()); + } bucketTask->run(bucket, makeLambdaCallback([this, bucket]() { std::unique_lock guard(_lock); assert(_inFlight.contains(bucket.getBucket())); diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt index b411bc739d6..21afe6220be 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt @@ -1,8 +1,16 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(searchcore_lidspace_test STATIC + SOURCES + lid_space_common.cpp +) + vespa_add_executable(searchcore_lid_space_compaction_test_app TEST SOURCES + lid_space_jobtest.cpp lid_space_compaction_test.cpp DEPENDS + searchcore_lidspace_test searchcore_test searchcore_server searchcore_initializer @@ -13,3 +21,19 @@ vespa_add_executable(searchcore_lid_space_compaction_test_app TEST GTest::GTest ) vespa_add_test(NAME searchcore_lid_space_compaction_test_app COMMAND searchcore_lid_space_compaction_test_app) + +vespa_add_executable(searchcore_lid_space_handler_test_app TEST + SOURCES + lid_space_handler_test.cpp + DEPENDS + searchcore_lidspace_test + searchcore_test + searchcore_server + searchcore_initializer + searchcore_feedoperation + searchcore_documentmetastore + searchcore_bucketdb + searchcore_pcommon + GTest::GTest +) +vespa_add_test(NAME searchcore_lid_space_handler_test_app COMMAND searchcore_lid_space_handler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp new file mode 100644 index 00000000000..e8d5b22acfc --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp @@ -0,0 +1,209 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lid_space_common.h" + +MyScanIterator::MyScanIterator(const LidVector &lids, bool bucketIdEqualLid) + : _lids(lids), + _itr(_lids.begin()), + _validItr(true), + _bucketIdEqualLid(bucketIdEqualLid) +{} +MyScanIterator::~MyScanIterator() = default; + +bool +MyScanIterator::valid() const { + return _validItr; +} + +search::DocumentMetaData MyScanIterator::next(uint32_t compactLidLimit, bool retry) { + if (!retry && _itr != _lids.begin()) { + ++_itr; + } + for (; _itr != _lids.end() && (*_itr) <= compactLidLimit; ++_itr) {} + if (_itr != _lids.end()) { + uint32_t lid = *_itr; + if (lid > compactLidLimit) { + return search::DocumentMetaData(lid, TIMESTAMP_1, createBucketId(lid), GID_1); + } + } else { + _validItr = false; + } + return search::DocumentMetaData(); +} + +search::DocumentMetaData MyScanIterator::getMetaData(uint32_t lid) const { + return search::DocumentMetaData(lid, TIMESTAMP_1, createBucketId(lid), GID_1); +} + +document::BucketId +MyScanIterator::createBucketId(uint32_t lid) const { + return _bucketIdEqualLid ? document::BucketId(lid) : BUCKET_ID_1; +} + +void +MyHandler::clearMoveDoneContexts() { + _moveDoneContexts.clear(); +} + +void +MyHandler::run_remove_ops(bool remove_batch) { + // This ensures to max out the threshold time in the operation rate tracker. + if (remove_batch) { + _op_listener->notify_remove_batch(); + _op_listener->notify_remove_batch(); + _op_listener->notify_remove_batch(); + } else { + _op_listener->notify_remove(); + _op_listener->notify_remove(); + _op_listener->notify_remove(); + } +} + +void +MyHandler::stop_remove_ops(bool remove_batch) const { + if (remove_batch) { + _rm_listener->get_remove_batch_tracker().reset(vespalib::steady_clock::now()); + } else { + _rm_listener->get_remove_tracker().reset(vespalib::steady_clock::now()); + } +} + +vespalib::string +MyHandler::getName() const { + return "myhandler"; +} + +void +MyHandler::set_operation_listener(documentmetastore::OperationListener::SP op_listener) { + auto* rm_listener = dynamic_cast<RemoveOperationsRateTracker*>(op_listener.get()); + assert(rm_listener != nullptr); + _op_listener = std::move(op_listener); + _rm_listener = rm_listener; +} + +LidUsageStats +MyHandler::getLidStatus() const { + assert(_handleMoveCnt < _stats.size()); + return _stats[_handleMoveCnt]; +} + +IDocumentScanIterator::UP +MyHandler::getIterator() const { + assert(_iteratorCnt < _lids.size()); + return std::make_unique<MyScanIterator>(_lids[_iteratorCnt++], _bucketIdEqualLid); +} + +MoveOperation::UP +MyHandler::createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const { + assert(document.lid > moveToLid); + _moveFromLid = document.lid; + _moveToLid = moveToLid; + return std::make_unique<MoveOperation>(); +} + +void +MyHandler::handleMove(const MoveOperation &, IDestructorCallback::SP moveDoneCtx) { + ++_handleMoveCnt; + if (_storeMoveDoneContexts) { + _moveDoneContexts.push_back(std::move(moveDoneCtx)); + } +} + +void +MyHandler::handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback>) { + _wantedLidLimit = op.getLidLimit(); +} + +MyHandler::MyHandler(bool storeMoveDoneContexts, bool bucketIdEqualLid) + : _stats(), + _moveFromLid(0), + _moveToLid(0), + _handleMoveCnt(0), + _wantedLidLimit(0), + _iteratorCnt(0), + _storeMoveDoneContexts(storeMoveDoneContexts), + _bucketIdEqualLid(bucketIdEqualLid), + _moveDoneContexts(), + _op_listener(), + _rm_listener() +{} + +MyHandler::~MyHandler() = default; + + +void +MyStorer::appendOperation(const FeedOperation &op, DoneCallback) { + if (op.getType() == FeedOperation::MOVE) { + ++ _moveCnt; + } else if (op.getType() == FeedOperation::COMPACT_LID_SPACE) { + ++_compactCnt; + } +} + +IOperationStorer::CommitResult +MyStorer::startCommit(DoneCallback) { + return CommitResult(); +} + +IFrozenBucketHandler::ExclusiveBucketGuard::UP +MyFrozenBucketHandler::acquireExclusiveBucket(BucketId bucket) { + return (_bucket == bucket) + ? ExclusiveBucketGuard::UP() + : std::make_unique<ExclusiveBucketGuard>(bucket); +} + +MyDocumentStore::MyDocumentStore() + : _readDoc(), + _readLid(0) +{} + +MyDocumentStore::~MyDocumentStore() = default; + +document::Document::UP +MyDocumentStore::read(search::DocumentIdT lid, const document::DocumentTypeRepo &) const { + _readLid = lid; + return Document::UP(_readDoc->clone()); +} + +MyDocumentRetriever::MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo_in, const MyDocumentStore& store_in) noexcept + : repo(std::move(repo_in)), + store(store_in) +{} + +MyDocumentRetriever::~MyDocumentRetriever() = default; + +const document::DocumentTypeRepo& +MyDocumentRetriever::getDocumentTypeRepo() const { + return *repo; +} + +void +MyDocumentRetriever::getBucketMetaData(const storage::spi::Bucket&, DocumentMetaData::Vector&) const { + abort(); +} + +DocumentMetaData +MyDocumentRetriever::getDocumentMetaData(const DocumentId&) const { + abort(); +} + +Document::UP +MyDocumentRetriever::getFullDocument(DocumentIdT lid) const { + return store.read(lid, *repo); +} + +CachedSelect::SP +MyDocumentRetriever::parseSelect(const vespalib::string&) const { + abort(); +} + +MySubDb::MySubDb(std::shared_ptr<BucketDBOwner> bucket_db, const MyDocumentStore& store, const std::shared_ptr<const DocumentTypeRepo> & repo) + : sub_db(std::move(bucket_db), SUBDB_ID), + maintenance_sub_db(sub_db.getName(), sub_db.getSubDbId(), sub_db.getDocumentMetaStoreContext().getSP(), + std::make_shared<MyDocumentRetriever>(repo, store), + std::make_shared<MyFeedView>(repo), + &_pendingLidsForCommit) +{ +} + +MySubDb::~MySubDb() = default; diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.h new file mode 100644 index 00000000000..ae83441737c --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.h @@ -0,0 +1,144 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcore/proton/server/i_document_scan_iterator.h> +#include <vespa/searchcore/proton/server/ifrozenbuckethandler.h> +#include <vespa/searchcore/proton/server/imaintenancejobrunner.h> +#include <vespa/searchcore/proton/server/lid_space_compaction_handler.h> +#include <vespa/searchcore/proton/server/remove_operations_rate_tracker.h> +#include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> +#include <vespa/searchcore/proton/server/i_operation_storer.h> +#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> +#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h> +#include <vespa/searchcore/proton/test/clusterstatehandler.h> +#include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> +#include <vespa/searchcore/proton/test/test.h> +#include <vespa/vespalib/util/idestructorcallback.h> +#include <vespa/searchlib/index/docbuilder.h> + +using namespace document; +using namespace proton; +using namespace search::index; +using namespace search; +using namespace vespalib; +using vespalib::IDestructorCallback; +using storage::spi::Timestamp; +using TimePoint = LidUsageStats::TimePoint; + +constexpr uint32_t SUBDB_ID = 2; +constexpr vespalib::duration JOB_DELAY = 1s; +constexpr uint32_t ALLOWED_LID_BLOAT = 1; +constexpr double ALLOWED_LID_BLOAT_FACTOR = 0.3; +constexpr double REMOVE_BATCH_BLOCK_RATE = 1.0 / 21.0; +constexpr double REMOVE_BLOCK_RATE = 1.0 / 20.0; +constexpr double RESOURCE_LIMIT_FACTOR = 1.0; +constexpr uint32_t MAX_OUTSTANDING_MOVE_OPS = 10; +const vespalib::string DOC_ID = "id:test:searchdocument::0"; +const BucketId BUCKET_ID_1(1); +const BucketId BUCKET_ID_2(2); +const Timestamp TIMESTAMP_1(1); +const GlobalId GID_1; + +using LidVector = std::vector<uint32_t>; +using LidPair = std::pair<uint32_t, uint32_t>; +using LidPairVector = std::vector<LidPair>; + +struct MyScanIterator : public IDocumentScanIterator { + LidVector _lids; + LidVector::const_iterator _itr; + bool _validItr; + bool _bucketIdEqualLid; + explicit MyScanIterator(const LidVector &lids, bool bucketIdEqualLid); + ~MyScanIterator() override; + bool valid() const override; + search::DocumentMetaData next(uint32_t compactLidLimit, bool retry) override; + search::DocumentMetaData getMetaData(uint32_t lid) const override; + + document::BucketId createBucketId(uint32_t lid) const; +}; + +struct MyHandler : public ILidSpaceCompactionHandler { + std::vector<LidUsageStats> _stats; + std::vector<LidVector> _lids; + mutable uint32_t _moveFromLid; + mutable uint32_t _moveToLid; + uint32_t _handleMoveCnt; + uint32_t _wantedLidLimit; + mutable uint32_t _iteratorCnt; + bool _storeMoveDoneContexts; + bool _bucketIdEqualLid; + std::vector<IDestructorCallback::SP> _moveDoneContexts; + documentmetastore::OperationListener::SP _op_listener; + RemoveOperationsRateTracker* _rm_listener; + + explicit MyHandler(bool storeMoveDoneContexts, bool _bucketIdEqualLid); + ~MyHandler() override; + void clearMoveDoneContexts(); + void run_remove_ops(bool remove_batch); + void stop_remove_ops(bool remove_batch) const; + vespalib::string getName() const override; + void set_operation_listener(documentmetastore::OperationListener::SP op_listener) override; + uint32_t getSubDbId() const override { return 2; } + LidUsageStats getLidStatus() const override; + IDocumentScanIterator::UP getIterator() const override; + MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document, + uint32_t moveToLid) const override; + void handleMove(const MoveOperation &, IDestructorCallback::SP moveDoneCtx) override; + void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback>) override; +}; + +struct MyStorer : public IOperationStorer { + uint32_t _moveCnt; + uint32_t _compactCnt; + MyStorer() + : _moveCnt(0), + _compactCnt(0) + {} + void appendOperation(const FeedOperation &op, DoneCallback) override; + CommitResult startCommit(DoneCallback) override; +}; + +struct MyFrozenBucketHandler : public IFrozenBucketHandler { + BucketId _bucket; + MyFrozenBucketHandler() : _bucket() {} + ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override; + void addListener(IBucketFreezeListener *) override { } + void removeListener(IBucketFreezeListener *) override { } +}; + +struct MyFeedView : public test::DummyFeedView { + explicit MyFeedView(std::shared_ptr<const DocumentTypeRepo> repo) + : test::DummyFeedView(std::move(repo)) + { + } +}; + +struct MyDocumentStore : public test::DummyDocumentStore { + Document::SP _readDoc; + mutable uint32_t _readLid; + MyDocumentStore(); + ~MyDocumentStore() override; + document::Document::UP read(search::DocumentIdT lid, const document::DocumentTypeRepo &) const override; +}; + +struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { + std::shared_ptr<const DocumentTypeRepo> repo; + const MyDocumentStore& store; + MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo_in, const MyDocumentStore& store_in) noexcept; + ~MyDocumentRetriever(); + const document::DocumentTypeRepo& getDocumentTypeRepo() const override; + void getBucketMetaData(const storage::spi::Bucket&, DocumentMetaData::Vector&) const override; + DocumentMetaData getDocumentMetaData(const DocumentId&) const override; + Document::UP getFullDocument(DocumentIdT lid) const override; + CachedSelect::SP parseSelect(const vespalib::string&) const override; +}; + +struct MySubDb { + test::DummyDocumentSubDb sub_db; + MaintenanceDocumentSubDB maintenance_sub_db; + PendingLidTracker _pendingLidsForCommit; + MySubDb(std::shared_ptr<BucketDBOwner> bucket_db, const MyDocumentStore& store, const std::shared_ptr<const DocumentTypeRepo> & repo); + ~MySubDb(); +}; diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index b4c878b26eb..688dd963f61 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -1,433 +1,15 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchcore/proton/server/i_document_scan_iterator.h> -#include <vespa/searchcore/proton/server/ifrozenbuckethandler.h> -#include <vespa/searchcore/proton/server/imaintenancejobrunner.h> -#include <vespa/searchcore/proton/server/lid_space_compaction_handler.h> -#include <vespa/searchcore/proton/server/lid_space_compaction_job.h> -#include <vespa/searchcore/proton/server/remove_operations_rate_tracker.h> -#include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> -#include <vespa/searchcore/proton/server/i_operation_storer.h> -#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> -#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h> -#include <vespa/searchcore/proton/test/clusterstatehandler.h> -#include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> -#include <vespa/searchcore/proton/test/test.h> -#include <vespa/vespalib/util/idestructorcallback.h> -#include <vespa/searchlib/index/docbuilder.h> -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("lid_space_compaction_test"); - -using namespace document; -using namespace proton; -using namespace search::index; -using namespace search; -using namespace vespalib; -using vespalib::IDestructorCallback; -using storage::spi::Timestamp; -using BlockedReason = IBlockableMaintenanceJob::BlockedReason; -using TimePoint = LidUsageStats::TimePoint; - -constexpr uint32_t SUBDB_ID = 2; -constexpr vespalib::duration JOB_DELAY = 1s; -constexpr uint32_t ALLOWED_LID_BLOAT = 1; -constexpr double ALLOWED_LID_BLOAT_FACTOR = 0.3; -constexpr double REMOVE_BATCH_BLOCK_RATE = 1.0 / 21.0; -constexpr double REMOVE_BLOCK_RATE = 1.0 / 20.0; -constexpr double RESOURCE_LIMIT_FACTOR = 1.0; -constexpr uint32_t MAX_OUTSTANDING_MOVE_OPS = 10; -const vespalib::string DOC_ID = "id:test:searchdocument::0"; -const BucketId BUCKET_ID_1(1); -const BucketId BUCKET_ID_2(2); -const Timestamp TIMESTAMP_1(1); -const GlobalId GID_1; - -using LidVector = std::vector<uint32_t>; -using LidPair = std::pair<uint32_t, uint32_t>; -using LidPairVector = std::vector<LidPair>; - -struct MyScanIterator : public IDocumentScanIterator { - LidVector _lids; - LidVector::const_iterator _itr; - bool _validItr; - explicit MyScanIterator(const LidVector &lids) : _lids(lids), _itr(_lids.begin()), _validItr(true) {} - bool valid() const override { - return _validItr; - } - search::DocumentMetaData next(uint32_t compactLidLimit, bool retry) override { - if (!retry && _itr != _lids.begin()) { - ++_itr; - } - for (; _itr != _lids.end() && (*_itr) <= compactLidLimit; ++_itr) {} - if (_itr != _lids.end()) { - uint32_t lid = *_itr; - if (lid > compactLidLimit) { - return search::DocumentMetaData(lid, TIMESTAMP_1, BUCKET_ID_1, GID_1); - } - } else { - _validItr = false; - } - return search::DocumentMetaData(); - } - search::DocumentMetaData getMetaData(uint32_t lid) const override { - return search::DocumentMetaData(lid, TIMESTAMP_1, BUCKET_ID_1, GID_1); - } -}; - -struct MyHandler : public ILidSpaceCompactionHandler { - std::vector<LidUsageStats> _stats; - std::vector<LidVector> _lids; - mutable uint32_t _moveFromLid; - mutable uint32_t _moveToLid; - uint32_t _handleMoveCnt; - uint32_t _wantedLidLimit; - mutable uint32_t _iteratorCnt; - bool _storeMoveDoneContexts; - std::vector<IDestructorCallback::SP> _moveDoneContexts; - documentmetastore::OperationListener::SP _op_listener; - RemoveOperationsRateTracker* _rm_listener; - - explicit MyHandler(bool storeMoveDoneContexts = false); - ~MyHandler() override; - void clearMoveDoneContexts() { _moveDoneContexts.clear(); } - void run_remove_ops(bool remove_batch) { - // This ensures to max out the threshold time in the operation rate tracker. - if (remove_batch) { - _op_listener->notify_remove_batch(); - _op_listener->notify_remove_batch(); - _op_listener->notify_remove_batch(); - } else { - _op_listener->notify_remove(); - _op_listener->notify_remove(); - _op_listener->notify_remove(); - } - } - void stop_remove_ops(bool remove_batch) const { - if (remove_batch) { - _rm_listener->get_remove_batch_tracker().reset(vespalib::steady_clock::now()); - } else { - _rm_listener->get_remove_tracker().reset(vespalib::steady_clock::now()); - } - } - vespalib::string getName() const override { - return "myhandler"; - } - void set_operation_listener(documentmetastore::OperationListener::SP op_listener) override { - auto* rm_listener = dynamic_cast<RemoveOperationsRateTracker*>(op_listener.get()); - assert(rm_listener != nullptr); - _op_listener = std::move(op_listener); - _rm_listener = rm_listener; - } - uint32_t getSubDbId() const override { return 2; } - LidUsageStats getLidStatus() const override { - assert(_handleMoveCnt < _stats.size()); - return _stats[_handleMoveCnt]; - } - IDocumentScanIterator::UP getIterator() const override { - assert(_iteratorCnt < _lids.size()); - return std::make_unique<MyScanIterator>(_lids[_iteratorCnt++]); - } - MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document, - uint32_t moveToLid) const override { - assert(document.lid > moveToLid); - _moveFromLid = document.lid; - _moveToLid = moveToLid; - return std::make_unique<MoveOperation>(); - } - void handleMove(const MoveOperation &, IDestructorCallback::SP moveDoneCtx) override { - ++_handleMoveCnt; - if (_storeMoveDoneContexts) { - _moveDoneContexts.push_back(std::move(moveDoneCtx)); - } - } - void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback>) override { - _wantedLidLimit = op.getLidLimit(); - } -}; - -MyHandler::MyHandler(bool storeMoveDoneContexts) - : _stats(), - _moveFromLid(0), - _moveToLid(0), - _handleMoveCnt(0), - _wantedLidLimit(0), - _iteratorCnt(0), - _storeMoveDoneContexts(storeMoveDoneContexts), - _moveDoneContexts(), - _op_listener(), - _rm_listener() -{} - -MyHandler::~MyHandler() = default; - -struct MyStorer : public IOperationStorer { - uint32_t _moveCnt; - uint32_t _compactCnt; - MyStorer() - : _moveCnt(0), - _compactCnt(0) - {} - void appendOperation(const FeedOperation &op, DoneCallback) override { - if (op.getType() == FeedOperation::MOVE) { - ++ _moveCnt; - } else if (op.getType() == FeedOperation::COMPACT_LID_SPACE) { - ++_compactCnt; - } - } - CommitResult startCommit(DoneCallback) override { - return CommitResult(); - } -}; - -struct MyFrozenBucketHandler : public IFrozenBucketHandler { - BucketId _bucket; - MyFrozenBucketHandler() : _bucket() {} - ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override { - return (_bucket == bucket) - ? ExclusiveBucketGuard::UP() - : std::make_unique<ExclusiveBucketGuard>(bucket); - } - void addListener(IBucketFreezeListener *) override { } - void removeListener(IBucketFreezeListener *) override { } -}; - -struct MyFeedView : public test::DummyFeedView { - explicit MyFeedView(std::shared_ptr<const DocumentTypeRepo> repo) - : test::DummyFeedView(std::move(repo)) - { - } -}; - -struct MyDocumentStore : public test::DummyDocumentStore { - Document::SP _readDoc; - mutable uint32_t _readLid; - MyDocumentStore() : _readDoc(), _readLid(0) {} - ~MyDocumentStore() override; - document::Document::UP read(search::DocumentIdT lid, const document::DocumentTypeRepo &) const override { - _readLid = lid; - return Document::UP(_readDoc->clone()); - } -}; - -MyDocumentStore::~MyDocumentStore() = default; - -struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { - std::shared_ptr<const DocumentTypeRepo> repo; - const MyDocumentStore& store; - MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo_in, const MyDocumentStore& store_in) noexcept - : repo(std::move(repo_in)), - store(store_in) - {} - const document::DocumentTypeRepo& getDocumentTypeRepo() const override { return *repo; } - void getBucketMetaData(const storage::spi::Bucket&, DocumentMetaData::Vector&) const override { abort(); } - DocumentMetaData getDocumentMetaData(const DocumentId&) const override { abort(); } - Document::UP getFullDocument(DocumentIdT lid) const override { - return store.read(lid, *repo); - } - CachedSelect::SP parseSelect(const vespalib::string&) const override { abort(); } -}; - -struct MySubDb { - test::DummyDocumentSubDb sub_db; - MaintenanceDocumentSubDB maintenance_sub_db; - PendingLidTracker _pendingLidsForCommit; - MySubDb(std::shared_ptr<BucketDBOwner> bucket_db, const MyDocumentStore& store, const std::shared_ptr<const DocumentTypeRepo> & repo); - ~MySubDb(); -}; - -MySubDb::MySubDb(std::shared_ptr<BucketDBOwner> bucket_db, const MyDocumentStore& store, const std::shared_ptr<const DocumentTypeRepo> & repo) - : sub_db(std::move(bucket_db), SUBDB_ID), - maintenance_sub_db(sub_db.getName(), sub_db.getSubDbId(), sub_db.getDocumentMetaStoreContext().getSP(), - std::make_shared<MyDocumentRetriever>(repo, store), - std::make_shared<MyFeedView>(repo), - &_pendingLidsForCommit) -{ -} - -MySubDb::~MySubDb() = default; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -struct MyDirectJobRunner : public IMaintenanceJobRunner { - IMaintenanceJob &_job; - explicit MyDirectJobRunner(IMaintenanceJob &job) - : _job(job) - { - _job.registerRunner(this); - } - void run() override { _job.run(); } -}; +#include "lid_space_jobtest.h" -struct MyCountJobRunner : public IMaintenanceJobRunner { - uint32_t runCnt; - explicit MyCountJobRunner(IMaintenanceJob &job) : runCnt(0) { - job.registerRunner(this); - } - void run() override { ++runCnt; } -}; - -struct JobTestBase : public ::testing::Test { - std::unique_ptr<MyHandler> _handler; - MyStorer _storer; - MyFrozenBucketHandler _frozenHandler; - test::DiskMemUsageNotifier _diskMemUsageNotifier; - test::ClusterStateHandler _clusterStateHandler; - std::unique_ptr<LidSpaceCompactionJob> _job; - JobTestBase() - : _handler(), - _storer(), - _frozenHandler(), - _diskMemUsageNotifier(), - _clusterStateHandler(), - _job() - { - init(); - } - void init(uint32_t allowedLidBloat = ALLOWED_LID_BLOAT, - double allowedLidBloatFactor = ALLOWED_LID_BLOAT_FACTOR, - double resourceLimitFactor = RESOURCE_LIMIT_FACTOR, - vespalib::duration interval = JOB_DELAY, - bool nodeRetired = false, - uint32_t maxOutstandingMoveOps = MAX_OUTSTANDING_MOVE_OPS) - { - _handler = std::make_unique<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS); - _job = std::make_unique<LidSpaceCompactionJob>(DocumentDBLidSpaceCompactionConfig(interval, allowedLidBloat, - allowedLidBloatFactor, - REMOVE_BATCH_BLOCK_RATE, - REMOVE_BLOCK_RATE, - false), - *_handler, _storer, _frozenHandler, _diskMemUsageNotifier, - BlockableMaintenanceJobConfig(resourceLimitFactor, maxOutstandingMoveOps), - _clusterStateHandler, nodeRetired); - } - ~JobTestBase() override; - JobTestBase &addStats(uint32_t docIdLimit, - const LidVector &usedLids, - const LidPairVector &usedFreePairs) { - return addMultiStats(docIdLimit, {usedLids}, usedFreePairs); - } - JobTestBase &addMultiStats(uint32_t docIdLimit, - const std::vector<LidVector> &usedLidsVector, - const LidPairVector &usedFreePairs) { - uint32_t usedLids = usedLidsVector[0].size(); - for (auto pair : usedFreePairs) { - uint32_t highestUsedLid = pair.first; - uint32_t lowestFreeLid = pair.second; - _handler->_stats.emplace_back(docIdLimit, usedLids, lowestFreeLid, highestUsedLid); - } - _handler->_lids = usedLidsVector; - return *this; - } - JobTestBase &addStats(uint32_t docIdLimit, - uint32_t numDocs, - uint32_t lowestFreeLid, - uint32_t highestUsedLid) { - _handler->_stats.emplace_back(docIdLimit, numDocs, lowestFreeLid, highestUsedLid); - return *this; - } - bool run() const { - return _job->run(); - } - JobTestBase &endScan() { - EXPECT_FALSE(run()); - return *this; - } - JobTestBase &compact() { - EXPECT_TRUE(run()); - return *this; - } - void notifyNodeRetired(bool nodeRetired) { - test::BucketStateCalculator::SP calc = std::make_shared<test::BucketStateCalculator>(); - calc->setNodeRetired(nodeRetired); - _clusterStateHandler.notifyClusterStateChanged(calc); - } - void assertJobContext(uint32_t moveToLid, - uint32_t moveFromLid, - uint32_t handleMoveCnt, - uint32_t wantedLidLimit, - uint32_t compactStoreCnt) const - { - EXPECT_EQ(moveToLid, _handler->_moveToLid); - EXPECT_EQ(moveFromLid, _handler->_moveFromLid); - EXPECT_EQ(handleMoveCnt, _handler->_handleMoveCnt); - EXPECT_EQ(handleMoveCnt, _storer._moveCnt); - EXPECT_EQ(wantedLidLimit, _handler->_wantedLidLimit); - EXPECT_EQ(compactStoreCnt, _storer._compactCnt); - } - void assertNoWorkDone() const { - assertJobContext(0, 0, 0, 0, 0); - } - JobTestBase &setupOneDocumentToCompact() { - addStats(10, {1,3,4,5,6,9}, - {{9,2}, // 30% bloat: move 9 -> 2 - {6,7}}); // no documents to move - return *this; - } - void assertOneDocumentCompacted() { - assertJobContext(2, 9, 1, 0, 0); - endScan().compact(); - assertJobContext(2, 9, 1, 7, 1); - } - JobTestBase &setupThreeDocumentsToCompact() { - addStats(10, {1,5,6,9,8,7}, - {{9,2}, // 30% bloat: move 9 -> 2 - {8,3}, // move 8 -> 3 - {7,4}, // move 7 -> 4 - {6,7}}); // no documents to move - return *this; - } -}; - -JobTestBase::~JobTestBase() = default; - -struct JobTest : public JobTestBase { - std::unique_ptr<MyDirectJobRunner> _jobRunner; - - JobTest() - : JobTestBase(), - _jobRunner(std::make_unique<MyDirectJobRunner>(*_job)) - {} - void init(uint32_t allowedLidBloat = ALLOWED_LID_BLOAT, - double allowedLidBloatFactor = ALLOWED_LID_BLOAT_FACTOR, - double resourceLimitFactor = RESOURCE_LIMIT_FACTOR, - vespalib::duration interval = JOB_DELAY, - bool nodeRetired = false, - uint32_t maxOutstandingMoveOps = MAX_OUTSTANDING_MOVE_OPS) { - JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, resourceLimitFactor, interval, nodeRetired, maxOutstandingMoveOps); - _jobRunner = std::make_unique<MyDirectJobRunner>(*_job); - } - void init_with_interval(vespalib::duration interval) { - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, interval); - } - void init_with_node_retired(bool retired) { - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, retired); - } -}; - -struct HandlerTest : public ::testing::Test { - DocBuilder _docBuilder; - std::shared_ptr<BucketDBOwner> _bucketDB; - MyDocumentStore _docStore; - MySubDb _subDb; - LidSpaceCompactionHandler _handler; - HandlerTest() - : _docBuilder(Schema()), - _bucketDB(std::make_shared<BucketDBOwner>()), - _docStore(), - _subDb(_bucketDB, _docStore, _docBuilder.getDocumentTypeRepo()), - _handler(_subDb.maintenance_sub_db, "test") - { - _docStore._readDoc = _docBuilder.startDocument(DOC_ID).endDocument(); - } -}; +using BlockedReason = IBlockableMaintenanceJob::BlockedReason; -TEST_F(JobTest, handler_name_is_used_as_part_of_job_name) +TEST_P(JobTest, handler_name_is_used_as_part_of_job_name) { EXPECT_EQ("lid_space_compaction.myhandler", _job->getName()); } -TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) +TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) { // 20% bloat < 30% allowed bloat addStats(10, {1,3,4,5,6,7,9}, {{9,2}}); @@ -435,7 +17,7 @@ TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) assertNoWorkDone(); } -TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) +TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) { init(3, 0.1); // 20% bloat >= 10% allowed bloat BUT lid bloat (2) < allowed lid bloat (3) @@ -444,7 +26,7 @@ TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) assertNoWorkDone(); } -TEST_F(JobTest, no_move_operation_is_created_and_compaction_is_initiated) +TEST_P(JobTest, no_move_operation_is_created_and_compaction_is_initiated) { // no documents to move: lowestFreeLid(7) > highestUsedLid(6) addStats(10, {1,2,3,4,5,6}, {{6,7}}); @@ -454,14 +36,14 @@ TEST_F(JobTest, no_move_operation_is_created_and_compaction_is_initiated) assertJobContext(0, 0, 0, 7, 1); } -TEST_F(JobTest, one_move_operation_is_created_and_compaction_is_initiated) +TEST_P(JobTest, one_move_operation_is_created_and_compaction_is_initiated) { setupOneDocumentToCompact(); EXPECT_FALSE(run()); // scan assertOneDocumentCompacted(); } -TEST_F(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed) +TEST_P(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed) { setupThreeDocumentsToCompact(); EXPECT_FALSE(run()); @@ -474,26 +56,29 @@ TEST_F(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_ar assertJobContext(4, 7, 3, 7, 1); } -TEST_F(JobTest, job_is_blocked_if_trying_to_move_document_for_frozen_bucket) +TEST_P(JobTest, job_is_blocked_if_trying_to_move_document_for_frozen_bucket) { - _frozenHandler._bucket = BUCKET_ID_1; - EXPECT_FALSE(_job->isBlocked()); - addStats(10, {1,3,4,5,6,9}, {{9,2}}); // 30% bloat: try to move 9 -> 2 - addStats(0, 0, 0, 0); + //TODO Remove test once we no longer use the frozen concept. + if ( ! useBucketDB() ) { + _frozenHandler._bucket = BUCKET_ID_1; + EXPECT_FALSE(_job->isBlocked()); + addStats(10, {1, 3, 4, 5, 6, 9}, {{9, 2}}); // 30% bloat: try to move 9 -> 2 + addStats(0, 0, 0, 0); - EXPECT_TRUE(run()); // bucket frozen - assertNoWorkDone(); - EXPECT_TRUE(_job->isBlocked()); + EXPECT_TRUE(run()); // bucket frozen + assertNoWorkDone(); + EXPECT_TRUE(_job->isBlocked()); - _frozenHandler._bucket = BUCKET_ID_2; - _job->unBlock(BlockedReason::FROZEN_BUCKET); + _frozenHandler._bucket = BUCKET_ID_2; + _job->unBlock(BlockedReason::FROZEN_BUCKET); - EXPECT_FALSE(run()); // unblocked - assertJobContext(2, 9, 1, 0, 0); - EXPECT_FALSE(_job->isBlocked()); + EXPECT_FALSE(run()); // unblocked + assertJobContext(2, 9, 1, 0, 0); + EXPECT_FALSE(_job->isBlocked()); + } } -TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) +TEST_P(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR); addMultiStats(10, {{1,3,4,5,6,9},{1,2,4,5,6,8}}, @@ -513,39 +98,7 @@ TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) assertJobContext(3, 8, 2, 7, 1); } -TEST_F(HandlerTest, handler_uses_doctype_and_subdb_name) -{ - EXPECT_EQ("test.dummysubdb", _handler.getName()); -} - -TEST_F(HandlerTest, createMoveOperation_works_as_expected) -{ - const uint32_t moveToLid = 5; - const uint32_t moveFromLid = 10; - const BucketId bucketId(100); - const Timestamp timestamp(200); - DocumentMetaData document(moveFromLid, timestamp, bucketId, GlobalId()); - { - EXPECT_FALSE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); - IPendingLidTracker::Token token = _subDb._pendingLidsForCommit.produce(moveFromLid); - EXPECT_TRUE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); - MoveOperation::UP op = _handler.createMoveOperation(document, moveToLid); - ASSERT_FALSE(op); - } - EXPECT_FALSE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); - MoveOperation::UP op = _handler.createMoveOperation(document, moveToLid); - ASSERT_TRUE(op); - EXPECT_EQ(10u, _docStore._readLid); - EXPECT_EQ(DbDocumentId(SUBDB_ID, moveFromLid).toString(), - op->getPrevDbDocumentId().toString()); // source - EXPECT_EQ(DbDocumentId(SUBDB_ID, moveToLid).toString(), - op->getDbDocumentId().toString()); // target - EXPECT_EQ(DocumentId(DOC_ID), op->getDocument()->getId()); - EXPECT_EQ(bucketId, op->getBucketId()); - EXPECT_EQ(timestamp, op->getTimestamp()); -} - -TEST_F(JobTest, held_lid_is_not_considered_free_and_blocks_job) +TEST_P(JobTest, held_lid_is_not_considered_free_and_blocks_job) { // Lid 1 on hold or pendingHold, i.e. neither free nor used. addMultiStats(3, {{2}}, {{2, 3}}); @@ -553,7 +106,7 @@ TEST_F(JobTest, held_lid_is_not_considered_free_and_blocks_job) assertNoWorkDone(); } -TEST_F(JobTest, held_lid_is_not_considered_free_with_only_compact) +TEST_P(JobTest, held_lid_is_not_considered_free_with_only_compact) { // Lid 1 on hold or pendingHold, i.e. neither free nor used. addMultiStats(10, {{2}}, {{2, 3}}); @@ -563,7 +116,7 @@ TEST_F(JobTest, held_lid_is_not_considered_free_with_only_compact) assertJobContext(0, 0, 0, 3, 1); } -TEST_F(JobTest, held_lids_are_not_considered_free_with_one_move) +TEST_P(JobTest, held_lids_are_not_considered_free_with_one_move) { // Lids 1,2,3 on hold or pendingHold, i.e. neither free nor used. addMultiStats(10, {{5}}, {{5, 4}, {4, 5}}); @@ -573,7 +126,7 @@ TEST_F(JobTest, held_lids_are_not_considered_free_with_one_move) assertJobContext(4, 5, 1, 5, 1); } -TEST_F(JobTest, resource_starvation_blocks_lid_space_compaction) +TEST_P(JobTest, resource_starvation_blocks_lid_space_compaction) { setupOneDocumentToCompact(); _diskMemUsageNotifier.notify({{100, 0}, {100, 101}}); @@ -581,7 +134,7 @@ TEST_F(JobTest, resource_starvation_blocks_lid_space_compaction) assertNoWorkDone(); } -TEST_F(JobTest, ending_resource_starvation_resumes_lid_space_compaction) +TEST_P(JobTest, ending_resource_starvation_resumes_lid_space_compaction) { setupOneDocumentToCompact(); _diskMemUsageNotifier.notify({{100, 0}, {100, 101}}); @@ -591,7 +144,7 @@ TEST_F(JobTest, ending_resource_starvation_resumes_lid_space_compaction) assertOneDocumentCompacted(); } -TEST_F(JobTest, resource_limit_factor_adjusts_limit) +TEST_P(JobTest, resource_limit_factor_adjusts_limit) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 1.05); setupOneDocumentToCompact(); @@ -600,21 +153,21 @@ TEST_F(JobTest, resource_limit_factor_adjusts_limit) assertOneDocumentCompacted(); } -TEST_F(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs) +TEST_P(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs) { init_with_interval(301s); EXPECT_EQ(300s, _job->getDelay()); EXPECT_EQ(301s, _job->getInterval()); } -TEST_F(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs) +TEST_P(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs) { init_with_interval(299s); EXPECT_EQ(299s, _job->getDelay()); EXPECT_EQ(299s, _job->getInterval()); } -TEST_F(JobTest, job_is_disabled_when_node_is_retired) +TEST_P(JobTest, job_is_disabled_when_node_is_retired) { init_with_node_retired(true); setupOneDocumentToCompact(); @@ -622,7 +175,7 @@ TEST_F(JobTest, job_is_disabled_when_node_is_retired) assertNoWorkDone(); } -TEST_F(JobTest, job_is_disabled_when_node_becomes_retired) +TEST_P(JobTest, job_is_disabled_when_node_becomes_retired) { init_with_node_retired(false); setupOneDocumentToCompact(); @@ -631,7 +184,7 @@ TEST_F(JobTest, job_is_disabled_when_node_becomes_retired) assertNoWorkDone(); } -TEST_F(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) +TEST_P(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) { init_with_node_retired(true); setupOneDocumentToCompact(); @@ -641,37 +194,7 @@ TEST_F(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) assertOneDocumentCompacted(); } -class JobDisabledByRemoveOpsTest : public JobTest { -public: - JobDisabledByRemoveOpsTest() : JobTest() {} - - void job_is_disabled_while_remove_ops_are_ongoing(bool remove_batch) { - setupOneDocumentToCompact(); - _handler->run_remove_ops(remove_batch); - EXPECT_TRUE(run()); // job is disabled - assertNoWorkDone(); - } - - void job_becomes_disabled_if_remove_ops_starts(bool remove_batch) { - setupThreeDocumentsToCompact(); - EXPECT_FALSE(run()); // job executed as normal (with more work to do) - assertJobContext(2, 9, 1, 0, 0); - - _handler->run_remove_ops(remove_batch); - EXPECT_TRUE(run()); // job is disabled - assertJobContext(2, 9, 1, 0, 0); - } - - void job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(bool remove_batch) { - job_becomes_disabled_if_remove_ops_starts(remove_batch); - - _handler->stop_remove_ops(remove_batch); - EXPECT_FALSE(run()); // job executed as normal (with more work to do) - assertJobContext(3, 8, 2, 0, 0); - } -}; - -TEST_F(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker) +TEST_P(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker) { auto& remove_batch_tracker = _handler->_rm_listener->get_remove_batch_tracker(); EXPECT_EQ(vespalib::from_s(21.0), remove_batch_tracker.get_time_budget_per_op()); @@ -682,65 +205,38 @@ TEST_F(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rat EXPECT_EQ(vespalib::from_s(20.0), remove_tracker.get_time_budget_window()); } -TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing) +TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing) { job_is_disabled_while_remove_ops_are_ongoing(true); } -TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts) +TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts) { job_becomes_disabled_if_remove_ops_starts(true); } -TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing) +TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing) { job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(true); } -TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing) +TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing) { job_is_disabled_while_remove_ops_are_ongoing(false); } -TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start) +TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start) { job_becomes_disabled_if_remove_ops_starts(false); } -TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing) +TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing) { job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(false); } -struct MaxOutstandingJobTest : public JobTest { - std::unique_ptr<MyCountJobRunner> runner; - MaxOutstandingJobTest() - : JobTest(), - runner() - {} - void init(uint32_t maxOutstandingMoveOps) { - JobTest::init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, - RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, maxOutstandingMoveOps); - runner = std::make_unique<MyCountJobRunner>(*_job); - } - void assertRunToBlocked() { - EXPECT_TRUE(run()); // job becomes blocked as max outstanding limit is reached - EXPECT_TRUE(_job->isBlocked()); - EXPECT_TRUE(_job->isBlocked(BlockedReason::OUTSTANDING_OPS)); - } - void assertRunToNotBlocked() { - EXPECT_FALSE(run()); - EXPECT_FALSE(_job->isBlocked()); - } - void unblockJob(uint32_t expRunnerCnt) { - _handler->clearMoveDoneContexts(); // unblocks job and try to execute it via runner - EXPECT_EQ(expRunnerCnt, runner->runCnt); - EXPECT_FALSE(_job->isBlocked()); - } -}; - -TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1) +TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1) { init(1); setupThreeDocumentsToCompact(); @@ -763,7 +259,7 @@ TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move assertJobContext(4, 7, 3, 7, 1); } -TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2) +TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2) { init(2); setupThreeDocumentsToCompact(); @@ -778,6 +274,11 @@ TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move assertJobContext(4, 7, 3, 0, 0); endScan().compact(); assertJobContext(4, 7, 3, 7, 1); + sync(); } +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobTest, ::testing::Values(false, true)); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobDisabledByRemoveOpsTest, ::testing::Values(false, true)); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, MaxOutstandingJobTest, ::testing::Values(false, true)); + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_handler_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_handler_test.cpp new file mode 100644 index 00000000000..2603c041db0 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_handler_test.cpp @@ -0,0 +1,60 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lid_space_common.h" +#include <vespa/vespalib/gtest/gtest.h> + +struct HandlerTest : public ::testing::Test { + DocBuilder _docBuilder; + std::shared_ptr<BucketDBOwner> _bucketDB; + MyDocumentStore _docStore; + MySubDb _subDb; + LidSpaceCompactionHandler _handler; + HandlerTest(); + ~HandlerTest(); +}; + +HandlerTest::HandlerTest() + : _docBuilder(Schema()), + _bucketDB(std::make_shared<BucketDBOwner>()), + _docStore(), + _subDb(_bucketDB, _docStore, _docBuilder.getDocumentTypeRepo()), + _handler(_subDb.maintenance_sub_db, "test") +{ + _docStore._readDoc = _docBuilder.startDocument(DOC_ID).endDocument(); +} + +HandlerTest::~HandlerTest() = default; + +TEST_F(HandlerTest, handler_uses_doctype_and_subdb_name) +{ + EXPECT_EQ("test.dummysubdb", _handler.getName()); +} + +TEST_F(HandlerTest, createMoveOperation_works_as_expected) +{ + const uint32_t moveToLid = 5; + const uint32_t moveFromLid = 10; + const BucketId bucketId(100); + const Timestamp timestamp(200); + DocumentMetaData document(moveFromLid, timestamp, bucketId, GlobalId()); + { + EXPECT_FALSE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); + IPendingLidTracker::Token token = _subDb._pendingLidsForCommit.produce(moveFromLid); + EXPECT_TRUE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); + MoveOperation::UP op = _handler.createMoveOperation(document, moveToLid); + ASSERT_FALSE(op); + } + EXPECT_FALSE(_subDb.maintenance_sub_db.lidNeedsCommit(moveFromLid)); + MoveOperation::UP op = _handler.createMoveOperation(document, moveToLid); + ASSERT_TRUE(op); + EXPECT_EQ(10u, _docStore._readLid); + EXPECT_EQ(DbDocumentId(SUBDB_ID, moveFromLid).toString(), + op->getPrevDbDocumentId().toString()); // source + EXPECT_EQ(DbDocumentId(SUBDB_ID, moveToLid).toString(), + op->getDbDocumentId().toString()); // target + EXPECT_EQ(DocumentId(DOC_ID), op->getDocument()->getId()); + EXPECT_EQ(bucketId, op->getBucketId()); + EXPECT_EQ(timestamp, op->getTimestamp()); +} + +GTEST_MAIN_RUN_ALL_TESTS()
\ No newline at end of file diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp new file mode 100644 index 00000000000..bac793fc6e5 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp @@ -0,0 +1,247 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lid_space_jobtest.h" +#include <vespa/searchcore/proton/server/lid_space_compaction_job.h> +#include <vespa/searchcore/proton/server/lid_space_compaction_job_take2.h> +#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> + +using BlockedReason = IBlockableMaintenanceJob::BlockedReason; + +struct MyDirectJobRunner : public IMaintenanceJobRunner { + IMaintenanceJob &_job; + explicit MyDirectJobRunner(IMaintenanceJob &job) + : _job(job) + { + _job.registerRunner(this); + } + void run() override { _job.run(); } +}; + +struct MyCountJobRunner : public IMaintenanceJobRunner { + uint32_t runCnt; + explicit MyCountJobRunner(IMaintenanceJob &job) : runCnt(0) { + job.registerRunner(this); + } + void run() override { ++runCnt; } +}; + +JobTestBase::JobTestBase() + : _handler(), + _storer(), + _frozenHandler(), + _diskMemUsageNotifier(), + _clusterStateHandler(), + _job() +{ + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, MAX_OUTSTANDING_MOVE_OPS); +} + +void +JobTestBase::init(uint32_t allowedLidBloat, + double allowedLidBloatFactor, + double resourceLimitFactor, + vespalib::duration interval, + bool nodeRetired, + uint32_t maxOutstandingMoveOps) +{ + _handler = std::make_unique<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, useBucketDB()); + DocumentDBLidSpaceCompactionConfig compactCfg(interval, allowedLidBloat, allowedLidBloatFactor, + REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, false); + BlockableMaintenanceJobConfig blockableCfg(resourceLimitFactor, maxOutstandingMoveOps); + + if (useBucketDB()) { + _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4); + _job = std::make_unique<lidspace::CompactionJob>(compactCfg, *_handler, _storer, *_bucketExecutor, _diskMemUsageNotifier, + blockableCfg, _clusterStateHandler, nodeRetired, + document::BucketSpace::placeHolder()); + } else { + _job = std::make_unique<LidSpaceCompactionJob>(compactCfg, *_handler, _storer, _frozenHandler, _diskMemUsageNotifier, + blockableCfg, _clusterStateHandler, nodeRetired); + } +} +void +JobTestBase::sync() const { + if (_bucketExecutor) { + _bucketExecutor->sync(); + } +} + +JobTestBase & +JobTestBase::addStats(uint32_t docIdLimit, const LidVector &usedLids, const LidPairVector &usedFreePairs) +{ + return addMultiStats(docIdLimit, {usedLids}, usedFreePairs); +} +JobTestBase & +JobTestBase::addMultiStats(uint32_t docIdLimit, + const std::vector<LidVector> &usedLidsVector, + const LidPairVector &usedFreePairs) { + uint32_t usedLids = usedLidsVector[0].size(); + for (auto pair : usedFreePairs) { + uint32_t highestUsedLid = pair.first; + uint32_t lowestFreeLid = pair.second; + _handler->_stats.emplace_back(docIdLimit, usedLids, lowestFreeLid, highestUsedLid); + } + _handler->_lids = usedLidsVector; + return *this; +} +JobTestBase & +JobTestBase::addStats(uint32_t docIdLimit, + uint32_t numDocs, + uint32_t lowestFreeLid, + uint32_t highestUsedLid) { + _handler->_stats.emplace_back(docIdLimit, numDocs, lowestFreeLid, highestUsedLid); + return *this; +} +bool +JobTestBase::run() const { + return _job->run(); +} +JobTestBase & +JobTestBase::endScan() { + EXPECT_FALSE(run()); + return *this; +} +JobTestBase & +JobTestBase::compact() { + EXPECT_TRUE(run()); + return *this; +} +void +JobTestBase::notifyNodeRetired(bool nodeRetired) { + test::BucketStateCalculator::SP calc = std::make_shared<test::BucketStateCalculator>(); + calc->setNodeRetired(nodeRetired); + _clusterStateHandler.notifyClusterStateChanged(calc); +} +void +JobTestBase::assertJobContext(uint32_t moveToLid, + uint32_t moveFromLid, + uint32_t handleMoveCnt, + uint32_t wantedLidLimit, + uint32_t compactStoreCnt) const +{ + sync(); + EXPECT_EQ(moveToLid, _handler->_moveToLid); + EXPECT_EQ(moveFromLid, _handler->_moveFromLid); + EXPECT_EQ(handleMoveCnt, _handler->_handleMoveCnt); + EXPECT_EQ(handleMoveCnt, _storer._moveCnt); + EXPECT_EQ(wantedLidLimit, _handler->_wantedLidLimit); + EXPECT_EQ(compactStoreCnt, _storer._compactCnt); +} +void +JobTestBase::assertNoWorkDone() const { + assertJobContext(0, 0, 0, 0, 0); +} +JobTestBase & +JobTestBase::setupOneDocumentToCompact() { + addStats(10, {1,3,4,5,6,9}, + {{9,2}, // 30% bloat: move 9 -> 2 + {6,7}}); // no documents to move + return *this; +} +void +JobTestBase::assertOneDocumentCompacted() { + assertJobContext(2, 9, 1, 0, 0); + endScan().compact(); + assertJobContext(2, 9, 1, 7, 1); +} +JobTestBase & +JobTestBase::setupThreeDocumentsToCompact() { + addStats(10, {1,5,6,9,8,7}, + {{9,2}, // 30% bloat: move 9 -> 2 + {8,3}, // move 8 -> 3 + {7,4}, // move 7 -> 4 + {6,7}}); // no documents to move + return *this; +} + +JobTestBase::~JobTestBase() { + _handler->clearMoveDoneContexts(); +} + +JobTest::JobTest() + : JobTestBase(), + _jobRunner(std::make_unique<MyDirectJobRunner>(*_job)) +{} +JobTest::~JobTest() = default; +void +JobTest::init(uint32_t allowedLidBloat, + double allowedLidBloatFactor, + double resourceLimitFactor, + vespalib::duration interval, + bool nodeRetired, + uint32_t maxOutstandingMoveOps) +{ + JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, resourceLimitFactor, interval, nodeRetired, maxOutstandingMoveOps); + _jobRunner = std::make_unique<MyDirectJobRunner>(*_job); +} +void +JobTest::init_with_interval(vespalib::duration interval) { + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, interval); +} +void +JobTest::init_with_node_retired(bool retired) { + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, retired); +} + + +JobDisabledByRemoveOpsTest::JobDisabledByRemoveOpsTest() : JobTest() {} +JobDisabledByRemoveOpsTest::~JobDisabledByRemoveOpsTest() = default; + +void +JobDisabledByRemoveOpsTest::job_is_disabled_while_remove_ops_are_ongoing(bool remove_batch) { + setupOneDocumentToCompact(); + _handler->run_remove_ops(remove_batch); + EXPECT_TRUE(run()); // job is disabled + assertNoWorkDone(); +} + +void +JobDisabledByRemoveOpsTest::job_becomes_disabled_if_remove_ops_starts(bool remove_batch) { + setupThreeDocumentsToCompact(); + EXPECT_FALSE(run()); // job executed as normal (with more work to do) + assertJobContext(2, 9, 1, 0, 0); + + _handler->run_remove_ops(remove_batch); + EXPECT_TRUE(run()); // job is disabled + assertJobContext(2, 9, 1, 0, 0); +} + +void +JobDisabledByRemoveOpsTest::job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(bool remove_batch) { + job_becomes_disabled_if_remove_ops_starts(remove_batch); + + _handler->stop_remove_ops(remove_batch); + EXPECT_FALSE(run()); // job executed as normal (with more work to do) + assertJobContext(3, 8, 2, 0, 0); +} + +MaxOutstandingJobTest::MaxOutstandingJobTest() + : JobTest(), + runner() +{} +MaxOutstandingJobTest::~MaxOutstandingJobTest() = default; + +void +MaxOutstandingJobTest::init(uint32_t maxOutstandingMoveOps) { + JobTest::init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, + RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, maxOutstandingMoveOps); + runner = std::make_unique<MyCountJobRunner>(*_job); +} +void +MaxOutstandingJobTest::assertRunToBlocked() { + EXPECT_TRUE(run()); // job becomes blocked as max outstanding limit is reached + EXPECT_TRUE(_job->isBlocked()); + EXPECT_TRUE(_job->isBlocked(BlockedReason::OUTSTANDING_OPS)); +} +void +MaxOutstandingJobTest::assertRunToNotBlocked() { + EXPECT_FALSE(run()); + EXPECT_FALSE(_job->isBlocked()); +} +void +MaxOutstandingJobTest::unblockJob(uint32_t expRunnerCnt) { + _handler->clearMoveDoneContexts(); // unblocks job and try to execute it via runner + EXPECT_EQ(expRunnerCnt, runner->runCnt); + EXPECT_FALSE(_job->isBlocked()); +} + diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h new file mode 100644 index 00000000000..2ca50c064bd --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h @@ -0,0 +1,86 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lid_space_common.h" +#include <vespa/searchcore/proton/server/blockable_maintenance_job.h> +#include <vespa/persistence/spi/bucketexecutor.h> +#include <vespa/vespalib/gtest/gtest.h> + +struct JobTestBase : public ::testing::TestWithParam<bool> { + std::unique_ptr<storage::spi::BucketExecutor> _bucketExecutor; + std::unique_ptr<MyHandler> _handler; + MyStorer _storer; + MyFrozenBucketHandler _frozenHandler; + test::DiskMemUsageNotifier _diskMemUsageNotifier; + test::ClusterStateHandler _clusterStateHandler; + std::unique_ptr<BlockableMaintenanceJob> _job; + JobTestBase(); + ~JobTestBase() override; + void init(uint32_t allowedLidBloat, + double allowedLidBloatFactor, + double resourceLimitFactor, + vespalib::duration interval, + bool nodeRetired, + uint32_t maxOutstandingMoveOps); + JobTestBase &addStats(uint32_t docIdLimit, + const LidVector &usedLids, + const LidPairVector &usedFreePairs); + JobTestBase &addMultiStats(uint32_t docIdLimit, + const std::vector<LidVector> &usedLidsVector, + const LidPairVector &usedFreePairs); + JobTestBase &addStats(uint32_t docIdLimit, + uint32_t numDocs, + uint32_t lowestFreeLid, + uint32_t highestUsedLid); + bool run() const; + JobTestBase &endScan(); + JobTestBase &compact(); + void notifyNodeRetired(bool nodeRetired); + void assertJobContext(uint32_t moveToLid, + uint32_t moveFromLid, + uint32_t handleMoveCnt, + uint32_t wantedLidLimit, + uint32_t compactStoreCnt) const; + void assertNoWorkDone() const; + JobTestBase &setupOneDocumentToCompact(); + void assertOneDocumentCompacted(); + JobTestBase &setupThreeDocumentsToCompact(); + void sync() const; + bool useBucketDB() const { return GetParam(); } +}; + +struct JobTest : public JobTestBase { + std::unique_ptr<IMaintenanceJobRunner> _jobRunner; + + JobTest(); + ~JobTest(); + void init(uint32_t allowedLidBloat, + double allowedLidBloatFactor, + double resourceLimitFactor = RESOURCE_LIMIT_FACTOR, + vespalib::duration interval = JOB_DELAY, + bool nodeRetired = false, + uint32_t maxOutstandingMoveOps = MAX_OUTSTANDING_MOVE_OPS); + void init_with_interval(vespalib::duration interval); + void init_with_node_retired(bool retired); +}; + +class JobDisabledByRemoveOpsTest : public JobTest { +public: + JobDisabledByRemoveOpsTest(); + ~JobDisabledByRemoveOpsTest(); + + void job_is_disabled_while_remove_ops_are_ongoing(bool remove_batch); + void job_becomes_disabled_if_remove_ops_starts(bool remove_batch); + void job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(bool remove_batch); +}; + +class MyCountJobRunner; + +struct MaxOutstandingJobTest : public JobTest { + std::unique_ptr<MyCountJobRunner> runner; + MaxOutstandingJobTest(); + ~MaxOutstandingJobTest(); + void init(uint32_t maxOutstandingMoveOps); + void assertRunToBlocked(); + void assertRunToNotBlocked(); + void unblockJob(uint32_t expRunnerCnt); +}; |