diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-05-03 12:34:45 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-05-03 12:55:20 +0000 |
commit | a6d1d5d762265e159fdf738c709b8296af29f4ce (patch) | |
tree | e6d784079735c65a2908bafdafd3f0bd538712d4 /searchcore | |
parent | 54164220f1d224808aa86bf5777eaef837d61034 (diff) |
- GC legacy lidspace compaction job and bucket move job.
- GC duplicate tests. They are tested more thoroughly in the bucket mover tests.
Diffstat (limited to 'searchcore')
6 files changed, 36 insertions, 1108 deletions
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt index d11da09b737..88bd84cbd16 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt @@ -5,18 +5,6 @@ vespa_add_library(searchcore_bucketmover_test STATIC bucketmover_common.cpp ) -vespa_add_executable(searchcore_documentbucketmover_test_app TEST - SOURCES - documentbucketmover_test.cpp - DEPENDS - searchcore_bucketmover_test - searchcore_test - searchcore_server - searchcore_feedoperation - GTest::GTest -) -vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app) - vespa_add_executable(searchcore_documentbucketmover_v2_test_app TEST SOURCES documentbucketmover_v2_test.cpp diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp deleted file mode 100644 index bb7180dadf1..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ /dev/null @@ -1,906 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketmover_common.h" -#include <vespa/searchcore/proton/server/bucketmovejob.h> -#include <vespa/searchcore/proton/server/document_db_maintenance_config.h> -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("document_bucket_mover_test"); - -using namespace proton; -using namespace proton::move::test; -using document::BucketId; -using document::test::makeBucketSpace; -using proton::bucketdb::BucketCreateNotifier; -using storage::spi::BucketInfo; -using BlockedReason = IBlockableMaintenanceJob::BlockedReason; -using MoveOperationVector = std::vector<MoveOperation>; - -struct MyFrozenBucketHandler : public IFrozenBucketHandler -{ - std::set<BucketId> _frozen; - std::set<IBucketFreezeListener *> _listeners; - - MyFrozenBucketHandler() - : IFrozenBucketHandler(), - _frozen(), - _listeners() - { - } - - ~MyFrozenBucketHandler() override { - assert(_listeners.empty()); - } - - MyFrozenBucketHandler &addFrozen(const BucketId &bucket) { - _frozen.insert(bucket); - return *this; - } - MyFrozenBucketHandler &remFrozen(const BucketId &bucket) { - _frozen.erase(bucket); - for (auto &listener : _listeners) { - listener->notifyThawedBucket(bucket); - } - return *this; - } - void addListener(IBucketFreezeListener *listener) override { - _listeners.insert(listener); - } - void removeListener(IBucketFreezeListener *listener) override { - _listeners.erase(listener); - } - - ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override { - return (_frozen.count(bucket) != 0) - ? ExclusiveBucketGuard::UP() - : std::make_unique<ExclusiveBucketGuard>(bucket); - } -}; - -struct ControllerFixtureBase : public ::testing::Test -{ - test::UserDocumentsBuilder _builder; - test::BucketStateCalculator::SP _calc; - test::ClusterStateHandler _clusterStateHandler; - test::BucketHandler _bucketHandler; - MyBucketModifiedHandler _modifiedHandler; - std::shared_ptr<bucketdb::BucketDBOwner> _bucketDB; - MyMoveHandler _moveHandler; - MySubDb _ready; - MySubDb _notReady; - MyFrozenBucketHandler _fbh; - BucketCreateNotifier _bucketCreateNotifier; - test::DiskMemUsageNotifier _diskMemUsageNotifier; - std::shared_ptr<BucketMoveJob> _bmj; - MyCountJobRunner _runner; - ControllerFixtureBase(const BlockableMaintenanceJobConfig &blockableConfig, bool storeMoveDoneContexts); - ~ControllerFixtureBase(); - ControllerFixtureBase &addReady(const BucketId &bucket) { - _calc->addReady(bucket); - return *this; - } - ControllerFixtureBase &remReady(const BucketId &bucket) { - _calc->remReady(bucket); - return *this; - } - ControllerFixtureBase &changeCalc() { - _calc->resetAsked(); - _moveHandler.reset(); - _modifiedHandler.reset(); - _clusterStateHandler.notifyClusterStateChanged(_calc); - return *this; - } - ControllerFixtureBase &addFrozen(const BucketId &bucket) { - _fbh.addFrozen(bucket); - return *this; - } - ControllerFixtureBase &remFrozen(const BucketId &bucket) { - _fbh.remFrozen(bucket); - _bmj->notifyThawedBucket(bucket); - return *this; - } - ControllerFixtureBase &activateBucket(const BucketId &bucket) { - _ready.setBucketState(bucket, true); - _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::ACTIVE); - return *this; - } - ControllerFixtureBase &deactivateBucket(const BucketId &bucket) { - _ready.setBucketState(bucket, false); - _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::NOT_ACTIVE); - return *this; - } - const MoveOperationVector &docsMoved() const { - return _moveHandler._moves; - } - const std::vector<BucketId> &bucketsModified() const { - return _modifiedHandler._modified; - } - const BucketId::List &calcAsked() const { - return _calc->asked(); - } - void runLoop() { - while (!_bmj->isBlocked() && !_bmj->run()) { - } - } -}; - -ControllerFixtureBase::ControllerFixtureBase(const BlockableMaintenanceJobConfig &blockableConfig, bool storeMoveDoneContexts) - : _builder(), - _calc(std::make_shared<test::BucketStateCalculator>()), - _bucketHandler(), - _modifiedHandler(), - _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), - _moveHandler(*_bucketDB, storeMoveDoneContexts), - _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY), - _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY), - _fbh(), - _bucketCreateNotifier(), - _diskMemUsageNotifier(), - _bmj(std::make_shared<BucketMoveJob>(_calc, _moveHandler, _modifiedHandler, _ready._subDb, _notReady._subDb, - _fbh, _bucketCreateNotifier, _clusterStateHandler, _bucketHandler, - _diskMemUsageNotifier, blockableConfig, "test", makeBucketSpace())), - _runner(*_bmj) -{ -} - -ControllerFixtureBase::~ControllerFixtureBase() = default; -constexpr double RESOURCE_LIMIT_FACTOR = 1.0; -constexpr uint32_t MAX_OUTSTANDING_OPS = 10; -const BlockableMaintenanceJobConfig BLOCKABLE_CONFIG(RESOURCE_LIMIT_FACTOR, MAX_OUTSTANDING_OPS); - -struct ControllerFixture : public ControllerFixtureBase -{ - ControllerFixture(const BlockableMaintenanceJobConfig &blockableConfig = BLOCKABLE_CONFIG) - : ControllerFixtureBase(blockableConfig, blockableConfig.getMaxOutstandingMoveOps() != MAX_OUTSTANDING_OPS) - { - _builder.createDocs(1, 1, 4); // 3 docs - _builder.createDocs(2, 4, 6); // 2 docs - _ready.insertDocs(_builder.getDocs()); - _builder.clearDocs(); - _builder.createDocs(3, 1, 3); // 2 docs - _builder.createDocs(4, 3, 6); // 3 docs - _notReady.insertDocs(_builder.getDocs()); - } -}; - -struct OnlyReadyControllerFixture : public ControllerFixtureBase -{ - OnlyReadyControllerFixture() : ControllerFixtureBase(BLOCKABLE_CONFIG, false) - { - _builder.createDocs(1, 1, 2); // 1 docs - _builder.createDocs(2, 2, 4); // 2 docs - _builder.createDocs(3, 4, 7); // 3 docs - _builder.createDocs(4, 7, 11); // 4 docs - _ready.insertDocs(_builder.getDocs()); - } -}; - -TEST_F(ControllerFixture, require_that_nothing_is_moved_if_bucket_state_says_so) -{ - EXPECT_FALSE(_bmj->done()); - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - _bmj->scanAndMove(4, 3); - EXPECT_TRUE(_bmj->done()); - EXPECT_TRUE(docsMoved().empty()); - EXPECT_TRUE(bucketsModified().empty()); -} - -TEST_F(ControllerFixture, require_that_not_ready_bucket_is_moved_to_ready_if_bucket_state_says_so) -{ - // bucket 4 should be moved - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(4)); - _bmj->scanAndMove(4, 3); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[0]); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[1]); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[2]); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(4), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_ready_bucket_is_moved_to_not_ready_if_bucket_state_says_so) -{ - // bucket 2 should be moved - addReady(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, docsMoved().size()); - assertEqual(_ready.bucket(2), _ready.docs(2)[0], 1, 2, docsMoved()[0]); - assertEqual(_ready.bucket(2), _ready.docs(2)[1], 1, 2, docsMoved()[1]); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_maxBucketsToScan_is_taken_into_consideration_between_not_ready_and_ready_scanning) -{ - // bucket 4 should moved (last bucket) - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(4)); - - // buckets 1, 2, and 3 considered - _bmj->scanAndMove(3, 3); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - // move bucket 4 - _bmj->scanAndMove(1, 4); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[0]); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[1]); - assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[2]); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(4), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_we_move_buckets_in_several_steps) -{ - // bucket 2, 3, and 4 should be moved - addReady(_ready.bucket(1)); - addReady(_notReady.bucket(3)); - addReady(_notReady.bucket(4)); - - // consider move bucket 1 - _bmj->scanAndMove(1, 2); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - // move bucket 2, docs 1,2 - _bmj->scanAndMove(1, 2); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_TRUE(assertEqual(_ready.bucket(2), _ready.docs(2)[0], 1, 2, docsMoved()[0])); - EXPECT_TRUE(assertEqual(_ready.bucket(2), _ready.docs(2)[1], 1, 2, docsMoved()[1])); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]); - - // move bucket 3, docs 1,2 - _bmj->scanAndMove(1, 2); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(4u, docsMoved().size()); - EXPECT_TRUE(assertEqual(_notReady.bucket(3), _notReady.docs(3)[0], 2, 1, docsMoved()[2])); - EXPECT_TRUE(assertEqual(_notReady.bucket(3), _notReady.docs(3)[1], 2, 1, docsMoved()[3])); - EXPECT_EQ(2u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(3), bucketsModified()[1]); - - // move bucket 4, docs 1,2 - _bmj->scanAndMove(1, 2); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(6u, docsMoved().size()); - EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[4])); - EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[5])); - EXPECT_EQ(2u, bucketsModified().size()); - - // move bucket 4, docs 3 - _bmj->scanAndMove(1, 2); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(7u, docsMoved().size()); - EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[6])); - EXPECT_EQ(3u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(4), bucketsModified()[2]); -} - -TEST_F(ControllerFixture, require_that_we_can_change_calculator_and_continue_scanning_where_we_left_off) -{ - // no buckets should move - // original scan sequence is bucket1, bucket2, bucket3, bucket4 - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - - // start with bucket2 - _bmj->scanAndMove(1, 0); - changeCalc(); - _bmj->scanAndMove(5, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(2), calcAsked()[0]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[3]); - - // start with bucket3 - changeCalc(); - _bmj->scanAndMove(2, 0); - changeCalc(); - _bmj->scanAndMove(5, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[0]); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[1]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[2]); - EXPECT_EQ(_ready.bucket(2), calcAsked()[3]); - - // start with bucket4 - changeCalc(); - _bmj->scanAndMove(3, 0); - changeCalc(); - _bmj->scanAndMove(5, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[0]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[1]); - EXPECT_EQ(_ready.bucket(2), calcAsked()[2]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[3]); - - // start with bucket1 - changeCalc(); - _bmj->scanAndMove(5, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); - EXPECT_EQ(_ready.bucket(2), calcAsked()[1]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[2]); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[3]); - - // change calc in second pass - changeCalc(); - _bmj->scanAndMove(3, 0); - changeCalc(); - _bmj->scanAndMove(2, 0); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[0]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[1]); - changeCalc(); - _bmj->scanAndMove(5, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(2), calcAsked()[0]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[3]); - - // check 1 bucket at a time, start with bucket2 - changeCalc(); - _bmj->scanAndMove(1, 0); - changeCalc(); - _bmj->scanAndMove(1, 0); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(2), calcAsked()[0]); - _bmj->scanAndMove(1, 0); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]); - _bmj->scanAndMove(1, 0); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(3u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]); - _bmj->scanAndMove(1, 0); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[3]); -} - -TEST_F(ControllerFixture, require_that_current_bucket_moving_is_cancelled_when_we_change_calculator) -{ - // bucket 1 should be moved - addReady(_ready.bucket(2)); - _bmj->scanAndMove(3, 1); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(1u, calcAsked().size()); - changeCalc(); // Not cancelled, bucket 1 still moving to notReady - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); - _calc->resetAsked(); - _bmj->scanAndMove(2, 1); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, calcAsked().size()); - addReady(_ready.bucket(1)); - changeCalc(); // cancelled, bucket 1 no longer moving to notReady - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); - _calc->resetAsked(); - remReady(_ready.bucket(1)); - changeCalc(); // not cancelled. No active bucket move - EXPECT_EQ(0u, calcAsked().size()); - _calc->resetAsked(); - _bmj->scanAndMove(2, 1); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(2u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(2), calcAsked()[0]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]); - _bmj->scanAndMove(2, 3); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]); - EXPECT_EQ(_ready.bucket(1), calcAsked()[3]); -} - -TEST_F(ControllerFixture, require_that_last_bucket_is_moved_before_reporting_done) -{ - // bucket 4 should be moved - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(4)); - _bmj->scanAndMove(4, 1); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(4u, calcAsked().size()); - _bmj->scanAndMove(0, 2); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(4u, calcAsked().size()); -} - -TEST_F(ControllerFixture, require_that_frozen_bucket_is_not_moved_until_thawed) -{ - // bucket 1 should be moved but is frozen - addReady(_ready.bucket(2)); - addFrozen(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 1 - remFrozen(_ready.bucket(1)); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - _bmj->scanAndMove(0, 3); // move delayed and thawed bucket 1 - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_thawed_bucket_is_moved_before_other_buckets) -{ - // bucket 2 should be moved but is frozen. - // bucket 3 & 4 should also be moved - addReady(_ready.bucket(1)); - addReady(_notReady.bucket(3)); - addReady(_notReady.bucket(4)); - addFrozen(_ready.bucket(2)); - _bmj->scanAndMove(3, 2); // delay bucket 2, move bucket 3 - remFrozen(_ready.bucket(2)); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]); - _bmj->scanAndMove(2, 2); // move thawed bucket 2 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(4u, docsMoved().size()); - EXPECT_EQ(2u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(2), bucketsModified()[1]); - _bmj->scanAndMove(1, 4); // move bucket 4 - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(7u, docsMoved().size()); - EXPECT_EQ(3u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(4), bucketsModified()[2]); -} - -TEST_F(ControllerFixture, require_that_re_frozen_thawed_bucket_is_not_moved_until_re_thawed) -{ - // bucket 1 should be moved but is re-frozen - addReady(_ready.bucket(2)); - addFrozen(_ready.bucket(1)); - _bmj->scanAndMove(1, 0); // scan, delay frozen bucket 1 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); - remFrozen(_ready.bucket(1)); - addFrozen(_ready.bucket(1)); - _bmj->scanAndMove(1, 0); // scan, but nothing to move - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(3u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[1]); - EXPECT_EQ(_ready.bucket(2), calcAsked()[2]); - remFrozen(_ready.bucket(1)); - _bmj->scanAndMove(3, 4); // move delayed and thawed bucket 1 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]); - EXPECT_EQ(4u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[3]); - _bmj->scanAndMove(2, 0); // scan the rest - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(6u, calcAsked().size()); -} - -TEST_F(ControllerFixture, require_that_thawed_bucket_is_not_moved_if_new_calculator_does_not_say_so) -{ - // bucket 3 should be moved - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(3)); - addFrozen(_notReady.bucket(3)); - _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 3 - remFrozen(_notReady.bucket(3)); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(4u, calcAsked().size()); - changeCalc(); - remReady(_notReady.bucket(3)); - _bmj->scanAndMove(0, 3); // consider delayed bucket 3 - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[0]); -} - -TEST_F(ControllerFixture, require_that_current_bucket_mover_is_cancelled_if_bucket_is_frozen) -{ - // bucket 3 should be moved - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(3)); - _bmj->scanAndMove(3, 1); // move 1 doc from bucket 3 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(3u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); - EXPECT_EQ(_ready.bucket(2), calcAsked()[1]); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[2]); - - addFrozen(_notReady.bucket(3)); - _bmj->scanAndMove(1, 3); // done scanning - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(3u, calcAsked().size()); - - _bmj->scanAndMove(1, 3); // done scanning - remFrozen(_notReady.bucket(3)); - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(4u, calcAsked().size()); - - EXPECT_EQ(_notReady.bucket(4), calcAsked()[3]); - _bmj->scanAndMove(0, 2); // move all docs from bucket 3 again - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]); - EXPECT_EQ(5u, calcAsked().size()); - EXPECT_EQ(_notReady.bucket(3), calcAsked()[4]); -} - -TEST_F(ControllerFixture, require_that_current_bucket_mover_is_not_cancelled_if_another_bucket_is_frozen) -{ - // bucket 3 and 4 should be moved - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(3)); - addReady(_notReady.bucket(4)); - _bmj->scanAndMove(3, 1); // move 1 doc from bucket 3 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(3u, calcAsked().size()); - addFrozen(_notReady.bucket(4)); - _bmj->scanAndMove(1, 2); // move rest of docs from bucket 3 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]); - EXPECT_EQ(3u, calcAsked().size()); -} - -TEST_F(ControllerFixture, require_that_active_bucket_is_not_moved_from_ready_to_not_ready_until_being_not_active) -{ - // bucket 1 should be moved but is active - addReady(_ready.bucket(2)); - activateBucket(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1 - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - deactivateBucket(_ready.bucket(1)); - EXPECT_FALSE(_bmj->done()); - _bmj->scanAndMove(0, 3); // move delayed and de-activated bucket 1 - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]); -} - -TEST_F(OnlyReadyControllerFixture, require_that_de_activated_bucket_is_moved_before_other_buckets) -{ - // bucket 1, 2, 3 should be moved (but bucket 1 is active) - addReady(_ready.bucket(4)); - activateBucket(_ready.bucket(1)); - _bmj->scanAndMove(2, 4); // delay bucket 1, move bucket 2 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]); - - deactivateBucket(_ready.bucket(1)); - _bmj->scanAndMove(2, 4); // move de-activated bucket 1 - EXPECT_FALSE(_bmj->done()); - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(2u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[1]); - - _bmj->scanAndMove(2, 4); // move bucket 3 - // EXPECT_TRUE(_bmj->done()); // TODO(geirst): fix this - EXPECT_EQ(6u, docsMoved().size()); - EXPECT_EQ(3u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(3), bucketsModified()[2]); -} - -TEST_F(ControllerFixture, require_that_de_activated_bucket_is_not_moved_if_new_calculator_does_not_say_so) -{ - // bucket 1 should be moved - addReady(_ready.bucket(2)); - activateBucket(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1 - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - deactivateBucket(_ready.bucket(1)); - addReady(_ready.bucket(1)); - changeCalc(); - _bmj->scanAndMove(0, 3); // consider delayed bucket 3 - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - EXPECT_EQ(1u, calcAsked().size()); - EXPECT_EQ(_ready.bucket(1), calcAsked()[0]); -} - -TEST_F(ControllerFixture, require_that_de_activated_bucket_is_not_moved_if_frozen_as_well) -{ - // bucket 1 should be moved - addReady(_ready.bucket(2)); - activateBucket(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1 - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - addFrozen(_ready.bucket(1)); - deactivateBucket(_ready.bucket(1)); - _bmj->scanAndMove(0, 3); // bucket 1 de-activated but frozen - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - remFrozen(_ready.bucket(1)); - _bmj->scanAndMove(0, 3); // handle thawed bucket 1 - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_thawed_bucket_is_not_moved_if_active_as_well) -{ - // bucket 1 should be moved - addReady(_ready.bucket(2)); - addFrozen(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 1 - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - activateBucket(_ready.bucket(1)); - remFrozen(_ready.bucket(1)); - _bmj->scanAndMove(0, 3); // bucket 1 thawed but active - EXPECT_EQ(0u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - - deactivateBucket(_ready.bucket(1)); - _bmj->scanAndMove(0, 3); // handle de-activated bucket 1 - EXPECT_EQ(3u, docsMoved().size()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, ready_bucket_not_moved_to_not_ready_if_node_is_marked_as_retired) -{ - _calc->setNodeRetired(true); - // Bucket 2 would be moved from ready to not ready in a non-retired case, but not when retired. - addReady(_ready.bucket(1)); - _bmj->scanAndMove(4, 3); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); -} - -// Technically this should never happen since a retired node is never in the ideal state, -// but test this case for the sake of completion. -TEST_F(ControllerFixture, inactive_not_ready_bucket_not_moved_to_ready_if_node_is_marked_as_retired) -{ - _calc->setNodeRetired(true); - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(3)); - _bmj->scanAndMove(4, 3); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(0u, docsMoved().size()); -} - -TEST_F(ControllerFixture, explicitly_active_not_ready_bucket_can_be_moved_to_ready_even_if_node_is_marked_as_retired) -{ - _calc->setNodeRetired(true); - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - addReady(_notReady.bucket(3)); - activateBucket(_notReady.bucket(3)); - _bmj->scanAndMove(4, 3); - EXPECT_FALSE(_bmj->done()); - ASSERT_EQ(2u, docsMoved().size()); - assertEqual(_notReady.bucket(3), _notReady.docs(3)[0], 2, 1, docsMoved()[0]); - assertEqual(_notReady.bucket(3), _notReady.docs(3)[1], 2, 1, docsMoved()[1]); - ASSERT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]); -} - -TEST_F(ControllerFixture, require_that_notifyCreateBucket_causes_bucket_to_be_reconsidered_by_job) -{ - EXPECT_FALSE(_bmj->done()); - addReady(_ready.bucket(1)); - addReady(_ready.bucket(2)); - runLoop(); - EXPECT_TRUE(_bmj->done()); - EXPECT_TRUE(docsMoved().empty()); - EXPECT_TRUE(bucketsModified().empty()); - addReady(_notReady.bucket(3)); // bucket 3 now ready, no notify - EXPECT_TRUE(_bmj->done()); // move job still believes work done - _bmj->notifyCreateBucket(_bucketDB->takeGuard(), _notReady.bucket(3)); // reconsider bucket 3 - EXPECT_FALSE(_bmj->done()); - runLoop(); - EXPECT_TRUE(_bmj->done()); - EXPECT_EQ(1u, bucketsModified().size()); - EXPECT_EQ(2u, docsMoved().size()); -} - -struct ResourceLimitControllerFixture : public ControllerFixture -{ - ResourceLimitControllerFixture(double resourceLimitFactor = RESOURCE_LIMIT_FACTOR) : - ControllerFixture(BlockableMaintenanceJobConfig(resourceLimitFactor, MAX_OUTSTANDING_OPS)) - {} - - void testJobStopping(DiskMemUsageState blockingUsageState) { - // Bucket 1 should be moved - addReady(_ready.bucket(2)); - // Note: This depends on _bmj->run() moving max 1 documents - EXPECT_TRUE(!_bmj->run()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - // Notify that we've over limit - _diskMemUsageNotifier.notify(blockingUsageState); - EXPECT_TRUE(_bmj->run()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - // Notify that we've under limit - _diskMemUsageNotifier.notify(DiskMemUsageState()); - EXPECT_TRUE(!_bmj->run()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - } - - void testJobNotStopping(DiskMemUsageState blockingUsageState) { - // Bucket 1 should be moved - addReady(_ready.bucket(2)); - // Note: This depends on _bmj->run() moving max 1 documents - EXPECT_TRUE(!_bmj->run()); - EXPECT_EQ(1u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - // Notify that we've over limit, but not over adjusted limit - _diskMemUsageNotifier.notify(blockingUsageState); - EXPECT_TRUE(!_bmj->run()); - EXPECT_EQ(2u, docsMoved().size()); - EXPECT_EQ(0u, bucketsModified().size()); - } -}; - -struct ResourceLimitControllerFixture_1_2 : public ResourceLimitControllerFixture { - ResourceLimitControllerFixture_1_2() : ResourceLimitControllerFixture(1.2) {} -}; - -TEST_F(ResourceLimitControllerFixture, require_that_bucket_move_stops_when_disk_limit_is_reached) -{ - testJobStopping(DiskMemUsageState(ResourceUsageState(0.7, 0.8), ResourceUsageState())); -} - -TEST_F(ResourceLimitControllerFixture, require_that_bucket_move_stops_when_memory_limit_is_reached) -{ - testJobStopping(DiskMemUsageState(ResourceUsageState(), ResourceUsageState(0.7, 0.8))); -} - -TEST_F(ResourceLimitControllerFixture_1_2, require_that_bucket_move_uses_resource_limit_factor_for_disk_resource_limit) -{ - testJobNotStopping(DiskMemUsageState(ResourceUsageState(0.7, 0.8), ResourceUsageState())); -} - -TEST_F(ResourceLimitControllerFixture_1_2, require_that_bucket_move_uses_resource_limit_factor_for_memory_resource_limit) -{ - testJobNotStopping(DiskMemUsageState(ResourceUsageState(), ResourceUsageState(0.7, 0.8))); -} - -struct MaxOutstandingMoveOpsFixture : public ControllerFixture -{ - MaxOutstandingMoveOpsFixture(uint32_t maxOutstandingOps) : - ControllerFixture(BlockableMaintenanceJobConfig(RESOURCE_LIMIT_FACTOR, maxOutstandingOps)) - { - // Bucket 1 should be moved from ready -> notready - addReady(_ready.bucket(2)); - } - - void assertRunToBlocked() { - EXPECT_TRUE(_bmj->run()); // job becomes blocked as max outstanding limit is reached - EXPECT_FALSE(_bmj->done()); - EXPECT_TRUE(_bmj->isBlocked()); - EXPECT_TRUE(_bmj->isBlocked(BlockedReason::OUTSTANDING_OPS)); - } - void assertRunToNotBlocked() { - EXPECT_FALSE(_bmj->run()); - EXPECT_FALSE(_bmj->done()); - EXPECT_FALSE(_bmj->isBlocked()); - } - void assertRunToFinished() { - EXPECT_TRUE(_bmj->run()); - EXPECT_TRUE(_bmj->done()); - EXPECT_FALSE(_bmj->isBlocked()); - } - void assertDocsMoved(uint32_t expDocsMovedCnt, uint32_t expMoveContextsCnt) { - EXPECT_EQ(expDocsMovedCnt, docsMoved().size()); - EXPECT_EQ(expMoveContextsCnt, _moveHandler._moveDoneContexts.size()); - } - void unblockJob(uint32_t expRunnerCnt) { - _moveHandler.clearMoveDoneContexts(); // unblocks job and try to execute it via runner - EXPECT_EQ(expRunnerCnt, _runner.runCount); - EXPECT_FALSE(_bmj->isBlocked()); - } -}; - -struct MaxOutstandingMoveOpsFixture_1 : public MaxOutstandingMoveOpsFixture { - MaxOutstandingMoveOpsFixture_1() : MaxOutstandingMoveOpsFixture(1) {} -}; - -struct MaxOutstandingMoveOpsFixture_2 : public MaxOutstandingMoveOpsFixture { - MaxOutstandingMoveOpsFixture_2() : MaxOutstandingMoveOpsFixture(2) {} -}; - -TEST_F(MaxOutstandingMoveOpsFixture_1, require_that_bucket_move_job_is_blocked_if_it_has_too_many_outstanding_move_operations__max_1) -{ - assertRunToBlocked(); - assertDocsMoved(1, 1); - assertRunToBlocked(); - assertDocsMoved(1, 1); - - unblockJob(1); - assertRunToBlocked(); - assertDocsMoved(2, 1); - - unblockJob(2); - assertRunToBlocked(); - assertDocsMoved(3, 1); - - unblockJob(3); - assertRunToFinished(); - assertDocsMoved(3, 0); -} - -TEST_F(MaxOutstandingMoveOpsFixture_2, require_that_bucket_move_job_is_blocked_if_it_has_too_many_outstanding_move_operations_max_2) -{ - assertRunToNotBlocked(); - assertDocsMoved(1, 1); - - assertRunToBlocked(); - assertDocsMoved(2, 2); - - unblockJob(1); - assertRunToNotBlocked(); - assertDocsMoved(3, 1); - - assertRunToFinished(); - assertDocsMoved(3, 1); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 688dd963f61..f1353a30887 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -4,12 +4,12 @@ using BlockedReason = IBlockableMaintenanceJob::BlockedReason; -TEST_P(JobTest, handler_name_is_used_as_part_of_job_name) +TEST_F(JobTest, handler_name_is_used_as_part_of_job_name) { EXPECT_EQ("lid_space_compaction.myhandler", _job->getName()); } -TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) +TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) { // 20% bloat < 30% allowed bloat addStats(10, {1,3,4,5,6,7,9}, {{9,2}}); @@ -17,7 +17,7 @@ TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit) assertNoWorkDone(); } -TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) +TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) { init(3, 0.1); // 20% bloat >= 10% allowed bloat BUT lid bloat (2) < allowed lid bloat (3) @@ -26,7 +26,7 @@ TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit) assertNoWorkDone(); } -TEST_P(JobTest, no_move_operation_is_created_and_compaction_is_initiated) +TEST_F(JobTest, no_move_operation_is_created_and_compaction_is_initiated) { // no documents to move: lowestFreeLid(7) > highestUsedLid(6) addStats(10, {1,2,3,4,5,6}, {{6,7}}); @@ -36,14 +36,14 @@ TEST_P(JobTest, no_move_operation_is_created_and_compaction_is_initiated) assertJobContext(0, 0, 0, 7, 1); } -TEST_P(JobTest, one_move_operation_is_created_and_compaction_is_initiated) +TEST_F(JobTest, one_move_operation_is_created_and_compaction_is_initiated) { setupOneDocumentToCompact(); EXPECT_FALSE(run()); // scan assertOneDocumentCompacted(); } -TEST_P(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed) +TEST_F(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed) { setupThreeDocumentsToCompact(); EXPECT_FALSE(run()); @@ -56,29 +56,7 @@ TEST_P(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_ar assertJobContext(4, 7, 3, 7, 1); } -TEST_P(JobTest, job_is_blocked_if_trying_to_move_document_for_frozen_bucket) -{ - //TODO Remove test once we no longer use the frozen concept. - if ( ! useBucketDB() ) { - _frozenHandler._bucket = BUCKET_ID_1; - EXPECT_FALSE(_job->isBlocked()); - addStats(10, {1, 3, 4, 5, 6, 9}, {{9, 2}}); // 30% bloat: try to move 9 -> 2 - addStats(0, 0, 0, 0); - - EXPECT_TRUE(run()); // bucket frozen - assertNoWorkDone(); - EXPECT_TRUE(_job->isBlocked()); - - _frozenHandler._bucket = BUCKET_ID_2; - _job->unBlock(BlockedReason::FROZEN_BUCKET); - - EXPECT_FALSE(run()); // unblocked - assertJobContext(2, 9, 1, 0, 0); - EXPECT_FALSE(_job->isBlocked()); - } -} - -TEST_P(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) +TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR); addMultiStats(10, {{1,3,4,5,6,9},{1,2,4,5,6,8}}, @@ -98,7 +76,7 @@ TEST_P(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) assertJobContext(3, 8, 2, 7, 1); } -TEST_P(JobTest, held_lid_is_not_considered_free_and_blocks_job) +TEST_F(JobTest, held_lid_is_not_considered_free_and_blocks_job) { // Lid 1 on hold or pendingHold, i.e. neither free nor used. addMultiStats(3, {{2}}, {{2, 3}}); @@ -106,7 +84,7 @@ TEST_P(JobTest, held_lid_is_not_considered_free_and_blocks_job) assertNoWorkDone(); } -TEST_P(JobTest, held_lid_is_not_considered_free_with_only_compact) +TEST_F(JobTest, held_lid_is_not_considered_free_with_only_compact) { // Lid 1 on hold or pendingHold, i.e. neither free nor used. addMultiStats(10, {{2}}, {{2, 3}}); @@ -116,7 +94,7 @@ TEST_P(JobTest, held_lid_is_not_considered_free_with_only_compact) assertJobContext(0, 0, 0, 3, 1); } -TEST_P(JobTest, held_lids_are_not_considered_free_with_one_move) +TEST_F(JobTest, held_lids_are_not_considered_free_with_one_move) { // Lids 1,2,3 on hold or pendingHold, i.e. neither free nor used. addMultiStats(10, {{5}}, {{5, 4}, {4, 5}}); @@ -126,7 +104,7 @@ TEST_P(JobTest, held_lids_are_not_considered_free_with_one_move) assertJobContext(4, 5, 1, 5, 1); } -TEST_P(JobTest, resource_starvation_blocks_lid_space_compaction) +TEST_F(JobTest, resource_starvation_blocks_lid_space_compaction) { setupOneDocumentToCompact(); _diskMemUsageNotifier.notify({{100, 0}, {100, 101}}); @@ -134,7 +112,7 @@ TEST_P(JobTest, resource_starvation_blocks_lid_space_compaction) assertNoWorkDone(); } -TEST_P(JobTest, ending_resource_starvation_resumes_lid_space_compaction) +TEST_F(JobTest, ending_resource_starvation_resumes_lid_space_compaction) { setupOneDocumentToCompact(); _diskMemUsageNotifier.notify({{100, 0}, {100, 101}}); @@ -144,7 +122,7 @@ TEST_P(JobTest, ending_resource_starvation_resumes_lid_space_compaction) assertOneDocumentCompacted(); } -TEST_P(JobTest, resource_limit_factor_adjusts_limit) +TEST_F(JobTest, resource_limit_factor_adjusts_limit) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 1.05); setupOneDocumentToCompact(); @@ -153,21 +131,21 @@ TEST_P(JobTest, resource_limit_factor_adjusts_limit) assertOneDocumentCompacted(); } -TEST_P(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs) +TEST_F(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs) { init_with_interval(301s); EXPECT_EQ(300s, _job->getDelay()); EXPECT_EQ(301s, _job->getInterval()); } -TEST_P(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs) +TEST_F(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs) { init_with_interval(299s); EXPECT_EQ(299s, _job->getDelay()); EXPECT_EQ(299s, _job->getInterval()); } -TEST_P(JobTest, job_is_disabled_when_node_is_retired) +TEST_F(JobTest, job_is_disabled_when_node_is_retired) { init_with_node_retired(true); setupOneDocumentToCompact(); @@ -175,7 +153,7 @@ TEST_P(JobTest, job_is_disabled_when_node_is_retired) assertNoWorkDone(); } -TEST_P(JobTest, job_is_disabled_when_node_becomes_retired) +TEST_F(JobTest, job_is_disabled_when_node_becomes_retired) { init_with_node_retired(false); setupOneDocumentToCompact(); @@ -184,7 +162,7 @@ TEST_P(JobTest, job_is_disabled_when_node_becomes_retired) assertNoWorkDone(); } -TEST_P(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) +TEST_F(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) { init_with_node_retired(true); setupOneDocumentToCompact(); @@ -194,7 +172,7 @@ TEST_P(JobTest, job_is_re_enabled_when_node_is_no_longer_retired) assertOneDocumentCompacted(); } -TEST_P(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker) +TEST_F(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker) { auto& remove_batch_tracker = _handler->_rm_listener->get_remove_batch_tracker(); EXPECT_EQ(vespalib::from_s(21.0), remove_batch_tracker.get_time_budget_per_op()); @@ -205,38 +183,38 @@ TEST_P(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rat EXPECT_EQ(vespalib::from_s(20.0), remove_tracker.get_time_budget_window()); } -TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing) +TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing) { job_is_disabled_while_remove_ops_are_ongoing(true); } -TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts) +TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts) { job_becomes_disabled_if_remove_ops_starts(true); } -TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing) +TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing) { job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(true); } -TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing) +TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing) { job_is_disabled_while_remove_ops_are_ongoing(false); } -TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start) +TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start) { job_becomes_disabled_if_remove_ops_starts(false); } -TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing) +TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing) { job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(false); } -TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1) +TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1) { init(1); setupThreeDocumentsToCompact(); @@ -259,7 +237,7 @@ TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move assertJobContext(4, 7, 3, 7, 1); } -TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2) +TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2) { init(2); setupThreeDocumentsToCompact(); @@ -277,8 +255,4 @@ TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move sync(); } -VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobTest, ::testing::Values(false, true)); -VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobDisabledByRemoveOpsTest, ::testing::Values(false, true)); -VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, MaxOutstandingJobTest, ::testing::Values(false, true)); - GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp index d394769c0ee..73cd6a8e6be 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "lid_space_jobtest.h" -#include <vespa/searchcore/proton/server/lid_space_compaction_job.h> #include <vespa/searchcore/proton/server/lid_space_compaction_job_take2.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> @@ -33,7 +32,6 @@ JobTestBase::JobTestBase() _diskMemUsageNotifier(), _handler(), _storer(), - _frozenHandler(), _job() { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, MAX_OUTSTANDING_MOVE_OPS); @@ -47,23 +45,18 @@ JobTestBase::init(uint32_t allowedLidBloat, bool nodeRetired, uint32_t maxOutstandingMoveOps) { - _handler = std::make_shared<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, useBucketDB()); + _handler = std::make_shared<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, true); DocumentDBLidSpaceCompactionConfig compactCfg(interval, allowedLidBloat, allowedLidBloatFactor, - REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, false, useBucketDB()); + REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, false, true); BlockableMaintenanceJobConfig blockableCfg(resourceLimitFactor, maxOutstandingMoveOps); _job.reset(); - if (useBucketDB()) { - _singleExecutor = std::make_unique<vespalib::ThreadStackExecutor>(1, 0x10000); - _master = std::make_unique<proton::ExecutorThreadService> (*_singleExecutor); - _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4); - _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor, - _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired, - document::BucketSpace::placeHolder()); - } else { - _job = std::make_shared<LidSpaceCompactionJob>(compactCfg, _handler, _storer, _frozenHandler, _diskMemUsageNotifier, - blockableCfg, _clusterStateHandler, nodeRetired); - } + _singleExecutor = std::make_unique<vespalib::ThreadStackExecutor>(1, 0x10000); + _master = std::make_unique<proton::ExecutorThreadService> (*_singleExecutor); + _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4); + _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor, + _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired, + document::BucketSpace::placeHolder()); } void diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h index dde48a0a620..747e5c9faca 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h @@ -8,7 +8,7 @@ #include <vespa/vespalib/gtest/gtest.h> namespace storage::spi::dummy { class DummyBucketExecutor; } -struct JobTestBase : public ::testing::TestWithParam<bool> { +struct JobTestBase : public ::testing::Test { MonitoredRefCount _refCount; test::ClusterStateHandler _clusterStateHandler; test::DiskMemUsageNotifier _diskMemUsageNotifier; @@ -17,7 +17,6 @@ struct JobTestBase : public ::testing::TestWithParam<bool> { std::unique_ptr<searchcorespi::index::IThreadService> _master; std::shared_ptr<MyHandler> _handler; MyStorer _storer; - MyFrozenBucketHandler _frozenHandler; std::shared_ptr<BlockableMaintenanceJob> _job; JobTestBase(); ~JobTestBase() override; @@ -51,7 +50,6 @@ struct JobTestBase : public ::testing::TestWithParam<bool> { void assertOneDocumentCompacted(); JobTestBase &setupThreeDocumentsToCompact(); void sync() const; - bool useBucketDB() const { return GetParam(); } }; struct JobTest : public JobTestBase { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index b0c46dbd789..f401d23b8ad 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -929,57 +929,6 @@ MaintenanceControllerFixture::removeDocs(const test::UserDocuments &docs, Timest } } -TEST_F("require that bucket move controller is active", MaintenanceControllerFixture) -{ - f._builder.createDocs(1, 1, 4); // 3 docs - f._builder.createDocs(2, 4, 6); // 2 docs - test::UserDocuments readyDocs(f._builder.getDocs()); - BucketId bucketId1(readyDocs.getBucket(1)); - BucketId bucketId2(readyDocs.getBucket(2)); - f.insertDocs(readyDocs, f._ready); - f._builder.clearDocs(); - f._builder.createDocs(3, 1, 3); // 2 docs - f._builder.createDocs(4, 3, 6); // 3 docs - test::UserDocuments notReadyDocs(f._builder.getDocs()); - BucketId bucketId4(notReadyDocs.getBucket(4)); - f.insertDocs(notReadyDocs, f._notReady); - f._builder.clearDocs(); - f.notifyClusterStateChanged(); - EXPECT_TRUE(f._executor.isIdle()); - EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(5u, f._ready.getDocumentCount()); - EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); - f.startMaintenance(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(0u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(0u, f._ready.getDocumentCount()); - EXPECT_EQUAL(10u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(10u, f._notReady.getDocumentCount()); - f._calc->addReady(bucketId1); - f.notifyClusterStateChanged(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(3u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(3u, f._ready.getDocumentCount()); - EXPECT_EQUAL(7u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(7u, f._notReady.getDocumentCount()); - MyFrozenBucket::UP frozen2(new MyFrozenBucket(f._mc, bucketId2)); - f._calc->addReady(bucketId2); - f._calc->addReady(bucketId4); - f.notifyClusterStateChanged(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(6u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(6u, f._ready.getDocumentCount()); - EXPECT_EQUAL(4u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(4u, f._notReady.getDocumentCount()); - frozen2.reset(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(8u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(8u, f._ready.getDocumentCount()); - EXPECT_EQUAL(2u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(2u, f._notReady.getDocumentCount()); -} - TEST_F("require that document pruner is active", MaintenanceControllerFixture) { uint64_t tshz = 1000000; @@ -1054,74 +1003,6 @@ TEST_F("require that periodic session prunings are scheduled", ASSERT_TRUE(f._gsp.isInvoked); } -TEST_F("require that active bucket is not moved until de-activated", MaintenanceControllerFixture) -{ - f._builder.createDocs(1, 1, 4); // 3 docs - f._builder.createDocs(2, 4, 6); // 2 docs - test::UserDocuments readyDocs(f._builder.getDocs()); - f.insertDocs(readyDocs, f._ready); - f._builder.clearDocs(); - f._builder.createDocs(3, 1, 3); // 2 docs - f._builder.createDocs(4, 3, 6); // 3 docs - test::UserDocuments notReadyDocs(f._builder.getDocs()); - f.insertDocs(notReadyDocs, f._notReady); - f._builder.clearDocs(); - - // bucket 1 (active) should be moved from ready to not ready according to cluster state - f._calc->addReady(readyDocs.getBucket(2)); - f._ready.setBucketState(readyDocs.getBucket(1), true); - - f.notifyClusterStateChanged(); - EXPECT_TRUE(f._executor.isIdle()); - EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(5u, f._ready.getDocumentCount()); - EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); - - f.startMaintenance(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(5u, f._ready.getDocumentCount()); - EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); - - // de-activate bucket 1 - f._ready.setBucketState(readyDocs.getBucket(1), false); - f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(2u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(2u, f._ready.getDocumentCount()); - EXPECT_EQUAL(8u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(8u, f._notReady.getDocumentCount()); - - // re-activate bucket 1 - f._ready.setBucketState(readyDocs.getBucket(1), true); - f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(5u, f._ready.getDocumentCount()); - EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); - - // de-activate bucket 1 - f._ready.setBucketState(readyDocs.getBucket(1), false); - f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(2u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(2u, f._ready.getDocumentCount()); - EXPECT_EQUAL(8u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(8u, f._notReady.getDocumentCount()); - - // re-activate bucket 1 - f._ready.setBucketState(readyDocs.getBucket(1), true); - f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); - EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); - EXPECT_EQUAL(5u, f._ready.getDocumentCount()); - EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); - EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); -} - TEST_F("require that a simple maintenance job is executed", MaintenanceControllerFixture) { auto job = std::make_unique<MySimpleJob>(200ms, 200ms, 3); |