aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2021-05-03 12:34:45 +0000
committerHenning Baldersheim <balder@yahoo-inc.com>2021-05-03 12:55:20 +0000
commita6d1d5d762265e159fdf738c709b8296af29f4ce (patch)
treee6d784079735c65a2908bafdafd3f0bd538712d4
parent54164220f1d224808aa86bf5777eaef837d61034 (diff)
- GC legacy lidspace compaction job and bucket move job.
- GC duplicate tests. They are tested more thoroughly in the bucket mover tests.
-rw-r--r--searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt12
-rw-r--r--searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp906
-rw-r--r--searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp80
-rw-r--r--searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp23
-rw-r--r--searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h4
-rw-r--r--searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp119
6 files changed, 36 insertions, 1108 deletions
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt
index d11da09b737..88bd84cbd16 100644
--- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt
@@ -5,18 +5,6 @@ vespa_add_library(searchcore_bucketmover_test STATIC
bucketmover_common.cpp
)
-vespa_add_executable(searchcore_documentbucketmover_test_app TEST
- SOURCES
- documentbucketmover_test.cpp
- DEPENDS
- searchcore_bucketmover_test
- searchcore_test
- searchcore_server
- searchcore_feedoperation
- GTest::GTest
-)
-vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app)
-
vespa_add_executable(searchcore_documentbucketmover_v2_test_app TEST
SOURCES
documentbucketmover_v2_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp
deleted file mode 100644
index bb7180dadf1..00000000000
--- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp
+++ /dev/null
@@ -1,906 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "bucketmover_common.h"
-#include <vespa/searchcore/proton/server/bucketmovejob.h>
-#include <vespa/searchcore/proton/server/document_db_maintenance_config.h>
-#include <vespa/vespalib/gtest/gtest.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP("document_bucket_mover_test");
-
-using namespace proton;
-using namespace proton::move::test;
-using document::BucketId;
-using document::test::makeBucketSpace;
-using proton::bucketdb::BucketCreateNotifier;
-using storage::spi::BucketInfo;
-using BlockedReason = IBlockableMaintenanceJob::BlockedReason;
-using MoveOperationVector = std::vector<MoveOperation>;
-
-struct MyFrozenBucketHandler : public IFrozenBucketHandler
-{
- std::set<BucketId> _frozen;
- std::set<IBucketFreezeListener *> _listeners;
-
- MyFrozenBucketHandler()
- : IFrozenBucketHandler(),
- _frozen(),
- _listeners()
- {
- }
-
- ~MyFrozenBucketHandler() override {
- assert(_listeners.empty());
- }
-
- MyFrozenBucketHandler &addFrozen(const BucketId &bucket) {
- _frozen.insert(bucket);
- return *this;
- }
- MyFrozenBucketHandler &remFrozen(const BucketId &bucket) {
- _frozen.erase(bucket);
- for (auto &listener : _listeners) {
- listener->notifyThawedBucket(bucket);
- }
- return *this;
- }
- void addListener(IBucketFreezeListener *listener) override {
- _listeners.insert(listener);
- }
- void removeListener(IBucketFreezeListener *listener) override {
- _listeners.erase(listener);
- }
-
- ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override {
- return (_frozen.count(bucket) != 0)
- ? ExclusiveBucketGuard::UP()
- : std::make_unique<ExclusiveBucketGuard>(bucket);
- }
-};
-
-struct ControllerFixtureBase : public ::testing::Test
-{
- test::UserDocumentsBuilder _builder;
- test::BucketStateCalculator::SP _calc;
- test::ClusterStateHandler _clusterStateHandler;
- test::BucketHandler _bucketHandler;
- MyBucketModifiedHandler _modifiedHandler;
- std::shared_ptr<bucketdb::BucketDBOwner> _bucketDB;
- MyMoveHandler _moveHandler;
- MySubDb _ready;
- MySubDb _notReady;
- MyFrozenBucketHandler _fbh;
- BucketCreateNotifier _bucketCreateNotifier;
- test::DiskMemUsageNotifier _diskMemUsageNotifier;
- std::shared_ptr<BucketMoveJob> _bmj;
- MyCountJobRunner _runner;
- ControllerFixtureBase(const BlockableMaintenanceJobConfig &blockableConfig, bool storeMoveDoneContexts);
- ~ControllerFixtureBase();
- ControllerFixtureBase &addReady(const BucketId &bucket) {
- _calc->addReady(bucket);
- return *this;
- }
- ControllerFixtureBase &remReady(const BucketId &bucket) {
- _calc->remReady(bucket);
- return *this;
- }
- ControllerFixtureBase &changeCalc() {
- _calc->resetAsked();
- _moveHandler.reset();
- _modifiedHandler.reset();
- _clusterStateHandler.notifyClusterStateChanged(_calc);
- return *this;
- }
- ControllerFixtureBase &addFrozen(const BucketId &bucket) {
- _fbh.addFrozen(bucket);
- return *this;
- }
- ControllerFixtureBase &remFrozen(const BucketId &bucket) {
- _fbh.remFrozen(bucket);
- _bmj->notifyThawedBucket(bucket);
- return *this;
- }
- ControllerFixtureBase &activateBucket(const BucketId &bucket) {
- _ready.setBucketState(bucket, true);
- _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::ACTIVE);
- return *this;
- }
- ControllerFixtureBase &deactivateBucket(const BucketId &bucket) {
- _ready.setBucketState(bucket, false);
- _bucketHandler.notifyBucketStateChanged(bucket, BucketInfo::ActiveState::NOT_ACTIVE);
- return *this;
- }
- const MoveOperationVector &docsMoved() const {
- return _moveHandler._moves;
- }
- const std::vector<BucketId> &bucketsModified() const {
- return _modifiedHandler._modified;
- }
- const BucketId::List &calcAsked() const {
- return _calc->asked();
- }
- void runLoop() {
- while (!_bmj->isBlocked() && !_bmj->run()) {
- }
- }
-};
-
-ControllerFixtureBase::ControllerFixtureBase(const BlockableMaintenanceJobConfig &blockableConfig, bool storeMoveDoneContexts)
- : _builder(),
- _calc(std::make_shared<test::BucketStateCalculator>()),
- _bucketHandler(),
- _modifiedHandler(),
- _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()),
- _moveHandler(*_bucketDB, storeMoveDoneContexts),
- _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY),
- _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY),
- _fbh(),
- _bucketCreateNotifier(),
- _diskMemUsageNotifier(),
- _bmj(std::make_shared<BucketMoveJob>(_calc, _moveHandler, _modifiedHandler, _ready._subDb, _notReady._subDb,
- _fbh, _bucketCreateNotifier, _clusterStateHandler, _bucketHandler,
- _diskMemUsageNotifier, blockableConfig, "test", makeBucketSpace())),
- _runner(*_bmj)
-{
-}
-
-ControllerFixtureBase::~ControllerFixtureBase() = default;
-constexpr double RESOURCE_LIMIT_FACTOR = 1.0;
-constexpr uint32_t MAX_OUTSTANDING_OPS = 10;
-const BlockableMaintenanceJobConfig BLOCKABLE_CONFIG(RESOURCE_LIMIT_FACTOR, MAX_OUTSTANDING_OPS);
-
-struct ControllerFixture : public ControllerFixtureBase
-{
- ControllerFixture(const BlockableMaintenanceJobConfig &blockableConfig = BLOCKABLE_CONFIG)
- : ControllerFixtureBase(blockableConfig, blockableConfig.getMaxOutstandingMoveOps() != MAX_OUTSTANDING_OPS)
- {
- _builder.createDocs(1, 1, 4); // 3 docs
- _builder.createDocs(2, 4, 6); // 2 docs
- _ready.insertDocs(_builder.getDocs());
- _builder.clearDocs();
- _builder.createDocs(3, 1, 3); // 2 docs
- _builder.createDocs(4, 3, 6); // 3 docs
- _notReady.insertDocs(_builder.getDocs());
- }
-};
-
-struct OnlyReadyControllerFixture : public ControllerFixtureBase
-{
- OnlyReadyControllerFixture() : ControllerFixtureBase(BLOCKABLE_CONFIG, false)
- {
- _builder.createDocs(1, 1, 2); // 1 docs
- _builder.createDocs(2, 2, 4); // 2 docs
- _builder.createDocs(3, 4, 7); // 3 docs
- _builder.createDocs(4, 7, 11); // 4 docs
- _ready.insertDocs(_builder.getDocs());
- }
-};
-
-TEST_F(ControllerFixture, require_that_nothing_is_moved_if_bucket_state_says_so)
-{
- EXPECT_FALSE(_bmj->done());
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- _bmj->scanAndMove(4, 3);
- EXPECT_TRUE(_bmj->done());
- EXPECT_TRUE(docsMoved().empty());
- EXPECT_TRUE(bucketsModified().empty());
-}
-
-TEST_F(ControllerFixture, require_that_not_ready_bucket_is_moved_to_ready_if_bucket_state_says_so)
-{
- // bucket 4 should be moved
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(4));
- _bmj->scanAndMove(4, 3);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[0]);
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[1]);
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[2]);
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(4), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_ready_bucket_is_moved_to_not_ready_if_bucket_state_says_so)
-{
- // bucket 2 should be moved
- addReady(_ready.bucket(1));
- _bmj->scanAndMove(4, 3);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, docsMoved().size());
- assertEqual(_ready.bucket(2), _ready.docs(2)[0], 1, 2, docsMoved()[0]);
- assertEqual(_ready.bucket(2), _ready.docs(2)[1], 1, 2, docsMoved()[1]);
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_maxBucketsToScan_is_taken_into_consideration_between_not_ready_and_ready_scanning)
-{
- // bucket 4 should moved (last bucket)
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(4));
-
- // buckets 1, 2, and 3 considered
- _bmj->scanAndMove(3, 3);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- // move bucket 4
- _bmj->scanAndMove(1, 4);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[0]);
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[1]);
- assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[2]);
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(4), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_we_move_buckets_in_several_steps)
-{
- // bucket 2, 3, and 4 should be moved
- addReady(_ready.bucket(1));
- addReady(_notReady.bucket(3));
- addReady(_notReady.bucket(4));
-
- // consider move bucket 1
- _bmj->scanAndMove(1, 2);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- // move bucket 2, docs 1,2
- _bmj->scanAndMove(1, 2);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_TRUE(assertEqual(_ready.bucket(2), _ready.docs(2)[0], 1, 2, docsMoved()[0]));
- EXPECT_TRUE(assertEqual(_ready.bucket(2), _ready.docs(2)[1], 1, 2, docsMoved()[1]));
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]);
-
- // move bucket 3, docs 1,2
- _bmj->scanAndMove(1, 2);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(4u, docsMoved().size());
- EXPECT_TRUE(assertEqual(_notReady.bucket(3), _notReady.docs(3)[0], 2, 1, docsMoved()[2]));
- EXPECT_TRUE(assertEqual(_notReady.bucket(3), _notReady.docs(3)[1], 2, 1, docsMoved()[3]));
- EXPECT_EQ(2u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(3), bucketsModified()[1]);
-
- // move bucket 4, docs 1,2
- _bmj->scanAndMove(1, 2);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(6u, docsMoved().size());
- EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[0], 2, 1, docsMoved()[4]));
- EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[1], 2, 1, docsMoved()[5]));
- EXPECT_EQ(2u, bucketsModified().size());
-
- // move bucket 4, docs 3
- _bmj->scanAndMove(1, 2);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(7u, docsMoved().size());
- EXPECT_TRUE(assertEqual(_notReady.bucket(4), _notReady.docs(4)[2], 2, 1, docsMoved()[6]));
- EXPECT_EQ(3u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(4), bucketsModified()[2]);
-}
-
-TEST_F(ControllerFixture, require_that_we_can_change_calculator_and_continue_scanning_where_we_left_off)
-{
- // no buckets should move
- // original scan sequence is bucket1, bucket2, bucket3, bucket4
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
-
- // start with bucket2
- _bmj->scanAndMove(1, 0);
- changeCalc();
- _bmj->scanAndMove(5, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(2), calcAsked()[0]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]);
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[3]);
-
- // start with bucket3
- changeCalc();
- _bmj->scanAndMove(2, 0);
- changeCalc();
- _bmj->scanAndMove(5, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[0]);
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[1]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[2]);
- EXPECT_EQ(_ready.bucket(2), calcAsked()[3]);
-
- // start with bucket4
- changeCalc();
- _bmj->scanAndMove(3, 0);
- changeCalc();
- _bmj->scanAndMove(5, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[0]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[1]);
- EXPECT_EQ(_ready.bucket(2), calcAsked()[2]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[3]);
-
- // start with bucket1
- changeCalc();
- _bmj->scanAndMove(5, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
- EXPECT_EQ(_ready.bucket(2), calcAsked()[1]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[2]);
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[3]);
-
- // change calc in second pass
- changeCalc();
- _bmj->scanAndMove(3, 0);
- changeCalc();
- _bmj->scanAndMove(2, 0);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[0]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[1]);
- changeCalc();
- _bmj->scanAndMove(5, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(2), calcAsked()[0]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]);
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[3]);
-
- // check 1 bucket at a time, start with bucket2
- changeCalc();
- _bmj->scanAndMove(1, 0);
- changeCalc();
- _bmj->scanAndMove(1, 0);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(2), calcAsked()[0]);
- _bmj->scanAndMove(1, 0);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]);
- _bmj->scanAndMove(1, 0);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(3u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]);
- _bmj->scanAndMove(1, 0);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[3]);
-}
-
-TEST_F(ControllerFixture, require_that_current_bucket_moving_is_cancelled_when_we_change_calculator)
-{
- // bucket 1 should be moved
- addReady(_ready.bucket(2));
- _bmj->scanAndMove(3, 1);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(1u, calcAsked().size());
- changeCalc(); // Not cancelled, bucket 1 still moving to notReady
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
- _calc->resetAsked();
- _bmj->scanAndMove(2, 1);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, calcAsked().size());
- addReady(_ready.bucket(1));
- changeCalc(); // cancelled, bucket 1 no longer moving to notReady
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
- _calc->resetAsked();
- remReady(_ready.bucket(1));
- changeCalc(); // not cancelled. No active bucket move
- EXPECT_EQ(0u, calcAsked().size());
- _calc->resetAsked();
- _bmj->scanAndMove(2, 1);
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(2u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(2), calcAsked()[0]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[1]);
- _bmj->scanAndMove(2, 3);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[2]);
- EXPECT_EQ(_ready.bucket(1), calcAsked()[3]);
-}
-
-TEST_F(ControllerFixture, require_that_last_bucket_is_moved_before_reporting_done)
-{
- // bucket 4 should be moved
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(4));
- _bmj->scanAndMove(4, 1);
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(4u, calcAsked().size());
- _bmj->scanAndMove(0, 2);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(4u, calcAsked().size());
-}
-
-TEST_F(ControllerFixture, require_that_frozen_bucket_is_not_moved_until_thawed)
-{
- // bucket 1 should be moved but is frozen
- addReady(_ready.bucket(2));
- addFrozen(_ready.bucket(1));
- _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 1
- remFrozen(_ready.bucket(1));
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- _bmj->scanAndMove(0, 3); // move delayed and thawed bucket 1
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_thawed_bucket_is_moved_before_other_buckets)
-{
- // bucket 2 should be moved but is frozen.
- // bucket 3 & 4 should also be moved
- addReady(_ready.bucket(1));
- addReady(_notReady.bucket(3));
- addReady(_notReady.bucket(4));
- addFrozen(_ready.bucket(2));
- _bmj->scanAndMove(3, 2); // delay bucket 2, move bucket 3
- remFrozen(_ready.bucket(2));
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]);
- _bmj->scanAndMove(2, 2); // move thawed bucket 2
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(4u, docsMoved().size());
- EXPECT_EQ(2u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(2), bucketsModified()[1]);
- _bmj->scanAndMove(1, 4); // move bucket 4
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(7u, docsMoved().size());
- EXPECT_EQ(3u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(4), bucketsModified()[2]);
-}
-
-TEST_F(ControllerFixture, require_that_re_frozen_thawed_bucket_is_not_moved_until_re_thawed)
-{
- // bucket 1 should be moved but is re-frozen
- addReady(_ready.bucket(2));
- addFrozen(_ready.bucket(1));
- _bmj->scanAndMove(1, 0); // scan, delay frozen bucket 1
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
- remFrozen(_ready.bucket(1));
- addFrozen(_ready.bucket(1));
- _bmj->scanAndMove(1, 0); // scan, but nothing to move
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(3u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[1]);
- EXPECT_EQ(_ready.bucket(2), calcAsked()[2]);
- remFrozen(_ready.bucket(1));
- _bmj->scanAndMove(3, 4); // move delayed and thawed bucket 1
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]);
- EXPECT_EQ(4u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[3]);
- _bmj->scanAndMove(2, 0); // scan the rest
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(6u, calcAsked().size());
-}
-
-TEST_F(ControllerFixture, require_that_thawed_bucket_is_not_moved_if_new_calculator_does_not_say_so)
-{
- // bucket 3 should be moved
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(3));
- addFrozen(_notReady.bucket(3));
- _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 3
- remFrozen(_notReady.bucket(3));
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(4u, calcAsked().size());
- changeCalc();
- remReady(_notReady.bucket(3));
- _bmj->scanAndMove(0, 3); // consider delayed bucket 3
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_current_bucket_mover_is_cancelled_if_bucket_is_frozen)
-{
- // bucket 3 should be moved
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(3));
- _bmj->scanAndMove(3, 1); // move 1 doc from bucket 3
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(3u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
- EXPECT_EQ(_ready.bucket(2), calcAsked()[1]);
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[2]);
-
- addFrozen(_notReady.bucket(3));
- _bmj->scanAndMove(1, 3); // done scanning
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(3u, calcAsked().size());
-
- _bmj->scanAndMove(1, 3); // done scanning
- remFrozen(_notReady.bucket(3));
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(4u, calcAsked().size());
-
- EXPECT_EQ(_notReady.bucket(4), calcAsked()[3]);
- _bmj->scanAndMove(0, 2); // move all docs from bucket 3 again
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]);
- EXPECT_EQ(5u, calcAsked().size());
- EXPECT_EQ(_notReady.bucket(3), calcAsked()[4]);
-}
-
-TEST_F(ControllerFixture, require_that_current_bucket_mover_is_not_cancelled_if_another_bucket_is_frozen)
-{
- // bucket 3 and 4 should be moved
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(3));
- addReady(_notReady.bucket(4));
- _bmj->scanAndMove(3, 1); // move 1 doc from bucket 3
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(3u, calcAsked().size());
- addFrozen(_notReady.bucket(4));
- _bmj->scanAndMove(1, 2); // move rest of docs from bucket 3
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]);
- EXPECT_EQ(3u, calcAsked().size());
-}
-
-TEST_F(ControllerFixture, require_that_active_bucket_is_not_moved_from_ready_to_not_ready_until_being_not_active)
-{
- // bucket 1 should be moved but is active
- addReady(_ready.bucket(2));
- activateBucket(_ready.bucket(1));
- _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- deactivateBucket(_ready.bucket(1));
- EXPECT_FALSE(_bmj->done());
- _bmj->scanAndMove(0, 3); // move delayed and de-activated bucket 1
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]);
-}
-
-TEST_F(OnlyReadyControllerFixture, require_that_de_activated_bucket_is_moved_before_other_buckets)
-{
- // bucket 1, 2, 3 should be moved (but bucket 1 is active)
- addReady(_ready.bucket(4));
- activateBucket(_ready.bucket(1));
- _bmj->scanAndMove(2, 4); // delay bucket 1, move bucket 2
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]);
-
- deactivateBucket(_ready.bucket(1));
- _bmj->scanAndMove(2, 4); // move de-activated bucket 1
- EXPECT_FALSE(_bmj->done());
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(2u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[1]);
-
- _bmj->scanAndMove(2, 4); // move bucket 3
- // EXPECT_TRUE(_bmj->done()); // TODO(geirst): fix this
- EXPECT_EQ(6u, docsMoved().size());
- EXPECT_EQ(3u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(3), bucketsModified()[2]);
-}
-
-TEST_F(ControllerFixture, require_that_de_activated_bucket_is_not_moved_if_new_calculator_does_not_say_so)
-{
- // bucket 1 should be moved
- addReady(_ready.bucket(2));
- activateBucket(_ready.bucket(1));
- _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- deactivateBucket(_ready.bucket(1));
- addReady(_ready.bucket(1));
- changeCalc();
- _bmj->scanAndMove(0, 3); // consider delayed bucket 3
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- EXPECT_EQ(1u, calcAsked().size());
- EXPECT_EQ(_ready.bucket(1), calcAsked()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_de_activated_bucket_is_not_moved_if_frozen_as_well)
-{
- // bucket 1 should be moved
- addReady(_ready.bucket(2));
- activateBucket(_ready.bucket(1));
- _bmj->scanAndMove(4, 3); // scan all, delay active bucket 1
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- addFrozen(_ready.bucket(1));
- deactivateBucket(_ready.bucket(1));
- _bmj->scanAndMove(0, 3); // bucket 1 de-activated but frozen
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- remFrozen(_ready.bucket(1));
- _bmj->scanAndMove(0, 3); // handle thawed bucket 1
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_thawed_bucket_is_not_moved_if_active_as_well)
-{
- // bucket 1 should be moved
- addReady(_ready.bucket(2));
- addFrozen(_ready.bucket(1));
- _bmj->scanAndMove(4, 3); // scan all, delay frozen bucket 1
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- activateBucket(_ready.bucket(1));
- remFrozen(_ready.bucket(1));
- _bmj->scanAndMove(0, 3); // bucket 1 thawed but active
- EXPECT_EQ(0u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
-
- deactivateBucket(_ready.bucket(1));
- _bmj->scanAndMove(0, 3); // handle de-activated bucket 1
- EXPECT_EQ(3u, docsMoved().size());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_ready.bucket(1), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, ready_bucket_not_moved_to_not_ready_if_node_is_marked_as_retired)
-{
- _calc->setNodeRetired(true);
- // Bucket 2 would be moved from ready to not ready in a non-retired case, but not when retired.
- addReady(_ready.bucket(1));
- _bmj->scanAndMove(4, 3);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
-}
-
-// Technically this should never happen since a retired node is never in the ideal state,
-// but test this case for the sake of completion.
-TEST_F(ControllerFixture, inactive_not_ready_bucket_not_moved_to_ready_if_node_is_marked_as_retired)
-{
- _calc->setNodeRetired(true);
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(3));
- _bmj->scanAndMove(4, 3);
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(0u, docsMoved().size());
-}
-
-TEST_F(ControllerFixture, explicitly_active_not_ready_bucket_can_be_moved_to_ready_even_if_node_is_marked_as_retired)
-{
- _calc->setNodeRetired(true);
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- addReady(_notReady.bucket(3));
- activateBucket(_notReady.bucket(3));
- _bmj->scanAndMove(4, 3);
- EXPECT_FALSE(_bmj->done());
- ASSERT_EQ(2u, docsMoved().size());
- assertEqual(_notReady.bucket(3), _notReady.docs(3)[0], 2, 1, docsMoved()[0]);
- assertEqual(_notReady.bucket(3), _notReady.docs(3)[1], 2, 1, docsMoved()[1]);
- ASSERT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]);
-}
-
-TEST_F(ControllerFixture, require_that_notifyCreateBucket_causes_bucket_to_be_reconsidered_by_job)
-{
- EXPECT_FALSE(_bmj->done());
- addReady(_ready.bucket(1));
- addReady(_ready.bucket(2));
- runLoop();
- EXPECT_TRUE(_bmj->done());
- EXPECT_TRUE(docsMoved().empty());
- EXPECT_TRUE(bucketsModified().empty());
- addReady(_notReady.bucket(3)); // bucket 3 now ready, no notify
- EXPECT_TRUE(_bmj->done()); // move job still believes work done
- _bmj->notifyCreateBucket(_bucketDB->takeGuard(), _notReady.bucket(3)); // reconsider bucket 3
- EXPECT_FALSE(_bmj->done());
- runLoop();
- EXPECT_TRUE(_bmj->done());
- EXPECT_EQ(1u, bucketsModified().size());
- EXPECT_EQ(2u, docsMoved().size());
-}
-
-struct ResourceLimitControllerFixture : public ControllerFixture
-{
- ResourceLimitControllerFixture(double resourceLimitFactor = RESOURCE_LIMIT_FACTOR) :
- ControllerFixture(BlockableMaintenanceJobConfig(resourceLimitFactor, MAX_OUTSTANDING_OPS))
- {}
-
- void testJobStopping(DiskMemUsageState blockingUsageState) {
- // Bucket 1 should be moved
- addReady(_ready.bucket(2));
- // Note: This depends on _bmj->run() moving max 1 documents
- EXPECT_TRUE(!_bmj->run());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- // Notify that we've over limit
- _diskMemUsageNotifier.notify(blockingUsageState);
- EXPECT_TRUE(_bmj->run());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- // Notify that we've under limit
- _diskMemUsageNotifier.notify(DiskMemUsageState());
- EXPECT_TRUE(!_bmj->run());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- }
-
- void testJobNotStopping(DiskMemUsageState blockingUsageState) {
- // Bucket 1 should be moved
- addReady(_ready.bucket(2));
- // Note: This depends on _bmj->run() moving max 1 documents
- EXPECT_TRUE(!_bmj->run());
- EXPECT_EQ(1u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- // Notify that we've over limit, but not over adjusted limit
- _diskMemUsageNotifier.notify(blockingUsageState);
- EXPECT_TRUE(!_bmj->run());
- EXPECT_EQ(2u, docsMoved().size());
- EXPECT_EQ(0u, bucketsModified().size());
- }
-};
-
-struct ResourceLimitControllerFixture_1_2 : public ResourceLimitControllerFixture {
- ResourceLimitControllerFixture_1_2() : ResourceLimitControllerFixture(1.2) {}
-};
-
-TEST_F(ResourceLimitControllerFixture, require_that_bucket_move_stops_when_disk_limit_is_reached)
-{
- testJobStopping(DiskMemUsageState(ResourceUsageState(0.7, 0.8), ResourceUsageState()));
-}
-
-TEST_F(ResourceLimitControllerFixture, require_that_bucket_move_stops_when_memory_limit_is_reached)
-{
- testJobStopping(DiskMemUsageState(ResourceUsageState(), ResourceUsageState(0.7, 0.8)));
-}
-
-TEST_F(ResourceLimitControllerFixture_1_2, require_that_bucket_move_uses_resource_limit_factor_for_disk_resource_limit)
-{
- testJobNotStopping(DiskMemUsageState(ResourceUsageState(0.7, 0.8), ResourceUsageState()));
-}
-
-TEST_F(ResourceLimitControllerFixture_1_2, require_that_bucket_move_uses_resource_limit_factor_for_memory_resource_limit)
-{
- testJobNotStopping(DiskMemUsageState(ResourceUsageState(), ResourceUsageState(0.7, 0.8)));
-}
-
-struct MaxOutstandingMoveOpsFixture : public ControllerFixture
-{
- MaxOutstandingMoveOpsFixture(uint32_t maxOutstandingOps) :
- ControllerFixture(BlockableMaintenanceJobConfig(RESOURCE_LIMIT_FACTOR, maxOutstandingOps))
- {
- // Bucket 1 should be moved from ready -> notready
- addReady(_ready.bucket(2));
- }
-
- void assertRunToBlocked() {
- EXPECT_TRUE(_bmj->run()); // job becomes blocked as max outstanding limit is reached
- EXPECT_FALSE(_bmj->done());
- EXPECT_TRUE(_bmj->isBlocked());
- EXPECT_TRUE(_bmj->isBlocked(BlockedReason::OUTSTANDING_OPS));
- }
- void assertRunToNotBlocked() {
- EXPECT_FALSE(_bmj->run());
- EXPECT_FALSE(_bmj->done());
- EXPECT_FALSE(_bmj->isBlocked());
- }
- void assertRunToFinished() {
- EXPECT_TRUE(_bmj->run());
- EXPECT_TRUE(_bmj->done());
- EXPECT_FALSE(_bmj->isBlocked());
- }
- void assertDocsMoved(uint32_t expDocsMovedCnt, uint32_t expMoveContextsCnt) {
- EXPECT_EQ(expDocsMovedCnt, docsMoved().size());
- EXPECT_EQ(expMoveContextsCnt, _moveHandler._moveDoneContexts.size());
- }
- void unblockJob(uint32_t expRunnerCnt) {
- _moveHandler.clearMoveDoneContexts(); // unblocks job and try to execute it via runner
- EXPECT_EQ(expRunnerCnt, _runner.runCount);
- EXPECT_FALSE(_bmj->isBlocked());
- }
-};
-
-struct MaxOutstandingMoveOpsFixture_1 : public MaxOutstandingMoveOpsFixture {
- MaxOutstandingMoveOpsFixture_1() : MaxOutstandingMoveOpsFixture(1) {}
-};
-
-struct MaxOutstandingMoveOpsFixture_2 : public MaxOutstandingMoveOpsFixture {
- MaxOutstandingMoveOpsFixture_2() : MaxOutstandingMoveOpsFixture(2) {}
-};
-
-TEST_F(MaxOutstandingMoveOpsFixture_1, require_that_bucket_move_job_is_blocked_if_it_has_too_many_outstanding_move_operations__max_1)
-{
- assertRunToBlocked();
- assertDocsMoved(1, 1);
- assertRunToBlocked();
- assertDocsMoved(1, 1);
-
- unblockJob(1);
- assertRunToBlocked();
- assertDocsMoved(2, 1);
-
- unblockJob(2);
- assertRunToBlocked();
- assertDocsMoved(3, 1);
-
- unblockJob(3);
- assertRunToFinished();
- assertDocsMoved(3, 0);
-}
-
-TEST_F(MaxOutstandingMoveOpsFixture_2, require_that_bucket_move_job_is_blocked_if_it_has_too_many_outstanding_move_operations_max_2)
-{
- assertRunToNotBlocked();
- assertDocsMoved(1, 1);
-
- assertRunToBlocked();
- assertDocsMoved(2, 2);
-
- unblockJob(1);
- assertRunToNotBlocked();
- assertDocsMoved(3, 1);
-
- assertRunToFinished();
- assertDocsMoved(3, 1);
-}
-
-GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
index 688dd963f61..f1353a30887 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
@@ -4,12 +4,12 @@
using BlockedReason = IBlockableMaintenanceJob::BlockedReason;
-TEST_P(JobTest, handler_name_is_used_as_part_of_job_name)
+TEST_F(JobTest, handler_name_is_used_as_part_of_job_name)
{
EXPECT_EQ("lid_space_compaction.myhandler", _job->getName());
}
-TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit)
+TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit)
{
// 20% bloat < 30% allowed bloat
addStats(10, {1,3,4,5,6,7,9}, {{9,2}});
@@ -17,7 +17,7 @@ TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_factor_is_below_limit)
assertNoWorkDone();
}
-TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit)
+TEST_F(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit)
{
init(3, 0.1);
// 20% bloat >= 10% allowed bloat BUT lid bloat (2) < allowed lid bloat (3)
@@ -26,7 +26,7 @@ TEST_P(JobTest, no_move_operation_is_created_if_lid_bloat_is_below_limit)
assertNoWorkDone();
}
-TEST_P(JobTest, no_move_operation_is_created_and_compaction_is_initiated)
+TEST_F(JobTest, no_move_operation_is_created_and_compaction_is_initiated)
{
// no documents to move: lowestFreeLid(7) > highestUsedLid(6)
addStats(10, {1,2,3,4,5,6}, {{6,7}});
@@ -36,14 +36,14 @@ TEST_P(JobTest, no_move_operation_is_created_and_compaction_is_initiated)
assertJobContext(0, 0, 0, 7, 1);
}
-TEST_P(JobTest, one_move_operation_is_created_and_compaction_is_initiated)
+TEST_F(JobTest, one_move_operation_is_created_and_compaction_is_initiated)
{
setupOneDocumentToCompact();
EXPECT_FALSE(run()); // scan
assertOneDocumentCompacted();
}
-TEST_P(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed)
+TEST_F(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_are_needed)
{
setupThreeDocumentsToCompact();
EXPECT_FALSE(run());
@@ -56,29 +56,7 @@ TEST_P(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_ar
assertJobContext(4, 7, 3, 7, 1);
}
-TEST_P(JobTest, job_is_blocked_if_trying_to_move_document_for_frozen_bucket)
-{
- //TODO Remove test once we no longer use the frozen concept.
- if ( ! useBucketDB() ) {
- _frozenHandler._bucket = BUCKET_ID_1;
- EXPECT_FALSE(_job->isBlocked());
- addStats(10, {1, 3, 4, 5, 6, 9}, {{9, 2}}); // 30% bloat: try to move 9 -> 2
- addStats(0, 0, 0, 0);
-
- EXPECT_TRUE(run()); // bucket frozen
- assertNoWorkDone();
- EXPECT_TRUE(_job->isBlocked());
-
- _frozenHandler._bucket = BUCKET_ID_2;
- _job->unBlock(BlockedReason::FROZEN_BUCKET);
-
- EXPECT_FALSE(run()); // unblocked
- assertJobContext(2, 9, 1, 0, 0);
- EXPECT_FALSE(_job->isBlocked());
- }
-}
-
-TEST_P(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large)
+TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large)
{
init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR);
addMultiStats(10, {{1,3,4,5,6,9},{1,2,4,5,6,8}},
@@ -98,7 +76,7 @@ TEST_P(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large)
assertJobContext(3, 8, 2, 7, 1);
}
-TEST_P(JobTest, held_lid_is_not_considered_free_and_blocks_job)
+TEST_F(JobTest, held_lid_is_not_considered_free_and_blocks_job)
{
// Lid 1 on hold or pendingHold, i.e. neither free nor used.
addMultiStats(3, {{2}}, {{2, 3}});
@@ -106,7 +84,7 @@ TEST_P(JobTest, held_lid_is_not_considered_free_and_blocks_job)
assertNoWorkDone();
}
-TEST_P(JobTest, held_lid_is_not_considered_free_with_only_compact)
+TEST_F(JobTest, held_lid_is_not_considered_free_with_only_compact)
{
// Lid 1 on hold or pendingHold, i.e. neither free nor used.
addMultiStats(10, {{2}}, {{2, 3}});
@@ -116,7 +94,7 @@ TEST_P(JobTest, held_lid_is_not_considered_free_with_only_compact)
assertJobContext(0, 0, 0, 3, 1);
}
-TEST_P(JobTest, held_lids_are_not_considered_free_with_one_move)
+TEST_F(JobTest, held_lids_are_not_considered_free_with_one_move)
{
// Lids 1,2,3 on hold or pendingHold, i.e. neither free nor used.
addMultiStats(10, {{5}}, {{5, 4}, {4, 5}});
@@ -126,7 +104,7 @@ TEST_P(JobTest, held_lids_are_not_considered_free_with_one_move)
assertJobContext(4, 5, 1, 5, 1);
}
-TEST_P(JobTest, resource_starvation_blocks_lid_space_compaction)
+TEST_F(JobTest, resource_starvation_blocks_lid_space_compaction)
{
setupOneDocumentToCompact();
_diskMemUsageNotifier.notify({{100, 0}, {100, 101}});
@@ -134,7 +112,7 @@ TEST_P(JobTest, resource_starvation_blocks_lid_space_compaction)
assertNoWorkDone();
}
-TEST_P(JobTest, ending_resource_starvation_resumes_lid_space_compaction)
+TEST_F(JobTest, ending_resource_starvation_resumes_lid_space_compaction)
{
setupOneDocumentToCompact();
_diskMemUsageNotifier.notify({{100, 0}, {100, 101}});
@@ -144,7 +122,7 @@ TEST_P(JobTest, ending_resource_starvation_resumes_lid_space_compaction)
assertOneDocumentCompacted();
}
-TEST_P(JobTest, resource_limit_factor_adjusts_limit)
+TEST_F(JobTest, resource_limit_factor_adjusts_limit)
{
init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 1.05);
setupOneDocumentToCompact();
@@ -153,21 +131,21 @@ TEST_P(JobTest, resource_limit_factor_adjusts_limit)
assertOneDocumentCompacted();
}
-TEST_P(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs)
+TEST_F(JobTest, delay_is_set_based_on_interval_and_is_max_300_secs)
{
init_with_interval(301s);
EXPECT_EQ(300s, _job->getDelay());
EXPECT_EQ(301s, _job->getInterval());
}
-TEST_P(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs)
+TEST_F(JobTest, delay_is_set_based_on_interval_and_can_be_less_than_300_secs)
{
init_with_interval(299s);
EXPECT_EQ(299s, _job->getDelay());
EXPECT_EQ(299s, _job->getInterval());
}
-TEST_P(JobTest, job_is_disabled_when_node_is_retired)
+TEST_F(JobTest, job_is_disabled_when_node_is_retired)
{
init_with_node_retired(true);
setupOneDocumentToCompact();
@@ -175,7 +153,7 @@ TEST_P(JobTest, job_is_disabled_when_node_is_retired)
assertNoWorkDone();
}
-TEST_P(JobTest, job_is_disabled_when_node_becomes_retired)
+TEST_F(JobTest, job_is_disabled_when_node_becomes_retired)
{
init_with_node_retired(false);
setupOneDocumentToCompact();
@@ -184,7 +162,7 @@ TEST_P(JobTest, job_is_disabled_when_node_becomes_retired)
assertNoWorkDone();
}
-TEST_P(JobTest, job_is_re_enabled_when_node_is_no_longer_retired)
+TEST_F(JobTest, job_is_re_enabled_when_node_is_no_longer_retired)
{
init_with_node_retired(true);
setupOneDocumentToCompact();
@@ -194,7 +172,7 @@ TEST_P(JobTest, job_is_re_enabled_when_node_is_no_longer_retired)
assertOneDocumentCompacted();
}
-TEST_P(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker)
+TEST_F(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rate_tracker)
{
auto& remove_batch_tracker = _handler->_rm_listener->get_remove_batch_tracker();
EXPECT_EQ(vespalib::from_s(21.0), remove_batch_tracker.get_time_budget_per_op());
@@ -205,38 +183,38 @@ TEST_P(JobDisabledByRemoveOpsTest, config_is_propagated_to_remove_operations_rat
EXPECT_EQ(vespalib::from_s(20.0), remove_tracker.get_time_budget_window());
}
-TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing)
+TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_remove_batch_is_ongoing)
{
job_is_disabled_while_remove_ops_are_ongoing(true);
}
-TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts)
+TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_remove_batch_starts)
{
job_becomes_disabled_if_remove_ops_starts(true);
}
-TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing)
+TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_remove_batch_is_no_longer_ongoing)
{
job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(true);
}
-TEST_P(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing)
+TEST_F(JobDisabledByRemoveOpsTest, job_is_disabled_while_removes_are_ongoing)
{
job_is_disabled_while_remove_ops_are_ongoing(false);
}
-TEST_P(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start)
+TEST_F(JobDisabledByRemoveOpsTest, job_becomes_disabled_if_removes_start)
{
job_becomes_disabled_if_remove_ops_starts(false);
}
-TEST_P(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing)
+TEST_F(JobDisabledByRemoveOpsTest, job_is_re_enabled_when_removes_are_no_longer_ongoing)
{
job_is_re_enabled_when_remove_ops_are_no_longer_ongoing(false);
}
-TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1)
+TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_1)
{
init(1);
setupThreeDocumentsToCompact();
@@ -259,7 +237,7 @@ TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move
assertJobContext(4, 7, 3, 7, 1);
}
-TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2)
+TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move_operations_with_max_2)
{
init(2);
setupThreeDocumentsToCompact();
@@ -277,8 +255,4 @@ TEST_P(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move
sync();
}
-VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobTest, ::testing::Values(false, true));
-VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, JobDisabledByRemoveOpsTest, ::testing::Values(false, true));
-VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(bool, MaxOutstandingJobTest, ::testing::Values(false, true));
-
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
index d394769c0ee..73cd6a8e6be 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp
@@ -1,7 +1,6 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "lid_space_jobtest.h"
-#include <vespa/searchcore/proton/server/lid_space_compaction_job.h>
#include <vespa/searchcore/proton/server/lid_space_compaction_job_take2.h>
#include <vespa/searchcore/proton/server/executorthreadingservice.h>
#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h>
@@ -33,7 +32,6 @@ JobTestBase::JobTestBase()
_diskMemUsageNotifier(),
_handler(),
_storer(),
- _frozenHandler(),
_job()
{
init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, MAX_OUTSTANDING_MOVE_OPS);
@@ -47,23 +45,18 @@ JobTestBase::init(uint32_t allowedLidBloat,
bool nodeRetired,
uint32_t maxOutstandingMoveOps)
{
- _handler = std::make_shared<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, useBucketDB());
+ _handler = std::make_shared<MyHandler>(maxOutstandingMoveOps != MAX_OUTSTANDING_MOVE_OPS, true);
DocumentDBLidSpaceCompactionConfig compactCfg(interval, allowedLidBloat, allowedLidBloatFactor,
- REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, false, useBucketDB());
+ REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, false, true);
BlockableMaintenanceJobConfig blockableCfg(resourceLimitFactor, maxOutstandingMoveOps);
_job.reset();
- if (useBucketDB()) {
- _singleExecutor = std::make_unique<vespalib::ThreadStackExecutor>(1, 0x10000);
- _master = std::make_unique<proton::ExecutorThreadService> (*_singleExecutor);
- _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4);
- _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor,
- _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired,
- document::BucketSpace::placeHolder());
- } else {
- _job = std::make_shared<LidSpaceCompactionJob>(compactCfg, _handler, _storer, _frozenHandler, _diskMemUsageNotifier,
- blockableCfg, _clusterStateHandler, nodeRetired);
- }
+ _singleExecutor = std::make_unique<vespalib::ThreadStackExecutor>(1, 0x10000);
+ _master = std::make_unique<proton::ExecutorThreadService> (*_singleExecutor);
+ _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4);
+ _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor,
+ _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired,
+ document::BucketSpace::placeHolder());
}
void
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h
index dde48a0a620..747e5c9faca 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h
@@ -8,7 +8,7 @@
#include <vespa/vespalib/gtest/gtest.h>
namespace storage::spi::dummy { class DummyBucketExecutor; }
-struct JobTestBase : public ::testing::TestWithParam<bool> {
+struct JobTestBase : public ::testing::Test {
MonitoredRefCount _refCount;
test::ClusterStateHandler _clusterStateHandler;
test::DiskMemUsageNotifier _diskMemUsageNotifier;
@@ -17,7 +17,6 @@ struct JobTestBase : public ::testing::TestWithParam<bool> {
std::unique_ptr<searchcorespi::index::IThreadService> _master;
std::shared_ptr<MyHandler> _handler;
MyStorer _storer;
- MyFrozenBucketHandler _frozenHandler;
std::shared_ptr<BlockableMaintenanceJob> _job;
JobTestBase();
~JobTestBase() override;
@@ -51,7 +50,6 @@ struct JobTestBase : public ::testing::TestWithParam<bool> {
void assertOneDocumentCompacted();
JobTestBase &setupThreeDocumentsToCompact();
void sync() const;
- bool useBucketDB() const { return GetParam(); }
};
struct JobTest : public JobTestBase {
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index b0c46dbd789..f401d23b8ad 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -929,57 +929,6 @@ MaintenanceControllerFixture::removeDocs(const test::UserDocuments &docs, Timest
}
}
-TEST_F("require that bucket move controller is active", MaintenanceControllerFixture)
-{
- f._builder.createDocs(1, 1, 4); // 3 docs
- f._builder.createDocs(2, 4, 6); // 2 docs
- test::UserDocuments readyDocs(f._builder.getDocs());
- BucketId bucketId1(readyDocs.getBucket(1));
- BucketId bucketId2(readyDocs.getBucket(2));
- f.insertDocs(readyDocs, f._ready);
- f._builder.clearDocs();
- f._builder.createDocs(3, 1, 3); // 2 docs
- f._builder.createDocs(4, 3, 6); // 3 docs
- test::UserDocuments notReadyDocs(f._builder.getDocs());
- BucketId bucketId4(notReadyDocs.getBucket(4));
- f.insertDocs(notReadyDocs, f._notReady);
- f._builder.clearDocs();
- f.notifyClusterStateChanged();
- EXPECT_TRUE(f._executor.isIdle());
- EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(5u, f._ready.getDocumentCount());
- EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
- f.startMaintenance();
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(0u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(0u, f._ready.getDocumentCount());
- EXPECT_EQUAL(10u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(10u, f._notReady.getDocumentCount());
- f._calc->addReady(bucketId1);
- f.notifyClusterStateChanged();
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(3u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(3u, f._ready.getDocumentCount());
- EXPECT_EQUAL(7u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(7u, f._notReady.getDocumentCount());
- MyFrozenBucket::UP frozen2(new MyFrozenBucket(f._mc, bucketId2));
- f._calc->addReady(bucketId2);
- f._calc->addReady(bucketId4);
- f.notifyClusterStateChanged();
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(6u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(6u, f._ready.getDocumentCount());
- EXPECT_EQUAL(4u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(4u, f._notReady.getDocumentCount());
- frozen2.reset();
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(8u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(8u, f._ready.getDocumentCount());
- EXPECT_EQUAL(2u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(2u, f._notReady.getDocumentCount());
-}
-
TEST_F("require that document pruner is active", MaintenanceControllerFixture)
{
uint64_t tshz = 1000000;
@@ -1054,74 +1003,6 @@ TEST_F("require that periodic session prunings are scheduled",
ASSERT_TRUE(f._gsp.isInvoked);
}
-TEST_F("require that active bucket is not moved until de-activated", MaintenanceControllerFixture)
-{
- f._builder.createDocs(1, 1, 4); // 3 docs
- f._builder.createDocs(2, 4, 6); // 2 docs
- test::UserDocuments readyDocs(f._builder.getDocs());
- f.insertDocs(readyDocs, f._ready);
- f._builder.clearDocs();
- f._builder.createDocs(3, 1, 3); // 2 docs
- f._builder.createDocs(4, 3, 6); // 3 docs
- test::UserDocuments notReadyDocs(f._builder.getDocs());
- f.insertDocs(notReadyDocs, f._notReady);
- f._builder.clearDocs();
-
- // bucket 1 (active) should be moved from ready to not ready according to cluster state
- f._calc->addReady(readyDocs.getBucket(2));
- f._ready.setBucketState(readyDocs.getBucket(1), true);
-
- f.notifyClusterStateChanged();
- EXPECT_TRUE(f._executor.isIdle());
- EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(5u, f._ready.getDocumentCount());
- EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
-
- f.startMaintenance();
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(5u, f._ready.getDocumentCount());
- EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
-
- // de-activate bucket 1
- f._ready.setBucketState(readyDocs.getBucket(1), false);
- f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE);
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(2u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(2u, f._ready.getDocumentCount());
- EXPECT_EQUAL(8u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(8u, f._notReady.getDocumentCount());
-
- // re-activate bucket 1
- f._ready.setBucketState(readyDocs.getBucket(1), true);
- f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE);
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(5u, f._ready.getDocumentCount());
- EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
-
- // de-activate bucket 1
- f._ready.setBucketState(readyDocs.getBucket(1), false);
- f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE);
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(2u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(2u, f._ready.getDocumentCount());
- EXPECT_EQUAL(8u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(8u, f._notReady.getDocumentCount());
-
- // re-activate bucket 1
- f._ready.setBucketState(readyDocs.getBucket(1), true);
- f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE);
- ASSERT_TRUE(f._executor.waitIdle(TIMEOUT));
- EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
- EXPECT_EQUAL(5u, f._ready.getDocumentCount());
- EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
- EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
-}
-
TEST_F("require that a simple maintenance job is executed", MaintenanceControllerFixture)
{
auto job = std::make_unique<MySimpleJob>(200ms, 200ms, 3);