diff options
Diffstat (limited to 'storage/src/tests/distributor')
4 files changed, 147 insertions, 62 deletions
diff --git a/storage/src/tests/distributor/distributor_stripe_test.cpp b/storage/src/tests/distributor/distributor_stripe_test.cpp index 556f5e20fb2..902dd6454f1 100644 --- a/storage/src/tests/distributor/distributor_stripe_test.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test.cpp @@ -179,6 +179,16 @@ struct DistributorStripeTest : Test, DistributorStripeTestUtil { configure_stripe(builder); } + void configure_implicitly_clear_priority_on_schedule(bool implicitly_clear) { + ConfigBuilder builder; + builder.implicitlyClearBucketPriorityOnSchedule = implicitly_clear; + configure_stripe(builder); + } + + bool scheduler_has_implicitly_clear_priority_on_schedule_set() const noexcept { + return _stripe->_scheduler->implicitly_clear_priority_on_schedule(); + } + void configureMaxClusterClockSkew(int seconds); void configure_mutation_sequencing(bool enabled); void configure_merge_busy_inhibit_duration(int seconds); @@ -888,6 +898,17 @@ TEST_F(DistributorStripeTest, max_activation_inhibited_out_of_sync_groups_config EXPECT_EQ(getConfig().max_activation_inhibited_out_of_sync_groups(), 0); } +TEST_F(DistributorStripeTest, implicitly_clear_priority_on_schedule_config_is_propagated_to_scheduler) +{ + setup_stripe(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); + + configure_implicitly_clear_priority_on_schedule(true); + EXPECT_TRUE(scheduler_has_implicitly_clear_priority_on_schedule_set()); + + configure_implicitly_clear_priority_on_schedule(false); + EXPECT_FALSE(scheduler_has_implicitly_clear_priority_on_schedule_set()); +} + TEST_F(DistributorStripeTest, wanted_split_bit_count_is_lower_bounded) { setup_stripe(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); diff --git a/storage/src/tests/distributor/maintenancemocks.h b/storage/src/tests/distributor/maintenancemocks.h index f708fbd41aa..2809a81f79b 100644 --- a/storage/src/tests/distributor/maintenancemocks.h +++ b/storage/src/tests/distributor/maintenancemocks.h @@ -4,6 +4,7 @@ #include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> +#include <vespa/storage/distributor/maintenance/pending_window_checker.h> #include <vespa/storage/distributor/operationstarter.h> #include <vespa/storage/distributor/operations/operation.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> @@ -122,4 +123,16 @@ public: } }; +class MockPendingWindowChecker : public PendingWindowChecker { + bool _allow = true; +public: + void allow_operations(bool allow) noexcept { + _allow = allow; + } + + bool may_allow_operation_with_priority(OperationStarter::Priority) const noexcept override { + return _allow; + } +}; + } diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp index a97ffeef24b..44b842a2622 100644 --- a/storage/src/tests/distributor/maintenanceschedulertest.cpp +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -17,61 +17,111 @@ using document::BucketId; using Priority = MaintenancePriority; using WaitTimeMs = MaintenanceScheduler::WaitTimeMs; -struct MaintenanceSchedulerTest : Test { - std::unique_ptr<SimpleBucketPriorityDatabase> _priorityDb; - std::unique_ptr<MockMaintenanceOperationGenerator> _operationGenerator; - std::unique_ptr<MockOperationStarter> _operationStarter; - std::unique_ptr<MaintenanceScheduler> _scheduler; +struct MaintenanceSchedulerTest : TestWithParam<bool> { + SimpleBucketPriorityDatabase _priority_db; + MockMaintenanceOperationGenerator _operation_generator; + MockOperationStarter _operation_starter; + MockPendingWindowChecker _pending_window_checker; + MaintenanceScheduler _scheduler; - void SetUp() override; + MaintenanceSchedulerTest() + : _priority_db(), + _operation_generator(), + _operation_starter(), + _pending_window_checker(), + _scheduler(_operation_generator, _priority_db, _pending_window_checker, _operation_starter) + {} + + void SetUp() override { + _scheduler.set_implicitly_clear_priority_on_schedule(GetParam()); + } }; -void -MaintenanceSchedulerTest::SetUp() -{ - _priorityDb = std::make_unique<SimpleBucketPriorityDatabase>(); - _operationGenerator = std::make_unique<MockMaintenanceOperationGenerator>(); - _operationStarter = std::make_unique<MockOperationStarter>(); - _scheduler = std::make_unique<MaintenanceScheduler>(*_operationGenerator, *_priorityDb, *_operationStarter); +TEST_P(MaintenanceSchedulerTest, priority_cleared_after_scheduled) { + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGHEST)); + _scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); + EXPECT_EQ("", _priority_db.toString()); } -TEST_F(MaintenanceSchedulerTest, priority_cleared_after_scheduled) { - _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGHEST)); - _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); - EXPECT_EQ("", _priorityDb->toString()); +TEST_P(MaintenanceSchedulerTest, operation_is_scheduled) { + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM)); + _scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); + EXPECT_EQ("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 100\n", + _operation_starter.toString()); } -TEST_F(MaintenanceSchedulerTest, operation_is_scheduled) { - _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM)); - _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); - EXPECT_EQ("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 100\n", - _operationStarter->toString()); +TEST_P(MaintenanceSchedulerTest, operation_is_not_scheduled_if_pending_ops_not_accepted) { + if (!GetParam()) { + return; // Only works when implicit clearing is enabled + } + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM)); + _pending_window_checker.allow_operations(false); + _scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); + EXPECT_EQ("", _operation_starter.toString()); + // Priority DB entry is not cleared + EXPECT_EQ("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri MEDIUM)\n", + _priority_db.toString()); } -TEST_F(MaintenanceSchedulerTest, no_operations_to_schedule) { - WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); +TEST_P(MaintenanceSchedulerTest, no_operations_to_schedule) { + WaitTimeMs waitMs(_scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); EXPECT_EQ(WaitTimeMs(1), waitMs); - EXPECT_EQ("", _operationStarter->toString()); + EXPECT_EQ("", _operation_starter.toString()); } -TEST_F(MaintenanceSchedulerTest, suppress_low_priorities_in_emergency_mode) { - _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::VERY_HIGH)); - _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 2)), Priority::HIGHEST)); - EXPECT_EQ(WaitTimeMs(0), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); - EXPECT_EQ(WaitTimeMs(1), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); +TEST_P(MaintenanceSchedulerTest, suppress_low_priorities_in_emergency_mode) { + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::VERY_HIGH)); + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 2)), Priority::HIGHEST)); + EXPECT_EQ(WaitTimeMs(0), _scheduler.tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); + EXPECT_EQ(WaitTimeMs(1), _scheduler.tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); EXPECT_EQ("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri 0\n", - _operationStarter->toString()); + _operation_starter.toString()); EXPECT_EQ("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n", - _priorityDb->toString()); + _priority_db.toString()); +} + +TEST_P(MaintenanceSchedulerTest, priority_not_cleared_if_operation_not_started) { + if (GetParam()) { + return; // Only works when implicit clearing is NOT enabled + } + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); + _operation_starter.setShouldStartOperations(false); + WaitTimeMs waitMs(_scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); + EXPECT_EQ(WaitTimeMs(1), waitMs); + EXPECT_EQ("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n", + _priority_db.toString()); } -TEST_F(MaintenanceSchedulerTest, priority_not_cleared_if_operation_not_started) { - _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); - _operationStarter->setShouldStartOperations(false); - WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); +TEST_P(MaintenanceSchedulerTest, priority_cleared_if_operation_not_started_inside_pending_window) { + if (!GetParam()) { + return; // Only works when implicit clearing is enabled + } + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); + _operation_starter.setShouldStartOperations(false); + WaitTimeMs waitMs(_scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); + EXPECT_EQ(WaitTimeMs(1), waitMs); + EXPECT_EQ("", _priority_db.toString()); +} + +TEST_P(MaintenanceSchedulerTest, priority_not_cleared_if_operation_not_started_inside_pending_window_for_highest_pri) { + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGHEST)); + _operation_starter.setShouldStartOperations(false); + WaitTimeMs waitMs(_scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); + EXPECT_EQ(WaitTimeMs(1), waitMs); + EXPECT_EQ("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGHEST)\n", + _priority_db.toString()); +} + +TEST_P(MaintenanceSchedulerTest, priority_not_cleared_if_operation_not_started_outside_pending_window) { + _priority_db.setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); + _operation_starter.setShouldStartOperations(false); + _pending_window_checker.allow_operations(false); + WaitTimeMs waitMs(_scheduler.tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); EXPECT_EQ(WaitTimeMs(1), waitMs); EXPECT_EQ("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n", - _priorityDb->toString()); + _priority_db.toString()); } +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(ImplicitClearOfDbPri, MaintenanceSchedulerTest, ::testing::Values(false, true)); + } diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp index eac345a243f..4e99388327b 100644 --- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -21,7 +21,7 @@ const MockOperation& as_mock_operation(const Operation& operation) { struct ThrottlingOperationStarterTest : Test { std::shared_ptr<Operation> createMockOperation() { - return std::shared_ptr<Operation>(new MockOperation(makeDocumentBucket(BucketId(16, 1)))); + return std::make_shared<MockOperation>(makeDocumentBucket(BucketId(16, 1))); } std::unique_ptr<MockOperationStarter> _starterImpl; @@ -48,14 +48,12 @@ ThrottlingOperationStarterTest::TearDown() TEST_F(ThrottlingOperationStarterTest, operation_not_throttled_when_slot_available) { auto operation = createMockOperation(); - EXPECT_TRUE(_operationStarter->start(operation, - OperationStarter::Priority(0))); + EXPECT_TRUE(_operationStarter->start(operation, OperationStarter::Priority(0))); EXPECT_FALSE(as_mock_operation(*operation).get_was_throttled()); } TEST_F(ThrottlingOperationStarterTest, operation_starting_is_forwarded_to_implementation) { - ASSERT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(0))); + ASSERT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); EXPECT_EQ("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 0\n", _starterImpl->toString()); } @@ -63,8 +61,8 @@ TEST_F(ThrottlingOperationStarterTest, operation_starting_is_forwarded_to_implem TEST_F(ThrottlingOperationStarterTest, operation_throttled_when_no_available_slots) { _operationStarter->setMaxPendingRange(0, 0); auto operation = createMockOperation(); - EXPECT_FALSE(_operationStarter->start(operation, - OperationStarter::Priority(0))); + EXPECT_FALSE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(0))); + EXPECT_FALSE(_operationStarter->start(operation, OperationStarter::Priority(0))); EXPECT_TRUE(as_mock_operation(*operation).get_was_throttled()); } @@ -88,32 +86,35 @@ TEST_F(ThrottlingOperationStarterTest, throttling_with_max_pending_range) { TEST_F(ThrottlingOperationStarterTest, starting_operations_fills_up_pending_window) { _operationStarter->setMaxPendingRange(1, 3); - EXPECT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(255))); - EXPECT_FALSE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(255))); - EXPECT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(100))); - EXPECT_FALSE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(100))); - EXPECT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(0))); - EXPECT_FALSE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(0))); + EXPECT_TRUE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(255))); + EXPECT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(255))); + + EXPECT_FALSE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(255))); + EXPECT_FALSE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(255))); + + EXPECT_TRUE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(100))); + EXPECT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(100))); + + EXPECT_FALSE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(255))); + EXPECT_FALSE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(100))); + + EXPECT_TRUE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(0))); + EXPECT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); + + EXPECT_FALSE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(0))); + EXPECT_FALSE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); } TEST_F(ThrottlingOperationStarterTest, finishing_operations_allows_more_to_start) { _operationStarter->setMaxPendingRange(1, 1); - EXPECT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(255))); - EXPECT_FALSE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(255))); + EXPECT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(255))); + EXPECT_FALSE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(255))); EXPECT_FALSE(_starterImpl->getOperations().empty()); _starterImpl->getOperations().pop_back(); - EXPECT_TRUE(_operationStarter->start(createMockOperation(), - OperationStarter::Priority(255))); + EXPECT_TRUE(_operationStarter->may_allow_operation_with_priority(OperationStarter::Priority(255))); + EXPECT_TRUE(_operationStarter->start(createMockOperation(), OperationStarter::Priority(255))); EXPECT_FALSE(_starterImpl->getOperations().empty()); } |