diff options
Diffstat (limited to 'storage/src/tests/distributor')
9 files changed, 94 insertions, 29 deletions
diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 923c7d1730b..f0b3db3adf8 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -32,7 +32,7 @@ DistributorStripeTestUtil::DistributorStripeTestUtil() _done_initializing(true), _messageSender(_sender, _senderDown) { - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } DistributorStripeTestUtil::~DistributorStripeTestUtil() = default; @@ -40,7 +40,7 @@ DistributorStripeTestUtil::~DistributorStripeTestUtil() = default; void DistributorStripeTestUtil::createLinks() { - _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); + _node = std::make_unique<TestDistributorApp>(_config->config_uri()); _metrics = std::make_shared<DistributorMetricSet>(); _ideal_state_metrics = std::make_shared<IdealStateMetricSet>(); _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), *_metrics, *_ideal_state_metrics, @@ -184,8 +184,8 @@ DistributorStripeTestUtil::close() { _stripe->flush_and_close(); _sender.clear(); - _node.reset(0); - _config = getStandardConfig(false); + _node.reset(); + _config = StorageConfigSet::make_distributor_node_config(); } namespace { diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index 801320e2bf8..862d9bfbfba 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -5,7 +5,9 @@ #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> +#include <tests/common/storage_config_set.h> #include <vespa/storage/common/hostreporter/hostinfo.h> +#include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/storage/distributor/stripe_host_info_notifier.h> #include <vespa/storage/storageutil/utils.h> @@ -132,8 +134,8 @@ public: const DistributorConfiguration& getConfig(); - vdstestlib::DirConfig& getDirConfig() { - return _config; + vespa::config::content::core::StorDistributormanagerConfigBuilder& backing_config() noexcept { + return _config->distributor_manager_config(); } // TODO explicit notion of bucket spaces for tests @@ -237,7 +239,7 @@ public: void tag_content_node_supports_condition_probing(uint16_t index, bool supported); protected: - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestDistributorApp> _node; std::shared_ptr<DistributorMetricSet> _metrics; std::shared_ptr<IdealStateMetricSet> _ideal_state_metrics; diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 634e4993d53..33da4727017 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -94,7 +94,7 @@ struct ExternalOperationHandlerTest : Test, DistributorStripeTestUtil { TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { { createLinks(); - getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "16"); + backing_config().minsplitcount = 16; EXPECT_EQ(document::BucketId(16, 0xffff), operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( @@ -115,7 +115,7 @@ TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { close(); } { - getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "20"); + backing_config().minsplitcount = 20; createLinks(); EXPECT_EQ(document::BucketId(20, 0x11111), operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 0cadaa3fc9f..4639a154e74 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -74,10 +74,10 @@ TEST_F(IdealStateManagerTest, sibling) { TEST_F(IdealStateManagerTest, status_page) { close(); - getDirConfig().getConfig("stor-distributormanager").set("splitsize", "100"); - getDirConfig().getConfig("stor-distributormanager").set("splitcount", "1000000"); - getDirConfig().getConfig("stor-distributormanager").set("joinsize", "0"); - getDirConfig().getConfig("stor-distributormanager").set("joincount", "0"); + backing_config().splitsize = 100; + backing_config().splitcount = 1000000; + backing_config().joinsize = 0; + backing_config().joincount = 0; createLinks(); setup_stripe(1, 1, "distributor:1 storage:1"); diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index f0f8a4359fd..171dc5a42c0 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -27,8 +27,7 @@ struct OperationTargetResolverTest : Test, DistributorStripeTestUtil { const document::DocumentType* _html_type; std::unique_ptr<Operation> op; - BucketInstanceList getInstances(const BucketId& bid, - bool stripToRedundancy); + BucketInstanceList getInstances(const BucketId& bid, bool stripToRedundancy, bool symmetry_mode); void SetUp() override { _repo.reset(new document::DocumentTypeRepo( @@ -62,7 +61,7 @@ namespace { TestTargets::createTest(id, *this, *_asserters.back()) struct Asserter { - virtual ~Asserter() {} + virtual ~Asserter() = default; virtual void assertEqualMsg(std::string t1, OperationTargetList t2, OperationTargetList t3) = 0; @@ -73,21 +72,29 @@ struct TestTargets { OperationTargetList _expected; OperationTargetResolverTest& _test; Asserter& _asserter; + bool _symmetry_mode; TestTargets(const BucketId& id, OperationTargetResolverTest& test, Asserter& asserter) - : _id(id), _test(test), _asserter(asserter) {} + : _id(id), _test(test), _asserter(asserter), _symmetry_mode(true) + { + } ~TestTargets() { - BucketInstanceList result(_test.getInstances(_id, true)); - BucketInstanceList all(_test.getInstances(_id, false)); + BucketInstanceList result(_test.getInstances(_id, true, _symmetry_mode)); + BucketInstanceList all(_test.getInstances(_id, false, _symmetry_mode)); _asserter.assertEqualMsg( all.toString(), _expected, result.createTargets(makeBucketSpace())); delete _asserters.back(); _asserters.pop_back(); } + TestTargets& with_symmetric_replica_selection(bool symmetry) noexcept { + _symmetry_mode = symmetry; + return *this; + } + TestTargets& sendsTo(const BucketId& id, uint16_t node) { _expected.push_back(OperationTarget( makeDocumentBucket(id), lib::Node(lib::NodeType::STORAGE, node), false)); @@ -110,7 +117,7 @@ struct TestTargets { } // anonymous BucketInstanceList -OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy) +OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy, bool symmetry_mode) { auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); @@ -118,6 +125,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedund distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16, distributorBucketSpace.getDistribution().getRedundancy(), makeBucketSpace()); + resolver.use_symmetric_replica_selection(symmetry_mode); if (stripToRedundancy) { return resolver.getInstances(OperationTargetResolver::PUT, id); } else { @@ -143,14 +151,48 @@ TEST_F(OperationTargetResolverTest, choose_ideal_state_when_many_copies) { .sendsTo(BucketId(16, 0), 3); } -TEST_F(OperationTargetResolverTest, trusted_over_ideal_state) { +TEST_F(OperationTargetResolverTest, legacy_prefers_trusted_over_ideal_state) { setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0/0/0/t,1=0,2=0/0/0/t,3=0"); // ideal nodes: 1, 3 + MY_ASSERT_THAT(BucketId(32, 0)).with_symmetric_replica_selection(false) + .sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +TEST_F(OperationTargetResolverTest, prefer_ready_over_ideal_state_order) { + setup_stripe(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=1/2/3/u/i/r,1=1/2/3,2=1/2/3/u/i/r,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 are ready. MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) .sendsTo(BucketId(16, 0), 2); } +TEST_F(OperationTargetResolverTest, prefer_ready_over_ideal_state_order_also_when_retired) { + setup_stripe(2, 4, "storage:4 .0.s:r distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=1/2/3/u/i/r,1=1/2/3,2=1/2/3/u/i/r,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 are ready. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +TEST_F(OperationTargetResolverTest, prefer_replicas_with_more_docs_over_replicas_with_fewer_docs) { + setup_stripe(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=2/3/4,1=1/2/3,2=3/4/5,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 have more docs. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 2) + .sendsTo(BucketId(16, 0), 0); +} + +TEST_F(OperationTargetResolverTest, fall_back_to_active_state_and_db_index_if_all_other_fields_equal) { + // All replica nodes tagged as retired, which means none are part of the ideal state order + setup_stripe(2, 4, "storage:4 .0.s:r .2.s:r .3.s:r distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=2/3/4/u/a,3=2/3/4,2=2/3/4"); + // ideal nodes: 1, 3. 0 is active and 3 is the remaining replica with the lowest DB order. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 3); +} + TEST_F(OperationTargetResolverTest, choose_highest_split_bucket) { setup_stripe(2, 2, "storage:2 distributor:1"); // 0, 1 are both in ideal state for both buckets. diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp index cc23fa7a22e..c70ab533af6 100644 --- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/storage/distributor/statusreporterdelegate.h> @@ -45,8 +46,8 @@ public: } TEST(StatusReporterDelegateTest, delegate_invokes_delegator_on_status_request) { - vdstestlib::DirConfig config(getStandardConfig(false)); - TestDistributorApp app(config.getConfigId()); + auto config = StorageConfigSet::make_distributor_node_config(); + TestDistributorApp app(config->config_uri()); MockDelegator mockDelegator; MockStatusReporter reporter; diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index 94031c6d71e..e1b2bc93f62 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -24,7 +24,7 @@ TopLevelDistributorTestUtil::TopLevelDistributorTestUtil() : _message_sender(_sender, _sender_down), _num_distributor_stripes(4) { - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } TopLevelDistributorTestUtil::~TopLevelDistributorTestUtil() = default; @@ -32,7 +32,7 @@ TopLevelDistributorTestUtil::~TopLevelDistributorTestUtil() = default; void TopLevelDistributorTestUtil::create_links() { - _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); + _node = std::make_unique<TestDistributorApp>(_config->config_uri()); _thread_pool = framework::TickingThreadPool::createDefault("distributor", 100ms); _stripe_pool = DistributorStripePool::make_non_threaded_pool_for_testing(); _distributor.reset(new TopLevelDistributor( @@ -123,7 +123,7 @@ TopLevelDistributorTestUtil::close() } _sender.clear(); _node.reset(); - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } void diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h index 1d4c81a5bfb..51f0739e3e6 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.h +++ b/storage/src/tests/distributor/top_level_distributor_test_util.h @@ -3,6 +3,7 @@ #include "distributor_message_sender_stub.h" #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/storage/common/hostreporter/hostinfo.h> @@ -140,7 +141,7 @@ public: static std::vector<document::BucketSpace> bucket_spaces(); protected: - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestDistributorApp> _node; std::unique_ptr<framework::TickingThreadPool> _thread_pool; std::unique_ptr<DistributorStripePool> _stripe_pool; diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index 31ebbe19cbb..e00ce249298 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -49,13 +49,13 @@ struct UpdateOperationTest : Test, DistributorStripeTestUtil { const api::ReturnCode& result = api::ReturnCode()); std::shared_ptr<UpdateOperation> - sendUpdate(const std::string& bucketState, bool create_if_missing = false); + sendUpdate(const std::string& bucketState, bool create_if_missing = false, bool cache_create_flag = false); document::BucketId _bId; }; std::shared_ptr<UpdateOperation> -UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_missing) +UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_missing, bool cache_create_flag) { auto update = std::make_shared<document::DocumentUpdate>( *_repo, *_html_type, @@ -67,6 +67,9 @@ UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_m addNodesToBucketDB(_bId, bucketState); auto msg = std::make_shared<api::UpdateCommand>(makeDocumentBucket(document::BucketId(0)), update, 100); + if (cache_create_flag) { + msg->set_cached_create_if_missing(create_if_missing); + } return std::make_shared<UpdateOperation>( node_context(), operation_context(), getDistributorBucketSpace(), msg, std::vector<BucketDatabase::Entry>(), @@ -271,4 +274,20 @@ TEST_F(UpdateOperationTest, cancelled_nodes_are_not_updated_in_db) { dumpBucket(_bId)); } +TEST_F(UpdateOperationTest, cached_create_if_missing_is_propagated_to_fanout_requests) { + setup_stripe(1, 1, "distributor:1 storage:1"); + for (bool cache_flag : {false, true}) { + for (bool create_if_missing : {false, true}) { + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3", create_if_missing, cache_flag)); + DistributorMessageSenderStub sender; + cb->start(sender); + + ASSERT_EQ("Update => 0", sender.getCommands(true)); + auto& cmd = dynamic_cast<api::UpdateCommand&>(*sender.command(0)); + EXPECT_EQ(cmd.has_cached_create_if_missing(), cache_flag); + EXPECT_EQ(cmd.create_if_missing(), create_if_missing); + } + } +} + } |