diff options
Diffstat (limited to 'storage')
16 files changed, 288 insertions, 240 deletions
diff --git a/storage/src/tests/distributor/bucketstateoperationtest.cpp b/storage/src/tests/distributor/bucketstateoperationtest.cpp index 4abf47cd210..34ab8e5ccdb 100644 --- a/storage/src/tests/distributor/bucketstateoperationtest.cpp +++ b/storage/src/tests/distributor/bucketstateoperationtest.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/document/test/make_document_bucket.h> @@ -12,7 +12,7 @@ using namespace ::testing; namespace storage::distributor { -struct BucketStateOperationTest : Test, DistributorTestUtil { +struct BucketStateOperationTest : Test, DistributorStripeTestUtil { void SetUp() override { createLinks(); } diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 6edaccc014b..2c0b34752c7 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -91,6 +91,17 @@ DistributorStripeTestUtil::setup_stripe(int redundancy, _stripe->update_distribution_config(new_configs); } +void +DistributorStripeTestUtil::set_redundancy(uint32_t redundancy) +{ + auto distribution = std::make_shared<lib::Distribution>( + lib::Distribution::getDefaultDistributionConfig(redundancy, 100)); + // Same rationale for not triggering a full distribution change as + // in setup_stripe() above + _node->getComponentRegister().setDistribution(distribution); + _stripe->propagateDefaultDistribution(std::move(distribution)); +} + std::shared_ptr<DistributorConfiguration> DistributorStripeTestUtil::make_config() const { @@ -398,6 +409,12 @@ DistributorStripeTestUtil::doc_selection_parser() const { return _stripe->_component; } +DistributorMetricSet& +DistributorStripeTestUtil::metrics() +{ + return *_metrics; +} + bool DistributorStripeTestUtil::tick() { diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index 07f878326ea..afe9e9da085 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -124,6 +124,7 @@ public: const storage::distributor::DistributorNodeContext& node_context() const; storage::distributor::DistributorStripeOperationContext& operation_context(); const DocumentSelectionParser& doc_selection_parser() const; + DistributorMetricSet& metrics(); bool tick(); @@ -168,6 +169,8 @@ public: uint32_t early_return = false, bool require_primary_to_be_written = true); + void set_redundancy(uint32_t redundancy); + using ConfigBuilder = vespa::config::content::core::StorDistributormanagerConfigBuilder; std::shared_ptr<DistributorConfiguration> make_config() const; diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index 751269fe586..89b6e4beea4 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -1,26 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storageapi/message/removelocation.h> -#include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> +#include "dummy_cluster_context.h" +#include <tests/distributor/distributor_stripe_test_util.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/storage/distributor/distributor.h> -#include <vespa/document/test/make_document_bucket.h> +#include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> +#include <vespa/storageapi/message/removelocation.h> #include <vespa/vespalib/gtest/gtest.h> -#include "dummy_cluster_context.h" using document::test::makeDocumentBucket; using namespace ::testing; namespace storage::distributor { -struct GarbageCollectionOperationTest : Test, DistributorTestUtil { +struct GarbageCollectionOperationTest : Test, DistributorStripeTestUtil { void SetUp() override { createLinks(); - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); addNodesToBucketDB(document::BucketId(16, 1), "0=250/50/300,1=250/50/300"); - getConfig().setGarbageCollection("music.date < 34", 3600s); + auto cfg = make_config(); + cfg->setGarbageCollection("music.date < 34", 3600s); + configure_stripe(cfg); getClock().setAbsoluteTimeInSeconds(34); }; diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index c853dd692a0..4860d7221a9 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/config/helper/configgetter.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/distributor_stripe.h> #include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/storage/distributor/operations/external/getoperation.h> -#include <tests/distributor/distributortestutil.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/document/test/make_document_bucket.h> -#include <vespa/config/helper/configgetter.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <iomanip> @@ -28,7 +28,7 @@ using namespace ::testing; namespace storage::distributor { -struct GetOperationTest : Test, DistributorTestUtil { +struct GetOperationTest : Test, DistributorStripeTestUtil { std::shared_ptr<const document::DocumentTypeRepo> _repo; document::DocumentId docId; @@ -59,7 +59,7 @@ struct GetOperationTest : Test, DistributorTestUtil { op = std::make_unique<GetOperation>( node_context(), getDistributorBucketSpace(), getDistributorBucketSpace().getBucketDatabase().acquire_read_guard(), - msg, getDistributor().getMetrics().gets, + msg, metrics().gets, consistency); op->start(_sender, framework::MilliSecTime(0)); } @@ -143,7 +143,7 @@ struct GetOperationTest : Test, DistributorTestUtil { } void setClusterState(const std::string& clusterState) { - enableDistributorClusterState(clusterState); + enable_cluster_state(clusterState); } void do_test_read_consistency_is_propagated(api::InternalReadConsistency consistency); @@ -415,7 +415,7 @@ TEST_F(GetOperationTest, not_found) { "timestamp 0) ReturnCode(NONE)", _sender.getLastReply()); - EXPECT_EQ(1, getDistributor().getMetrics().gets.failures.notfound.getValue()); + EXPECT_EQ(1, metrics().gets.failures.notfound.getValue()); EXPECT_FALSE(op->any_replicas_failed()); // "Not found" is not a failure. EXPECT_TRUE(last_reply_had_consistent_replicas()); EXPECT_TRUE(op->newest_replica().has_value()); diff --git a/storage/src/tests/distributor/joinbuckettest.cpp b/storage/src/tests/distributor/joinbuckettest.cpp index 9382648e881..920aa5e9e83 100644 --- a/storage/src/tests/distributor/joinbuckettest.cpp +++ b/storage/src/tests/distributor/joinbuckettest.cpp @@ -1,19 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storage/distributor/operations/idealstate/joinoperation.h> -#include <vespa/storage/distributor/distributor.h> -#include <tests/distributor/distributortestutil.h> +#include "dummy_cluster_context.h" +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/operations/idealstate/joinoperation.h> +#include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> -#include "dummy_cluster_context.h" using document::test::makeDocumentBucket; using namespace ::testing; namespace storage::distributor { -struct JoinOperationTest : Test, DistributorTestUtil { +struct JoinOperationTest : Test, DistributorStripeTestUtil { void checkSourceBucketsAndSendReply( JoinOperation& op, size_t msgIndex, @@ -29,13 +29,15 @@ struct JoinOperationTest : Test, DistributorTestUtil { }; TEST_F(JoinOperationTest, simple) { - getConfig().setJoinCount(100); - getConfig().setJoinSize(1000); + auto cfg = make_config(); + cfg->setJoinCount(100); + cfg->setJoinSize(1000); + configure_stripe(cfg); addNodesToBucketDB(document::BucketId(33, 1), "0=250/50/300"); addNodesToBucketDB(document::BucketId(33, 0x100000001), "0=300/40/200"); - enableDistributorClusterState("distributor:1 storage:1"); + enable_cluster_state("distributor:1 storage:1"); JoinOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(32, 0)), @@ -84,13 +86,15 @@ JoinOperationTest::checkSourceBucketsAndSendReply( * the buckets. */ TEST_F(JoinOperationTest, send_sparse_joins_to_nodes_without_both_source_buckets) { - getConfig().setJoinCount(100); - getConfig().setJoinSize(1000); + auto cfg = make_config(); + cfg->setJoinCount(100); + cfg->setJoinSize(1000); + configure_stripe(cfg); addNodesToBucketDB(document::BucketId(33, 1), "0=250/50/300,1=250/50/300"); addNodesToBucketDB(document::BucketId(33, 0x100000001), "0=300/40/200"); - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); JoinOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(32, 0)), diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 33baa572c64..b661075402a 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/document/test/make_document_bucket.h> +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/bucketdbupdater.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/operation_sequencer.h> +#include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/text/stringtokenizer.h> using document::test::makeDocumentBucket; using document::test::makeBucketSpace; @@ -23,7 +23,7 @@ namespace { vespalib::string _g_storage("storage"); } -struct MergeOperationTest : Test, DistributorTestUtil { +struct MergeOperationTest : Test, DistributorStripeTestUtil { OperationSequencer _operation_sequencer; void SetUp() override { @@ -45,7 +45,7 @@ TEST_F(MergeOperationTest, simple) { "1=20/1/1," "2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), toVector<uint16_t>(0, 1, 2))); @@ -73,7 +73,7 @@ TEST_F(MergeOperationTest, fail_if_source_only_copies_changed) { "1=20/1/1," "2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), toVector<uint16_t>(0, 1, 2))); @@ -236,7 +236,7 @@ TEST_F(MergeOperationTest, do_not_remove_copies_with_pending_messages) { document::BucketId bucket(16, 1); getClock().setAbsoluteTimeInSeconds(10); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); addNodesToBucketDB(bucket, "0=10/1/1/t," "1=20/1/1," @@ -300,7 +300,7 @@ TEST_F(MergeOperationTest, allow_deleting_active_source_only_replica) { "1=20/1/1/u/a," "2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); @@ -398,7 +398,7 @@ TEST_F(MergeOperationTest, mark_post_merge_redundant_replicas_source_only) { TEST_F(MergeOperationTest, merge_operation_is_blocked_by_any_busy_target_node) { getClock().setAbsoluteTimeInSeconds(10); addNodesToBucketDB(document::BucketId(16, 1), "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); @@ -423,7 +423,7 @@ TEST_F(MergeOperationTest, global_bucket_merges_are_not_blocked_by_busy_nodes) { getClock().setAbsoluteTimeInSeconds(10); document::BucketId bucket_id(16, 1); addNodesToBucketDB(bucket_id, "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); document::Bucket global_bucket(document::FixedBucketSpaces::global_space(), bucket_id); MergeOperation op(BucketAndNodes(global_bucket, toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); @@ -436,7 +436,7 @@ TEST_F(MergeOperationTest, global_bucket_merges_are_not_blocked_by_busy_nodes) { TEST_F(MergeOperationTest, merge_operation_is_blocked_by_locked_bucket) { getClock().setAbsoluteTimeInSeconds(10); addNodesToBucketDB(document::BucketId(16, 1), "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); @@ -447,7 +447,7 @@ TEST_F(MergeOperationTest, merge_operation_is_blocked_by_locked_bucket) { } TEST_F(MergeOperationTest, missing_replica_is_included_in_limited_node_list) { - setupDistributor(Redundancy(4), NodeCount(4), "distributor:1 storage:4"); + setup_stripe(Redundancy(4), NodeCount(4), "distributor:1 storage:4"); getClock().setAbsoluteTimeInSeconds(10); addNodesToBucketDB(document::BucketId(16, 1), "1=0/0/0/t,2=0/0/0/t,3=0/0/0/t"); const uint16_t max_merge_size = 2; @@ -466,7 +466,7 @@ TEST_F(MergeOperationTest, merge_operation_is_blocked_by_request_bucket_info_to_ getClock().setAbsoluteTimeInSeconds(10); document::BucketId bucket_id(16, 1); addNodesToBucketDB(bucket_id, "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(bucket_id), toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); @@ -487,7 +487,7 @@ TEST_F(MergeOperationTest, merge_operation_is_not_blocked_by_request_bucket_info document::BucketId bucket_id(16, 1); document::BucketId other_bucket_id(16, 2); addNodesToBucketDB(bucket_id, "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); MergeOperation op(BucketAndNodes(makeDocumentBucket(bucket_id), toVector<uint16_t>(0, 1, 2))); op.setIdealStateManager(&getIdealStateManager()); diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index aea251e81de..3d5c452054c 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/config/helper/configgetter.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storage/distributor/operationtargetresolverimpl.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/storage/distributor/distributor_bucket_space_repo.h> -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/operationtargetresolverimpl.h> -#include <vespa/storage/distributor/externaloperationhandler.h> -#include <vespa/config/helper/configgetter.hpp> +#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vespalib/gtest/gtest.h> using document::BucketId; @@ -24,7 +24,7 @@ using namespace ::testing; namespace storage::distributor { -struct OperationTargetResolverTest : Test, DistributorTestUtil { +struct OperationTargetResolverTest : Test, DistributorStripeTestUtil { std::shared_ptr<const document::DocumentTypeRepo> _repo; const document::DocumentType* _html_type; std::unique_ptr<Operation> op; @@ -135,7 +135,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, * Test basic case with no inconsistencies */ TEST_F(OperationTargetResolverTest, simple) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0,1=0"); MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 1) @@ -143,7 +143,7 @@ TEST_F(OperationTargetResolverTest, simple) { } TEST_F(OperationTargetResolverTest, multiple_nodes) { - setupDistributor(1, 2, "storage:2 distributor:1"); + setup_stripe(1, 2, "storage:2 distributor:1"); auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); @@ -161,14 +161,14 @@ TEST_F(OperationTargetResolverTest, multiple_nodes) { } TEST_F(OperationTargetResolverTest, choose_ideal_state_when_many_copies) { - setupDistributor(2, 4, "storage:4 distributor:1"); + setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0,1=0,2=0,3=0"); // ideal nodes: 1, 3 MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 1) .sendsTo(BucketId(16, 0), 3); } TEST_F(OperationTargetResolverTest, trusted_over_ideal_state) { - setupDistributor(2, 4, "storage:4 distributor:1"); + setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0/0/0/t,1=0,2=0/0/0/t,3=0"); // ideal nodes: 1, 3 MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) @@ -176,7 +176,7 @@ TEST_F(OperationTargetResolverTest, trusted_over_ideal_state) { } TEST_F(OperationTargetResolverTest, choose_highest_split_bucket) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); // 0, 1 are both in ideal state for both buckets. addNodesToBucketDB(BucketId(16, 0), "0=0,1=0"); addNodesToBucketDB(BucketId(17, 0), "0=0,1=0"); @@ -185,7 +185,7 @@ TEST_F(OperationTargetResolverTest, choose_highest_split_bucket) { } TEST_F(OperationTargetResolverTest, choose_highest_split_bucket_per_node) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "1=0"); addNodesToBucketDB(BucketId(17, 0), "0=0"); MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(17, 0), 0) @@ -193,7 +193,7 @@ TEST_F(OperationTargetResolverTest, choose_highest_split_bucket_per_node) { } TEST_F(OperationTargetResolverTest, choose_highest_split_bucket_with_trusted) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); // Unfinished split scenario: split done on 0, not on 1. // Copy on 1 is only remaining for (16, 0), so always trusted. addNodesToBucketDB(BucketId(16, 0), "1=1/2/3/t"); @@ -204,7 +204,7 @@ TEST_F(OperationTargetResolverTest, choose_highest_split_bucket_with_trusted) { } TEST_F(OperationTargetResolverTest, inconsistent_buckets_are_not_explicitly_created) { - setupDistributor(2, 2, "bits:8 storage:2 distributor:1"); + setup_stripe(2, 2, "bits:8 storage:2 distributor:1"); addNodesToBucketDB(BucketId(15, 0), "1=9/9/9/t"); addNodesToBucketDB(BucketId(16, 1 << 15), "0=9/9/9/t"); // (32, 0) belongs in (16, 0) subtree, but it does not exist. We cannot @@ -218,7 +218,7 @@ TEST_F(OperationTargetResolverTest, inconsistent_buckets_are_not_explicitly_crea } TEST_F(OperationTargetResolverTest, no_trusted_or_ideal_state_copy_available) { - setupDistributor(2, 4, "storage:4 distributor:1"); + setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0,2=0"); addNodesToBucketDB(BucketId(18, 0), "0=0"); // ideal nodes: 1, 3 MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(18, 0), 0) @@ -226,7 +226,7 @@ TEST_F(OperationTargetResolverTest, no_trusted_or_ideal_state_copy_available) { } TEST_F(OperationTargetResolverTest, create_missing_copies) { - setupDistributor(4, 10, "storage:10 distributor:1"); + setup_stripe(4, 10, "storage:10 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "6=0"); addNodesToBucketDB(BucketId(18, 0), "4=0"); // ideal nodes: 6, 8, 7, 1 @@ -237,21 +237,21 @@ TEST_F(OperationTargetResolverTest, create_missing_copies) { } TEST_F(OperationTargetResolverTest, no_existing_copies) { - setupDistributor(2, 5, "storage:5 distributor:1"); + setup_stripe(2, 5, "storage:5 distributor:1"); MY_ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 1) .createsAt(BucketId(16, 0), 3); } TEST_F(OperationTargetResolverTest, count_maintenance_nodes_as_down) { - setupDistributor(2, 5, "storage:5 .1.s:m distributor:1"); + setup_stripe(2, 5, "storage:5 .1.s:m distributor:1"); MY_ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 3) .createsAt(BucketId(16, 0), 2); } TEST_F(OperationTargetResolverTest, resolving_does_not_mutate_database) { - setupDistributor(2, 5, "storage:5 distributor:1"); + setup_stripe(2, 5, "storage:5 distributor:1"); MY_ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 1) .createsAt(BucketId(16, 0), 3); diff --git a/storage/src/tests/distributor/read_for_write_visitor_operation_test.cpp b/storage/src/tests/distributor/read_for_write_visitor_operation_test.cpp index e6f86f56d47..6a4351a2ec6 100644 --- a/storage/src/tests/distributor/read_for_write_visitor_operation_test.cpp +++ b/storage/src/tests/distributor/read_for_write_visitor_operation_test.cpp @@ -1,21 +1,21 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/documentupdate.h> #include <vespa/storage/common/reindexing_constants.h> -#include <vespa/storage/distributor/operations/external/read_for_write_visitor_operation.h> -#include <vespa/storage/distributor/operations/external/visitoroperation.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_stripe.h> #include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storage/distributor/operations/external/read_for_write_visitor_operation.h> +#include <vespa/storage/distributor/operations/external/visitoroperation.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/distributor/uuid_generator.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/visitor.h> -#include <tests/distributor/distributortestutil.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -46,7 +46,7 @@ struct MockUuidGenerator : UuidGenerator { } -struct ReadForWriteVisitorOperationStarterTest : Test, DistributorTestUtil { +struct ReadForWriteVisitorOperationStarterTest : Test, DistributorStripeTestUtil { document::TestDocMan _test_doc_man; VisitorOperation::Config _default_config; std::unique_ptr<OperationOwner> _op_owner; @@ -65,7 +65,7 @@ struct ReadForWriteVisitorOperationStarterTest : Test, DistributorTestUtil { void SetUp() override { createLinks(); - setupDistributor(1, 1, "version:1 distributor:1 storage:1"); + setup_stripe(1, 1, "version:1 distributor:1 storage:1"); _op_owner = std::make_unique<OperationOwner>(_sender, getClock()); _sender.setPendingMessageTracker(pending_message_tracker()); @@ -86,7 +86,7 @@ struct ReadForWriteVisitorOperationStarterTest : Test, DistributorTestUtil { return std::make_shared<VisitorOperation>( node_context(), operation_context(), getDistributorBucketSpace(), cmd, _default_config, - getDistributor().getMetrics().visits); + metrics().visits); } OperationSequencer& operation_sequencer() { diff --git a/storage/src/tests/distributor/removebucketoperationtest.cpp b/storage/src/tests/distributor/removebucketoperationtest.cpp index ed669865313..e1dc13ce2e1 100644 --- a/storage/src/tests/distributor/removebucketoperationtest.cpp +++ b/storage/src/tests/distributor/removebucketoperationtest.cpp @@ -1,23 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "dummy_cluster_context.h" #include <tests/common/dummystoragelink.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> -#include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/storage/distributor/distributor.h> -#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/gtest/gtest.h> -#include "dummy_cluster_context.h" using document::test::makeDocumentBucket; using namespace ::testing; namespace storage::distributor { -struct RemoveBucketOperationTest : Test, DistributorTestUtil { +struct RemoveBucketOperationTest : Test, DistributorStripeTestUtil { void SetUp() override { createLinks(); }; @@ -32,8 +32,8 @@ TEST_F(RemoveBucketOperationTest, simple) { "0=10/100/1/t," "1=10/100/1/t," "2=10/100/1/t"); - setRedundancy(1); - enableDistributorClusterState("distributor:1 storage:3"); + set_redundancy(1); + enable_cluster_state("distributor:1 storage:3"); RemoveBucketOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), @@ -65,7 +65,7 @@ TEST_F(RemoveBucketOperationTest, bucket_info_mismatch_failure) { getComponentRegisterImpl().setDistribution( std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(1, 10))); - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); RemoveBucketOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), @@ -100,7 +100,7 @@ TEST_F(RemoveBucketOperationTest, fail_with_invalid_bucket_info) { getComponentRegisterImpl().setDistribution( std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(1, 10))); - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); RemoveBucketOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), diff --git a/storage/src/tests/distributor/removelocationtest.cpp b/storage/src/tests/distributor/removelocationtest.cpp index 02164d67a46..ff48ed6f18a 100644 --- a/storage/src/tests/distributor/removelocationtest.cpp +++ b/storage/src/tests/distributor/removelocationtest.cpp @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storageapi/message/removelocation.h> -#include <vespa/storage/distributor/operations/external/removelocationoperation.h> -#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_stripe.h> +#include <vespa/storage/distributor/operations/external/removelocationoperation.h> +#include <vespa/storageapi/message/removelocation.h> #include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; @@ -13,7 +13,7 @@ using namespace ::testing; namespace storage::distributor { -struct RemoveLocationOperationTest : Test, DistributorTestUtil { +struct RemoveLocationOperationTest : Test, DistributorStripeTestUtil { std::unique_ptr<RemoveLocationOperation> op; void SetUp() override { @@ -33,15 +33,14 @@ struct RemoveLocationOperationTest : Test, DistributorTestUtil { doc_selection_parser(), getDistributorBucketSpace(), msg, - getDistributor().getMetrics(). - removelocations); + metrics().removelocations); op->start(_sender, framework::MilliSecTime(0)); } }; TEST_F(RemoveLocationOperationTest, simple) { - enableDistributorClusterState("distributor:1 storage:3"); + enable_cluster_state("distributor:1 storage:3"); addNodesToBucketDB(document::BucketId(34, 0x000001234), "0=1,1=1"); addNodesToBucketDB(document::BucketId(34, 0x100001234), "0=1,2=1"); diff --git a/storage/src/tests/distributor/removeoperationtest.cpp b/storage/src/tests/distributor/removeoperationtest.cpp index 77ecb5d7aeb..f7ea22f4537 100644 --- a/storage/src/tests/distributor/removeoperationtest.cpp +++ b/storage/src/tests/distributor/removeoperationtest.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <iomanip> -#include <tests/common/dummystoragelink.h> +#include <tests/distributor/distributor_stripe_test_util.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/storageapi/message/persistence.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/operations/external/removeoperation.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; @@ -15,7 +14,7 @@ using namespace ::testing; namespace storage::distributor { -struct RemoveOperationTest : Test, DistributorTestUtil { +struct RemoveOperationTest : Test, DistributorStripeTestUtil { document::DocumentId docId; document::BucketId bucketId; std::unique_ptr<RemoveOperation> op; @@ -25,7 +24,7 @@ struct RemoveOperationTest : Test, DistributorTestUtil { docId = document::DocumentId("id:test:test::uri"); bucketId = operation_context().make_split_bit_constrained_bucket_id(docId); - enableDistributorClusterState("distributor:1 storage:4"); + enable_cluster_state("distributor:1 storage:4"); }; void TearDown() override { @@ -40,8 +39,7 @@ struct RemoveOperationTest : Test, DistributorTestUtil { operation_context(), getDistributorBucketSpace(), msg, - getDistributor().getMetrics(). - removes); + metrics().removes); op->start(_sender, framework::MilliSecTime(0)); } @@ -146,7 +144,7 @@ TEST_F(RemoveOperationTest, multiple_copies) { } TEST_F(RemoveOperationTest, can_send_remove_when_all_replica_nodes_retired) { - enableDistributorClusterState("distributor:1 storage:1 .0.s:r"); + enable_cluster_state("distributor:1 storage:1 .0.s:r"); addNodesToBucketDB(bucketId, "0=123"); sendRemove(); diff --git a/storage/src/tests/distributor/splitbuckettest.cpp b/storage/src/tests/distributor/splitbuckettest.cpp index ec58992ed3e..d9052c36db3 100644 --- a/storage/src/tests/distributor/splitbuckettest.cpp +++ b/storage/src/tests/distributor/splitbuckettest.cpp @@ -1,4 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "dummy_cluster_context.h" +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/base/documentid.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/distributor.h> @@ -7,11 +9,8 @@ #include <vespa/storage/distributor/operations/idealstate/splitoperation.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/persistence.h> -#include <tests/common/dummystoragelink.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/gtest/gtest.h> -#include "dummy_cluster_context.h" +#include <vespa/vespalib/util/size_literals.h> using document::test::makeDocumentBucket; using namespace document; @@ -19,7 +18,7 @@ using namespace ::testing; namespace storage::distributor { -struct SplitOperationTest : Test, DistributorTestUtil { +struct SplitOperationTest : Test, DistributorStripeTestUtil { uint32_t splitByteSize; uint32_t tooLargeBucketSize; uint32_t splitCount; @@ -29,9 +28,10 @@ struct SplitOperationTest : Test, DistributorTestUtil { void SetUp() override { createLinks(); - getConfig().setSplitCount(splitCount); - getConfig().setSplitSize(splitByteSize); - + auto cfg = make_config(); + cfg->setSplitCount(splitCount); + cfg->setSplitSize(splitByteSize); + configure_stripe(cfg); } void TearDown() override { @@ -52,7 +52,7 @@ namespace { } TEST_F(SplitOperationTest, simple) { - enableDistributorClusterState("distributor:1 storage:1"); + enable_cluster_state("distributor:1 storage:1"); insertBucketInfo(document::BucketId(16, 1), 0, 0xabc, 1000, tooLargeBucketSize, 250); @@ -124,7 +124,7 @@ TEST_F(SplitOperationTest, multi_node_failure) { getBucketDatabase().update(entry); } - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); SplitOperation op(dummy_cluster_context, BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), @@ -198,7 +198,7 @@ TEST_F(SplitOperationTest, multi_node_failure) { } TEST_F(SplitOperationTest, copy_trusted_status_not_carried_over_after_split) { - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); document::BucketId sourceBucket(16, 1); /* @@ -263,7 +263,7 @@ TEST_F(SplitOperationTest, operation_blocked_by_pending_join) { clock.setAbsoluteTimeInSeconds(1); OperationSequencer op_seq; - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); document::BucketId joinTarget(2, 1); std::vector<document::BucketId> joinSources = { @@ -304,7 +304,7 @@ TEST_F(SplitOperationTest, split_is_blocked_by_locked_bucket) { clock.setAbsoluteTimeInSeconds(1); OperationSequencer op_seq; - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); document::BucketId source_bucket(16, 1); insertBucketInfo(source_bucket, 0, 0xabc, 1000, tooLargeBucketSize, 250); diff --git a/storage/src/tests/distributor/statoperationtest.cpp b/storage/src/tests/distributor/statoperationtest.cpp index be4fe414b8b..4148b9df114 100644 --- a/storage/src/tests/distributor/statoperationtest.cpp +++ b/storage/src/tests/distributor/statoperationtest.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <tests/common/dummystoragelink.h> -#include <vespa/storageapi/message/stat.h> -#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/document/test/make_document_bucket.h> -#include <vespa/storage/distributor/operations/external/statbucketoperation.h> -#include <vespa/storage/distributor/operations/external/statbucketlistoperation.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storage/distributor/operations/external/statbucketlistoperation.h> +#include <vespa/storage/distributor/operations/external/statbucketoperation.h> +#include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> @@ -16,7 +15,7 @@ using namespace ::testing; namespace storage::distributor { -struct StatOperationTest : Test, DistributorTestUtil { +struct StatOperationTest : Test, DistributorStripeTestUtil { void SetUp() override { createLinks(); }; @@ -27,7 +26,7 @@ struct StatOperationTest : Test, DistributorTestUtil { }; TEST_F(StatOperationTest, bucket_info) { - enableDistributorClusterState("distributor:1 storage:2"); + enable_cluster_state("distributor:1 storage:2"); addNodesToBucketDB(document::BucketId(16, 5), "0=4/2/100,1=4/2/100"); @@ -58,10 +57,12 @@ TEST_F(StatOperationTest, bucket_info) { } TEST_F(StatOperationTest, bucket_list) { - setupDistributor(2, 2, "distributor:1 storage:2"); + setup_stripe(2, 2, "distributor:1 storage:2"); - getConfig().setSplitCount(10); - getConfig().setSplitSize(100); + auto cfg = make_config(); + cfg->setSplitCount(10); + cfg->setSplitSize(100); + configure_stripe(cfg); for (uint32_t i = 0; i < 2; ++i) { insertBucketInfo(document::BucketId(16, 5), i, diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index ea170441a13..99e9a84af0c 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/config/helper/configgetter.h> -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/base/testdocrepo.h> -#include <vespa/document/update/arithmeticvalueupdate.h> -#include <vespa/storage/distributor/externaloperationhandler.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h> -#include <tests/distributor/distributortestutil.h> +#include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/document/fieldset/fieldsets.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> @@ -27,7 +27,7 @@ using namespace storage::api; using namespace storage::lib; using namespace ::testing; -struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil { +struct TwoPhaseUpdateOperationTest : Test, DistributorStripeTestUtil { document::TestDocRepo _testRepo; std::shared_ptr<const DocumentTypeRepo> _repo; const DocumentType* _doc_type; @@ -145,17 +145,19 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil { Timestamp expected_response_timestamp); std::shared_ptr<TwoPhaseUpdateOperation> set_up_2_inconsistent_replicas_and_start_update(bool enable_3phase = true) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(enable_3phase); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(enable_3phase); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); return cb; } void set_up_distributor_with_feed_blocked_state() { - setup_distributor(2, 2, - lib::ClusterStateBundle(lib::ClusterState("distributor:1 storage:2"), - {}, {true, "full disk"}, false)); + setup_stripe(2, 2, + lib::ClusterStateBundle(lib::ClusterState("distributor:1 storage:2"), + {}, {true, "full disk"}, false)); } }; @@ -334,11 +336,11 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, return std::make_shared<TwoPhaseUpdateOperation>( node_context(), operation_context(), doc_selection_parser(), - getDistributorBucketSpace(), msg, getDistributor().getMetrics()); + getDistributorBucketSpace(), msg, metrics()); } TEST_F(TwoPhaseUpdateOperationTest, simple) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); auto cb = sendUpdate("0=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -352,7 +354,7 @@ TEST_F(TwoPhaseUpdateOperationTest, simple) { } TEST_F(TwoPhaseUpdateOperationTest, non_existing) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); auto cb = sendUpdate(""); cb->start(_sender, framework::MilliSecTime(0)); @@ -362,7 +364,7 @@ TEST_F(TwoPhaseUpdateOperationTest, non_existing) { } TEST_F(TwoPhaseUpdateOperationTest, update_failed) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); auto cb = sendUpdate("0=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -377,7 +379,7 @@ TEST_F(TwoPhaseUpdateOperationTest, update_failed) { } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -404,7 +406,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps) { } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_not_found) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -425,7 +427,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_not_found) } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_update_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -442,7 +444,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_update_err } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_get_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -464,7 +466,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_get_error) } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -492,7 +494,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_error) } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_not_started) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -505,7 +507,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_not_st _sender.getLastCommand(true)); checkMessageSettingsPropagatedTo(_sender.commands().back()); - enableDistributorClusterState("storage:0 distributor:1"); + enable_cluster_state("storage:0 distributor:1"); ASSERT_TRUE(_sender.replies().empty()); replyToGet(*cb, _sender, 2, 110); @@ -517,7 +519,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_not_st } TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_inconsistent_split) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3", UpdateOptions().makeInconsistentSplit(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -558,7 +560,7 @@ TwoPhaseUpdateOperationTest::checkMessageSettingsPropagatedTo( } TEST_F(TwoPhaseUpdateOperationTest, fast_path_propagates_message_settings_to_update) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); auto cb = sendUpdate("0=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -569,7 +571,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_propagates_message_settings_to_upd } TEST_F(TwoPhaseUpdateOperationTest, n_of_m) { - setupDistributor(2, 2, "storage:2 distributor:1", 1); + setup_stripe(2, 2, "storage:2 distributor:1", 1); auto cb = sendUpdate("0=1/2/3,1=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -596,7 +598,7 @@ TwoPhaseUpdateOperationTest::getUpdatedValueFromLastPut( } TEST_F(TwoPhaseUpdateOperationTest, safe_path_updates_newest_received_document) { - setupDistributor(3, 3, "storage:3 distributor:1"); + setup_stripe(3, 3, "storage:3 distributor:1"); // 0,1 in sync. 2 out of sync. auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"); cb->start(_sender, framework::MilliSecTime(0)); @@ -628,7 +630,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_updates_newest_received_document) } TEST_F(TwoPhaseUpdateOperationTest, create_if_non_existent_creates_document_if_all_empty_gets) { - setupDistributor(3, 3, "storage:3 distributor:1"); + setup_stripe(3, 3, "storage:3 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -657,7 +659,7 @@ TEST_F(TwoPhaseUpdateOperationTest, create_if_non_existent_creates_document_if_a } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_has_failed_put) { - setupDistributor(3, 3, "storage:3 distributor:1"); + setup_stripe(3, 3, "storage:3 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -681,7 +683,7 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_has_failed_put) { } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_gets_fail) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -697,7 +699,7 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_gets_fail) { } TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_apply_throws_exception) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); // Create update for wrong doctype which will fail the update. auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().withError()); cb->start(_sender, framework::MilliSecTime(0)); @@ -717,7 +719,7 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_apply_throws_exception) { } TEST_F(TwoPhaseUpdateOperationTest, non_existing_with_auto_create) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -742,7 +744,7 @@ TEST_F(TwoPhaseUpdateOperationTest, non_existing_with_auto_create) { } TEST_F(TwoPhaseUpdateOperationTest, safe_path_fails_update_when_mismatching_timestamp_constraint) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().timestampToUpdate(1234)); cb->start(_sender, framework::MilliSecTime(0)); @@ -760,7 +762,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_fails_update_when_mismatching_time } TEST_F(TwoPhaseUpdateOperationTest, safe_path_update_propagates_message_settings_to_gets_and_puts) { - setupDistributor(3, 3, "storage:3 distributor:1"); + setup_stripe(3, 3, "storage:3 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"); cb->start(_sender, framework::MilliSecTime(0)); @@ -779,7 +781,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_update_propagates_message_settings } TEST_F(TwoPhaseUpdateOperationTest, safe_path_propagates_mbus_traces_from_replies) { - setupDistributor(3, 3, "storage:3 distributor:1"); + setup_stripe(3, 3, "storage:3 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"); cb->start(_sender, framework::MilliSecTime(0)); @@ -805,7 +807,7 @@ void TwoPhaseUpdateOperationTest::do_test_ownership_changed_between_gets_and_sec Timestamp highest_get_timestamp, Timestamp expected_response_timestamp) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); // Update towards inconsistent bucket invokes safe path. auto cb = sendUpdate("0=1/2/3,1=2/3/4"); cb->start(_sender, framework::MilliSecTime(0)); @@ -817,7 +819,7 @@ void TwoPhaseUpdateOperationTest::do_test_ownership_changed_between_gets_and_sec // this new state, the distributor no longer owns the bucket in question // and the operation should thus be failed. We must not try to send Puts // to a bucket we no longer own. - enableDistributorClusterState("storage:2 distributor:1 .0.s:d"); + enable_cluster_state("storage:2 distributor:1 .0.s:d"); getBucketDatabase().clear(); replyToGet(*cb, _sender, 0, lowest_get_timestamp); replyToGet(*cb, _sender, 1, highest_get_timestamp); @@ -848,7 +850,7 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_ownership_changes_between_ge } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_mismatch_fails_with_tas_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==120")); cb->start(_sender, framework::MilliSecTime(0)); @@ -864,7 +866,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_mismatch_fails_with_tas_ } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_match_sends_puts_with_updated_doc) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==110")); cb->start(_sender, framework::MilliSecTime(0)); @@ -874,7 +876,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_match_sends_puts_with_up } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_parse_failure_fails_with_illegal_params_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.san==fran...cisco")); cb->start(_sender, framework::MilliSecTime(0)); @@ -894,7 +896,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_parse_failure_fails_with } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_unknown_doc_type_fails_with_illegal_params_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("langbein.headerval=1234")); cb->start(_sender, framework::MilliSecTime(0)); @@ -913,7 +915,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_unknown_doc_type_fails_w } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_no_auto_create_fails_with_tas_error) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==120")); cb->start(_sender, framework::MilliSecTime(0)); @@ -929,7 +931,7 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_no_ } TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_auto_create_sends_puts) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions() .condition("testdoctype1.headerval==120") .createIfNonExistent(true)); @@ -953,7 +955,7 @@ TwoPhaseUpdateOperationTest::assertAbortedUpdateReplyWithContextPresent( } TEST_F(TwoPhaseUpdateOperationTest, fast_path_close_edge_sends_correct_reply) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); // Only 1 replica; consistent with itself by definition. auto cb = sendUpdate("0=1/2/3"); cb->start(_sender, framework::MilliSecTime(0)); @@ -971,7 +973,7 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_close_edge_sends_correct_reply) { } TEST_F(TwoPhaseUpdateOperationTest, safe_path_close_edge_sends_correct_reply) { - setupDistributor(2, 2, "storage:2 distributor:1"); + setup_stripe(2, 2, "storage:2 distributor:1"); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -988,8 +990,10 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_close_edge_sends_correct_reply) { } TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_restarts_with_fast_path_if_enabled) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1009,13 +1013,15 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_re "ReturnCode(NONE)", _sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(1, metrics.fast_path_restarts.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(1, m.fast_path_restarts.getValue()); } TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_does_not_restart_with_fast_path_if_disabled) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_update_fast_path_restart_enabled(false); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_update_fast_path_restart_enabled(false); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1028,13 +1034,15 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_do // Should _not_ be restarted with fast path, as it has been config disabled ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(0, metrics.fast_path_restarts.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(0, m.fast_path_restarts.getValue()); } TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_replica_set_altered_between_get_send_and_receive) { - setupDistributor(3, 3, "storage:3 distributor:1"); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(3, 3, "storage:3 distributor:1"); + auto cfg = make_config(); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1057,8 +1065,10 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_replica_set_alter } TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_document_not_found_on_a_replica_node) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1073,8 +1083,10 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_document_not_foun // Buckets must be created from scratch by Put operations, updates alone cannot do this. TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_no_initial_replicas_exist) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); // No replicas, technically consistent but cannot use fast path. auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true)); @@ -1087,8 +1099,10 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_no_initial_replic // by the client, not those indirectly initiated by the distributor in order to // fulfill update write-repairs. TEST_F(TwoPhaseUpdateOperationTest, update_gets_are_sent_with_strong_consistency_even_if_weak_consistency_configured) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_use_weak_internal_read_consistency_for_client_gets(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_use_weak_internal_read_consistency_for_client_gets(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1134,8 +1148,8 @@ TEST_F(ThreePhaseUpdateTest, full_document_get_sent_to_replica_with_highest_time reply_to_metadata_get(*cb, _sender, 0, 1000U); reply_to_metadata_get(*cb, _sender, 1, 2000U); - auto& metrics = getDistributor().getMetrics().update_metadata_gets; - EXPECT_EQ(1, metrics.ok.getValue()); // Technically tracks an entire operation covering multiple Gets. + auto& m = metrics().update_metadata_gets; + EXPECT_EQ(1, m.ok.getValue()); // Technically tracks an entire operation covering multiple Gets. // Node 1 has newest document version at ts=2000 ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2)); @@ -1155,8 +1169,8 @@ TEST_F(ThreePhaseUpdateTest, puts_are_sent_after_receiving_full_document_get) { replyToGet(*cb, _sender, 2, 2000U); ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 3)); - auto& metrics = getDistributor().getMetrics().update_gets; - EXPECT_EQ(1, metrics.ok.getValue()); + auto& m = metrics().update_gets; + EXPECT_EQ(1, m.ok.getValue()); } TEST_F(ThreePhaseUpdateTest, consistent_meta_get_timestamps_can_restart_in_fast_path) { @@ -1176,8 +1190,8 @@ TEST_F(ThreePhaseUpdateTest, consistent_meta_get_timestamps_can_restart_in_fast_ "ReturnCode(NONE)", _sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(1, metrics.fast_path_restarts.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(1, m.fast_path_restarts.getValue()); } TEST_F(ThreePhaseUpdateTest, fast_path_not_restarted_if_document_not_found_subset_of_replicas) { @@ -1196,8 +1210,8 @@ TEST_F(ThreePhaseUpdateTest, no_document_found_on_any_replicas_is_considered_con reply_to_metadata_get(*cb, _sender, 1, no_document_timestamp); ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true, false, 2)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(1, metrics.fast_path_restarts.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(1, m.fast_path_restarts.getValue()); } TEST_F(ThreePhaseUpdateTest, metadata_get_phase_fails_if_any_replicas_return_failure) { @@ -1215,8 +1229,10 @@ TEST_F(ThreePhaseUpdateTest, metadata_get_phase_fails_if_any_replicas_return_fai } TEST_F(ThreePhaseUpdateTest, update_failed_with_transient_error_code_if_replica_set_changed_after_metadata_gets) { - setupDistributor(3, 3, "storage:3 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + setup_stripe(3, 3, "storage:3 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // 2 replicas, room for 1 more. cb->start(_sender, framework::MilliSecTime(0)); // Add new replica to deterministic test bucket after gets have been sent @@ -1236,9 +1252,11 @@ TEST_F(ThreePhaseUpdateTest, update_failed_with_transient_error_code_if_replica_ } TEST_F(ThreePhaseUpdateTest, single_full_get_cannot_restart_in_fast_path) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas. cb->start(_sender, framework::MilliSecTime(0)); @@ -1280,7 +1298,7 @@ TEST_F(ThreePhaseUpdateTest, update_aborted_if_ownership_changed_between_gets_an ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); // See do_test_ownership_changed_between_gets_and_second_phase() for more in-depth // comments on why this particular cluster state is used. - enableDistributorClusterState("storage:2 distributor:1 .0.s:d"); + enable_cluster_state("storage:2 distributor:1 .0.s:d"); getBucketDatabase().clear(); reply_to_metadata_get(*cb, _sender, 0, api::Timestamp(70)); reply_to_metadata_get(*cb, _sender, 1, api::Timestamp(71)); @@ -1296,8 +1314,10 @@ TEST_F(ThreePhaseUpdateTest, update_aborted_if_ownership_changed_between_gets_an } TEST_F(ThreePhaseUpdateTest, safe_mode_is_implicitly_triggered_if_no_replicas_exist) { - setupDistributor(1, 1, "storage:1 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + setup_stripe(1, 1, "storage:1 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + configure_stripe(cfg); auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); @@ -1352,9 +1372,11 @@ TEST_F(ThreePhaseUpdateTest, single_full_get_reply_received_after_close_is_no_op } TEST_F(ThreePhaseUpdateTest, single_full_get_tombstone_is_no_op_without_auto_create) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4"); cb->start(_sender, framework::MilliSecTime(0)); @@ -1374,9 +1396,11 @@ TEST_F(ThreePhaseUpdateTest, single_full_get_tombstone_is_no_op_without_auto_cre } TEST_F(ThreePhaseUpdateTest, single_full_get_tombstone_sends_puts_with_auto_create) { - setupDistributor(2, 2, "storage:2 distributor:1"); - getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); - getConfig().set_update_fast_path_restart_enabled(true); + setup_stripe(2, 2, "storage:2 distributor:1"); + auto cfg = make_config(); + cfg->set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + cfg->set_update_fast_path_restart_enabled(true); + configure_stripe(cfg); auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().createIfNonExistent(true)); cb->start(_sender, framework::MilliSecTime(0)); diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index e31a5f0a768..a246a2054ac 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/storageapi/message/bucket.h> -#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/distributor_stripe_test_util.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/update/documentupdate.h> #include <vespa/document/test/make_document_bucket.h> -#include <vespa/storage/distributor/operations/external/updateoperation.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/config/helper/configgetter.hpp> +#include <vespa/storage/distributor/operations/external/updateoperation.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> #include <vespa/vespalib/gtest/gtest.h> using namespace document; @@ -26,7 +26,7 @@ using document::test::makeDocumentBucket; namespace storage::distributor { -struct UpdateOperationTest : Test, DistributorTestUtil { +struct UpdateOperationTest : Test, DistributorStripeTestUtil { std::shared_ptr<const DocumentTypeRepo> _repo; const DocumentType* _html_type; @@ -68,7 +68,7 @@ UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_m return std::make_shared<UpdateOperation>( node_context(), operation_context(), getDistributorBucketSpace(), msg, std::vector<BucketDatabase::Entry>(), - getDistributor().getMetrics().updates); + metrics().updates); } void @@ -87,7 +87,7 @@ UpdateOperationTest::replyToMessage(UpdateOperation& callback, DistributorMessag } TEST_F(UpdateOperationTest, simple) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); DistributorMessageSenderStub sender; @@ -101,12 +101,12 @@ TEST_F(UpdateOperationTest, simple) { "timestamp 100, timestamp of updated doc: 90) ReturnCode(NONE)", sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(0, metrics.diverging_timestamp_updates.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(0, m.diverging_timestamp_updates.getValue()); } TEST_F(UpdateOperationTest, not_found) { - setupDistributor(1, 1, "storage:1 distributor:1"); + setup_stripe(1, 1, "storage:1 distributor:1"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); DistributorMessageSenderStub sender; @@ -122,7 +122,7 @@ TEST_F(UpdateOperationTest, not_found) { } TEST_F(UpdateOperationTest, multi_node) { - setupDistributor(2, 2, "distributor:1 storage:2"); + setup_stripe(2, 2, "distributor:1 storage:2"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); DistributorMessageSenderStub sender; cb->start(sender, framework::MilliSecTime(0)); @@ -141,12 +141,12 @@ TEST_F(UpdateOperationTest, multi_node) { "node(idx=0,crc=0x2,docs=4/4,bytes=6/6,trusted=true,active=false,ready=false)", dumpBucket(_bId)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(0, metrics.diverging_timestamp_updates.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(0, m.diverging_timestamp_updates.getValue()); } TEST_F(UpdateOperationTest, multi_node_inconsistent_timestamp) { - setupDistributor(2, 2, "distributor:1 storage:2"); + setup_stripe(2, 2, "distributor:1 storage:2"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); DistributorMessageSenderStub sender; cb->start(sender, framework::MilliSecTime(0)); @@ -161,12 +161,12 @@ TEST_F(UpdateOperationTest, multi_node_inconsistent_timestamp) { "(best node 1)) ReturnCode(NONE)", sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(1, metrics.diverging_timestamp_updates.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(1, m.diverging_timestamp_updates.getValue()); } TEST_F(UpdateOperationTest, test_and_set_failures_increment_tas_metric) { - setupDistributor(2, 2, "distributor:1 storage:1"); + setup_stripe(2, 2, "distributor:1 storage:1"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); DistributorMessageSenderStub sender; cb->start(sender, framework::MilliSecTime(0)); @@ -179,8 +179,8 @@ TEST_F(UpdateOperationTest, test_and_set_failures_increment_tas_metric) { "ReturnCode(TEST_AND_SET_CONDITION_FAILED, bork bork)", sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(1, metrics.failures.test_and_set_failed.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(1, m.failures.test_and_set_failed.getValue()); } // Create-if-missing updates have a rather finicky behavior in the backend, wherein they'll @@ -195,7 +195,7 @@ TEST_F(UpdateOperationTest, test_and_set_failures_increment_tas_metric) { // of zero in this case, but this would cause complications during rolling upgrades that would // need explicit workaround logic anyway. TEST_F(UpdateOperationTest, create_if_missing_update_sentinel_timestamp_is_treated_as_zero_timestamp) { - setupDistributor(2, 2, "distributor:1 storage:2"); + setup_stripe(2, 2, "distributor:1 storage:2"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3", true)); DistributorMessageSenderStub sender; cb->start(sender, framework::MilliSecTime(0)); @@ -212,12 +212,12 @@ TEST_F(UpdateOperationTest, create_if_missing_update_sentinel_timestamp_is_treat "timestamp 100, timestamp of updated doc: 0) ReturnCode(NONE)", sender.getLastReply(true)); - auto& metrics = getDistributor().getMetrics().updates; - EXPECT_EQ(0, metrics.diverging_timestamp_updates.getValue()); + auto& m = metrics().updates; + EXPECT_EQ(0, m.diverging_timestamp_updates.getValue()); } TEST_F(UpdateOperationTest, inconsistent_create_if_missing_updates_picks_largest_non_auto_created_replica) { - setupDistributor(2, 3, "distributor:1 storage:3"); + setup_stripe(2, 3, "distributor:1 storage:3"); std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3,2=1/2/3", true)); DistributorMessageSenderStub sender; cb->start(sender, framework::MilliSecTime(0)); @@ -236,10 +236,10 @@ TEST_F(UpdateOperationTest, inconsistent_create_if_missing_updates_picks_largest EXPECT_NE(newest.first, BucketId()); EXPECT_EQ(newest.second, 1); - auto& metrics = getDistributor().getMetrics().updates; + auto& m = metrics().updates; // Implementation detail: since we get diverging results from nodes 2 and 1, these are // counted as separate diverging updates. - EXPECT_EQ(2, metrics.diverging_timestamp_updates.getValue()); + EXPECT_EQ(2, m.diverging_timestamp_updates.getValue()); } } |