diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-12-16 10:37:51 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-12-16 10:37:51 +0100 |
commit | 77795c8db32ce03f0b32dbad84a550a60b736d23 (patch) | |
tree | de7c1858b4d7f904db9fb92f2cfaf78ea6bf6044 /storage | |
parent | b9166e24f9cd01144da00cdb0bcf7c1bfeb6d9ad (diff) | |
parent | 284765d452553caeecc222b513bfd5f3147ef950 (diff) |
Merge pull request #15809 from vespa-engine/toregge/handle-more-up-states-variants
Store more variants of ideal service layer nodes for a given bucket in hash table.
Diffstat (limited to 'storage')
12 files changed, 195 insertions, 127 deletions
diff --git a/storage/src/tests/distributor/distributor_bucket_space_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_test.cpp index e206308618f..8db7955b8a7 100644 --- a/storage/src/tests/distributor/distributor_bucket_space_test.cpp +++ b/storage/src/tests/distributor/distributor_bucket_space_test.cpp @@ -16,6 +16,8 @@ namespace { std::shared_ptr<ClusterState> stable_state(std::make_shared<ClusterState>("distributor:4 storage:4 bits:8")); std::shared_ptr<ClusterState> node_1_down_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:d bits:8")); +std::shared_ptr<ClusterState> node_1_retired_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:r bits:8")); +std::shared_ptr<ClusterState> node_1_maintenance_state(std::make_shared<ClusterState>("distributor:4 .1.s:d storage:4 .1.s:m bits:8")); std::shared_ptr<Distribution> distribution_r1(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(1, 4))); std::shared_ptr<Distribution> distribution_r2(std::make_shared<Distribution>(Distribution::getDefaultDistributionConfig(2, 4))); @@ -33,70 +35,59 @@ struct DistributorBucketSpaceTest : public ::testing::Test { } ~DistributorBucketSpaceTest() = default; + + // make normal buckets + std::vector<BucketId> make_normal_buckets(); + + // make deep split buckets. Ideal service layer nodes for a bucket changes for each split level when bucket used bits > 33. + std::vector<BucketId> make_deep_split_buckets(std::function<bool(BucketId)> owned); + // Count normal buckets using this distributor - uint32_t count_distributor_buckets(); + uint32_t count_distributor_buckets(const std::vector<BucketId>& buckets); // Count normal buckets using service layer node 0. - uint32_t count_storage_buckets(); - // Count deep split buckets using this distributor - uint32_t count_deep_split_distributor_buckets(); - // Count deep split buckets using service layer node 0. Ideal nodes for a bucket changes for each split level when bucket used bits > 33. - uint32_t count_deep_split_storage_buckets(); + CountVector count_service_layer_buckets(const std::vector<BucketId>& buckets); // Count normal buckets using this distributor and service layer node 0 CountVector count_buckets(); // Count deep split buckets using this distributor and service layer node 0. CountVector count_deep_split_buckets(); }; -uint32_t -DistributorBucketSpaceTest::count_distributor_buckets() +std::vector<BucketId> +DistributorBucketSpaceTest::make_normal_buckets() { - uint32_t owned_buckets = 0; + std::vector<BucketId> buckets; uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); for (uint32_t i = 0; i < (1u << distribution_bits); ++i) { - BucketId bucket(distribution_bits, i); - bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); - if (owned) { - ++owned_buckets; - } + buckets.emplace_back(distribution_bits, i); } - return owned_buckets; + return buckets; } -uint32_t -DistributorBucketSpaceTest::count_storage_buckets() +std::vector<BucketId> +DistributorBucketSpaceTest::make_deep_split_buckets(std::function<bool(BucketId)> owned) { - uint32_t owned_buckets = 0; - uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); - for (uint32_t i = 0; i < (1u << distribution_bits); ++i) { - BucketId bucket(distribution_bits, i); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket); - EXPECT_EQ(check_ideal_nodes, ideal_nodes); - for (auto node : ideal_nodes) { - if (node == 0u) { - ++owned_buckets; - } - } - } - return owned_buckets; -} - -uint32_t -DistributorBucketSpaceTest::count_deep_split_distributor_buckets() -{ - uint32_t owned_buckets = 0; + std::vector<BucketId> buckets; uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); uint32_t bias = 0; uint32_t bias_max = std::min(1u << distribution_bits, 1000u); for (; bias < bias_max; ++bias) { BucketId bucket(distribution_bits, bias); - if (bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned()) { + if (owned(bucket)) { break; } } assert(bias < bias_max); for (uint32_t i = 0; i < 100; ++i) { - BucketId bucket(42u, i * (1ul << 32) + bias); + buckets.emplace_back(42u, i * (1ul << 32) + bias); + } + return buckets; +} + +uint32_t +DistributorBucketSpaceTest::count_distributor_buckets(const std::vector<BucketId>& buckets) +{ + uint32_t owned_buckets = 0; + for (auto& bucket : buckets) { bool owned = bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); if (owned) { ++owned_buckets; @@ -105,47 +96,45 @@ DistributorBucketSpaceTest::count_deep_split_distributor_buckets() return owned_buckets; } -uint32_t -DistributorBucketSpaceTest::count_deep_split_storage_buckets() +DistributorBucketSpaceTest::CountVector +DistributorBucketSpaceTest::count_service_layer_buckets(const std::vector<BucketId>& buckets) { - uint32_t owned_buckets = 0; - uint16_t distribution_bits = bucket_space.getClusterState().getDistributionBitCount(); - uint32_t bias = 0; - uint32_t bias_max = std::min(1u << distribution_bits, 1000u); - for (; bias < bias_max; ++bias) { - BucketId bucket(distribution_bits, bias); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - bool found = false; - for (auto node : ideal_nodes) { - if (node == 0u) { - found = true; + CountVector result(3); + std::vector<uint16_t> ideal_nodes; + for (auto& bucket : buckets) { + auto &ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket); + for (uint32_t i = 0; i < 3; ++i) { + switch (i) { + case 0: + ideal_nodes = ideal_nodes_bundle.get_available_nodes(); + break; + case 1: + ideal_nodes = ideal_nodes_bundle.get_available_nonretired_nodes(); + break; + case 2: + ideal_nodes = ideal_nodes_bundle.get_available_nonretired_or_maintenance_nodes(); + break; + default: + ; } - } - if (found) { - break; - } - } - assert(bias < bias_max); - for (uint32_t i = 0; i < 100; ++i) { - BucketId bucket(42u, i * (1ul << 32) + bias); - auto ideal_nodes = bucket_space.get_ideal_nodes(bucket); - auto check_ideal_nodes = bucket_space.get_ideal_nodes_fallback(bucket); - EXPECT_EQ(check_ideal_nodes, ideal_nodes); - for (auto node : ideal_nodes) { - if (node == 0u) { - ++owned_buckets; + for (auto node : ideal_nodes) { + if (node == 0u) { + ++result[i]; + } } } } - return owned_buckets; + return result; } DistributorBucketSpaceTest::CountVector DistributorBucketSpaceTest::count_buckets() { CountVector result; - result.push_back(count_distributor_buckets()); - result.push_back(count_storage_buckets()); + auto buckets = make_normal_buckets(); + result.push_back(count_distributor_buckets(buckets)); + auto service_layer_result = count_service_layer_buckets(buckets); + result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend()); return result; } @@ -153,8 +142,10 @@ DistributorBucketSpaceTest::CountVector DistributorBucketSpaceTest::count_deep_split_buckets() { CountVector result; - result.push_back(count_deep_split_distributor_buckets()); - result.push_back(count_deep_split_storage_buckets()); + auto buckets = make_deep_split_buckets([this](BucketId bucket) { return bucket_space.check_ownership_in_pending_and_current_state(bucket).isOwned(); }); + result.push_back(count_distributor_buckets(buckets)); + auto service_layer_result = count_service_layer_buckets(buckets); + result.insert(result.end(), service_layer_result.cbegin(), service_layer_result.cend()); return result; } @@ -162,19 +153,24 @@ TEST_F(DistributorBucketSpaceTest, check_owned_buckets) { bucket_space.setDistribution(distribution_r1); bucket_space.setClusterState(stable_state); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.set_pending_cluster_state(node_1_down_state); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.setClusterState(node_1_down_state); bucket_space.set_pending_cluster_state({}); - EXPECT_EQ((CountVector{86u, 86u}), count_buckets()); + EXPECT_EQ((CountVector{86u, 86u, 86u, 86u}), count_buckets()); bucket_space.set_pending_cluster_state(stable_state); - EXPECT_EQ((CountVector{64u, 86u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 86u, 86u, 86u}), count_buckets()); bucket_space.setClusterState(stable_state); bucket_space.set_pending_cluster_state({}); - EXPECT_EQ((CountVector{64u, 64u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 64u, 64u, 64u}), count_buckets()); bucket_space.setDistribution(distribution_r2); - EXPECT_EQ((CountVector{64u, 125u}), count_buckets()); + EXPECT_EQ((CountVector{64u, 125u, 125u, 125u}), count_buckets()); + bucket_space.setClusterState(node_1_maintenance_state); + bucket_space.setDistribution(distribution_r1); + EXPECT_EQ((CountVector{86u, 86u, 86u, 64u}), count_buckets()); + bucket_space.setClusterState(node_1_retired_state); + EXPECT_EQ((CountVector{86u, 64u, 86u, 86u}), count_buckets()); } TEST_F(DistributorBucketSpaceTest, check_available_nodes) @@ -198,7 +194,7 @@ TEST_F(DistributorBucketSpaceTest, check_owned_deep_split_buckets) { bucket_space.setDistribution(distribution_r1); bucket_space.setClusterState(stable_state); - EXPECT_EQ((CountVector{100u, 19u}), count_deep_split_buckets()); + EXPECT_EQ((CountVector{100u, 19u, 19u, 19u}), count_deep_split_buckets()); } } diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index 3098d8382c8..721809d4515 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -120,7 +120,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState()); OperationTargetResolverImpl resolver( - distributorBucketSpace.getBucketDatabase(), idealNodeCalc, 16, + distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16, distributorBucketSpace.getDistribution().getRedundancy(), makeBucketSpace()); if (stripToRedundancy) { diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index 2a220cb9ef8..9838bd9be01 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -484,7 +484,7 @@ PutOperationTest::getNodes(const std::string& infoString) { std::vector<uint16_t> targetNodes; std::vector<uint16_t> createNodes; - PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_nodes(bid), + PutOperation::getTargetNodes(getDistributorBucketSpace().get_ideal_service_layer_nodes_bundle(bid).get_available_nodes(), targetNodes, createNodes, entry, 2); ost << "target( "; diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index dd301f0c284..f3ba6af6e0c 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_library(storage_distributor externaloperationhandler.cpp idealstatemanager.cpp idealstatemetricsset.cpp + ideal_service_layer_nodes_bundle.cpp messagetracker.cpp nodeinfo.cpp operation_routing_snapshot.cpp diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index f9122650311..784ae5ab8af 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -12,6 +12,8 @@ namespace storage::distributor { namespace { const char *up_states = "uri"; +const char *nonretired_up_states = "ui"; +const char *nonretired_or_maintenance_up_states = "uim"; } @@ -79,12 +81,6 @@ DistributorBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> clear(); } -std::vector<uint16_t> -DistributorBucketSpace::get_ideal_nodes_fallback(document::BucketId bucket) const -{ - return _distribution->getIdealStorageNodes(*_clusterState, bucket, up_states); -} - void DistributorBucketSpace::set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state) { @@ -118,20 +114,41 @@ DistributorBucketSpace::owns_bucket_in_state( return owns_bucket_in_state(*_distribution, clusterState, bucket); } -std::vector<uint16_t> -DistributorBucketSpace::get_ideal_nodes(document::BucketId bucket) const +namespace { + +void +setup_ideal_nodes_bundle(IdealServiceLayerNodesBundle& ideal_nodes_bundle, + const lib::Distribution& distribution, + const lib::ClusterState& cluster_state, + document::BucketId bucket) +{ + ideal_nodes_bundle.set_available_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, up_states)); + ideal_nodes_bundle.set_available_nonretired_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_up_states)); + ideal_nodes_bundle.set_available_nonretired_or_maintenance_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_or_maintenance_up_states)); +} + +// Ideal service layer nodes bundle used when bucket id used bits > 33. +thread_local IdealServiceLayerNodesBundle fallback_ideal_nodes_bundle; + +} + +const IdealServiceLayerNodesBundle& +DistributorBucketSpace::get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const { assert(bucket.getUsedBits() >= _distribution_bits); - if (bucket.getUsedBits() > 33) { // cf. storage::lib::Distribution::getStorageSeed - // Cannot map to super bucket ==> cannot cache result - return get_ideal_nodes_fallback(bucket); + if (bucket.getUsedBits() > 33) { + IdealServiceLayerNodesBundle &ideal_nodes_bundle = fallback_ideal_nodes_bundle; + setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, bucket); + return ideal_nodes_bundle; } - document::BucketId super_bucket(_distribution_bits, bucket.getId()); - auto itr = _ideal_nodes.find(super_bucket); + document::BucketId lookup_bucket((bucket.getUsedBits() > 33) ? bucket.getUsedBits() : _distribution_bits, bucket.getId()); + auto itr = _ideal_nodes.find(lookup_bucket); if (itr != _ideal_nodes.end()) { return itr->second; } - auto insres = _ideal_nodes.insert(std::make_pair(super_bucket, get_ideal_nodes_fallback(super_bucket))); + IdealServiceLayerNodesBundle ideal_nodes_bundle; + setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, lookup_bucket); + auto insres = _ideal_nodes.insert(std::make_pair(lookup_bucket, std::move(ideal_nodes_bundle))); assert(insres.second); return insres.first->second; } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index b3722ed9c91..f4d5bd6f5aa 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -3,6 +3,7 @@ #include "bucketownership.h" #include "bucket_ownership_flags.h" +#include "ideal_service_layer_nodes_bundle.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/stllike/hash_map.h> #include <memory> @@ -39,7 +40,7 @@ class DistributorBucketSpace { std::shared_ptr<const lib::ClusterState> _pending_cluster_state; std::vector<bool> _available_nodes; mutable vespalib::hash_map<document::BucketId, BucketOwnershipFlags, document::BucketId::hash> _ownerships; - mutable vespalib::hash_map<document::BucketId, std::vector<uint16_t>, document::BucketId::hash> _ideal_nodes; + mutable vespalib::hash_map<document::BucketId, IdealServiceLayerNodesBundle, document::BucketId::hash> _ideal_nodes; void clear(); void enumerate_available_nodes(); @@ -81,8 +82,6 @@ public: void set_pending_cluster_state(std::shared_ptr<const lib::ClusterState> pending_cluster_state); const lib::ClusterState& get_pending_cluster_state() const noexcept { return *_pending_cluster_state; } - std::vector<uint16_t> get_ideal_nodes_fallback(document::BucketId bucket) const; - /** * Returns true if this distributor owns the given bucket in the * given cluster and current distribution config. @@ -91,10 +90,11 @@ public: bool owns_bucket_in_state(const lib::ClusterState& clusterState, document::BucketId bucket) const; const std::vector<bool>& get_available_nodes() const { return _available_nodes; } + /** - * Returns the ideal nodes for the given bucket. + * Returns the ideal nodes bundle for the given bucket. */ - std::vector<uint16_t> get_ideal_nodes(document::BucketId bucket) const; + const IdealServiceLayerNodesBundle &get_ideal_service_layer_nodes_bundle(document::BucketId bucket) const; /* * Return bucket ownership flags for the given bucket. Bucket is always diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp index 86c98cc7b78..d2a2a6fc6a0 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp +++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp @@ -225,7 +225,7 @@ DistributorComponent::updateBucketDatabase( } } - UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_nodes(bucket.getBucketId()), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0); + UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0); bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (updateFlags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0); } diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp new file mode 100644 index 00000000000..069be02eb10 --- /dev/null +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp @@ -0,0 +1,17 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ideal_service_layer_nodes_bundle.h" +#include <vespa/vdslib/distribution/idealnodecalculator.h> + +namespace storage::distributor { + +IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept + : _available_nodes(), + _available_nonretired_nodes(), + _available_nonretired_or_maintenance_nodes() +{ +} + +IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; + +} diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h new file mode 100644 index 00000000000..2fd1bc8ad4b --- /dev/null +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <cstdint> + +namespace storage::distributor { + +/* + * Bundle of ideal service layer nodes for a bucket. + */ +class IdealServiceLayerNodesBundle { + std::vector<uint16_t> _available_nodes; + std::vector<uint16_t> _available_nonretired_nodes; + std::vector<uint16_t> _available_nonretired_or_maintenance_nodes; +public: + IdealServiceLayerNodesBundle() noexcept; + ~IdealServiceLayerNodesBundle(); + + void set_available_nodes(std::vector<uint16_t> available_nodes) { _available_nodes = std::move(available_nodes); } + void set_available_nonretired_nodes(std::vector<uint16_t> available_nonretired_nodes) { _available_nonretired_nodes = std::move(available_nonretired_nodes); } + void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); } + std::vector<uint16_t> get_available_nodes() const { return _available_nodes; } + std::vector<uint16_t> get_available_nonretired_nodes() const { return _available_nonretired_nodes; } + std::vector<uint16_t> get_available_nonretired_or_maintenance_nodes() const { return _available_nonretired_or_maintenance_nodes; } +}; + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 8a4f527c43a..7a247f6c524 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -108,7 +108,7 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi (void) multipleBuckets; BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(lastBucket)); std::vector<uint16_t> idealState( - _bucketSpace.getDistribution().getIdealStorageNodes(_bucketSpace.getClusterState(), lastBucket, "ui")); + _bucketSpace.get_ideal_service_layer_nodes_bundle(lastBucket).get_available_nodes()); active = ActiveCopy::calculate(idealState, _bucketSpace.getDistribution(), entry); LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str()); for (uint32_t i=0; i<active.size(); ++i) { @@ -182,10 +182,7 @@ PutOperation::onStart(DistributorMessageSender& sender) if (up) { std::vector<document::BucketId> bucketsToCheckForSplit; - lib::IdealNodeCalculatorImpl idealNodeCalculator; - idealNodeCalculator.setDistribution(_bucketSpace.getDistribution()); - idealNodeCalculator.setClusterState(_bucketSpace.getClusterState()); - OperationTargetResolverImpl targetResolver(_bucketSpace.getBucketDatabase(), idealNodeCalculator, + OperationTargetResolverImpl targetResolver(_bucketSpace, _bucketSpace.getBucketDatabase(), _op_ctx.distributor_config().getMinimalBucketSplit(), _bucketSpace.getDistribution().getRedundancy(), _msg->getBucket().getBucketSpace()); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 23bb6b1db78..7e2da056b5a 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "operationtargetresolverimpl.h" +#include "distributor_bucket_space.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/printable.hpp> #include <sstream> @@ -8,6 +9,20 @@ namespace storage::distributor { +namespace { + +lib::IdealNodeList +make_node_list(const std::vector<uint16_t>& nodes) +{ + lib::IdealNodeList list; + for (auto node : nodes) { + list.push_back(lib::Node(lib::NodeType::STORAGE, node)); + } + return list; +} + +} + BucketInstance::BucketInstance( const document::BucketId& id, const api::BucketInfo& info, lib::Node node, uint16_t idealLocationPriority, bool trusted, bool exist) @@ -54,15 +69,12 @@ BucketInstanceList::add(BucketDatabase::Entry& e, } void -BucketInstanceList::populate(const document::BucketId& specificId, BucketDatabase& db, - const lib::IdealNodeCalculator& idealNodeCalc) +BucketInstanceList::populate(const document::BucketId& specificId, const DistributorBucketSpace& distributor_bucket_space, BucketDatabase& db) { std::vector<BucketDatabase::Entry> entries; db.getParents(specificId, entries); for (uint32_t i=0; i<entries.size(); ++i) { - lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes( - entries[i].getBucketId(), - lib::IdealNodeCalculator::UpInitMaintenance)); + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entries[i].getBucketId()).get_available_nonretired_or_maintenance_nodes())); add(entries[i], idealNodes); } } @@ -108,17 +120,16 @@ BucketInstanceList::leastSpecificLeafBucketInSubtree( void BucketInstanceList::extendToEnoughCopies( + const DistributorBucketSpace& distributor_bucket_space, const BucketDatabase& db, const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId, - const lib::IdealNodeCalculator& idealNodeCalc) + const document::BucketId& mostSpecificId) { document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting : _instances[0]._bucket); newTarget = leastSpecificLeafBucketInSubtree(newTarget, mostSpecificId, db); - lib::IdealNodeList idealNodes(idealNodeCalc.getIdealStorageNodes( - newTarget, lib::IdealNodeCalculator::UpInit)); + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).get_available_nonretired_nodes())); for (uint32_t i=0; i<idealNodes.size(); ++i) { if (!contains(idealNodes[i])) { _instances.push_back(BucketInstance( @@ -182,14 +193,14 @@ OperationTargetResolverImpl::getAllInstances(OperationType type, { BucketInstanceList instances; if (type == PUT) { - instances.populate(id, _bucketDatabase, _idealNodeCalculator); + instances.populate(id, _distributor_bucket_space, _bucketDatabase); instances.sort(InstanceOrder()); instances.removeNodeDuplicates(); instances.extendToEnoughCopies( + _distributor_bucket_space, _bucketDatabase, _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id), - id, - _idealNodeCalculator); + id); } else { throw vespalib::IllegalArgumentException( "Unsupported operation type given", VESPA_STRLOC); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 73a2c281b18..a23c8ba7f59 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -9,6 +9,8 @@ namespace storage::distributor { +class DistributorBucketSpace; + struct BucketInstance : public vespalib::AsciiPrintable { document::BucketId _bucket; api::BucketInfo _info; @@ -57,13 +59,12 @@ public: * _instances.size() >= configured redundancy level, unless insufficient * number of nodes are available */ - void extendToEnoughCopies(const BucketDatabase& db, + void extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space, + const BucketDatabase& db, const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId, - const lib::IdealNodeCalculator& idealNodeCalc); + const document::BucketId& mostSpecificId); - void populate(const document::BucketId&, BucketDatabase&, - const lib::IdealNodeCalculator&); + void populate(const document::BucketId&, const DistributorBucketSpace&, BucketDatabase&); void add(BucketDatabase::Entry& e, const lib::IdealNodeList& idealState); template <typename Order> @@ -77,20 +78,20 @@ public: }; class OperationTargetResolverImpl : public OperationTargetResolver { + const DistributorBucketSpace& _distributor_bucket_space; BucketDatabase& _bucketDatabase; - const lib::IdealNodeCalculator& _idealNodeCalculator; uint32_t _minUsedBucketBits; uint16_t _redundancy; document::BucketSpace _bucketSpace; public: - OperationTargetResolverImpl(BucketDatabase& bucketDatabase, - const lib::IdealNodeCalculator& idealNodeCalc, + OperationTargetResolverImpl(const DistributorBucketSpace& distributor_bucket_space, + BucketDatabase& bucketDatabase, uint32_t minUsedBucketBits, uint16_t redundancy, document::BucketSpace bucketSpace) - : _bucketDatabase(bucketDatabase), - _idealNodeCalculator(idealNodeCalc), + : _distributor_bucket_space(distributor_bucket_space), + _bucketDatabase(bucketDatabase), _minUsedBucketBits(minUsedBucketBits), _redundancy(redundancy), _bucketSpace(bucketSpace) |