diff options
author | Geir Storli <geirst@verizonmedia.com> | 2021-06-10 12:56:25 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2021-06-10 12:56:25 +0000 |
commit | b4de0d54d61e70f790753bcd948e61aba9d151db (patch) | |
tree | e660ea96524a88f0aa1f10b417fb02ab061302fb /storage | |
parent | b11f9f754f7e368e41d5c021cc4de20cc94b11da (diff) |
Implement aggregation across distributor stripes for min replica, bucket spaces, and pending maintenance stats.
Diffstat (limited to 'storage')
12 files changed, 316 insertions, 15 deletions
diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index e1010285dba..77b42ea9a94 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -11,10 +11,12 @@ namespace storage::distributor { -using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using End = vespalib::JsonStream::End; using File = vespalib::File; +using MinReplicaStats = std::unordered_map<uint16_t, uint32_t>; using Object = vespalib::JsonStream::Object; +using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; +using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; using namespace ::testing; struct DistributorHostInfoReporterTest : Test { @@ -35,7 +37,7 @@ namespace { // My kingdom for GoogleMock! struct MockedMinReplicaProvider : MinReplicaProvider { - std::unordered_map<uint16_t, uint32_t> minReplica; + MinReplicaStats minReplica; std::unordered_map<uint16_t, uint32_t> getMinReplica() const override { return minReplica; @@ -121,7 +123,7 @@ struct Fixture { TEST_F(DistributorHostInfoReporterTest, min_replica_stats_are_reported) { Fixture f; - std::unordered_map<uint16_t, uint32_t> minReplica; + MinReplicaStats minReplica; minReplica[0] = 2; minReplica[5] = 9; f.minReplicaProvider.minReplica = minReplica; @@ -133,10 +135,30 @@ TEST_F(DistributorHostInfoReporterTest, min_replica_stats_are_reported) { EXPECT_EQ(9, getMinReplica(root, 5)); } +TEST_F(DistributorHostInfoReporterTest, merge_min_replica_stats) { + + MinReplicaStats min_replica_a; + min_replica_a[3] = 2; + min_replica_a[5] = 4; + + MinReplicaStats min_replica_b; + min_replica_b[5] = 6; + min_replica_b[7] = 8; + + MinReplicaStats result; + merge_min_replica_stats(result, min_replica_a); + merge_min_replica_stats(result, min_replica_b); + + EXPECT_EQ(3, result.size()); + EXPECT_EQ(2, result[3]); + EXPECT_EQ(10, result[5]); + EXPECT_EQ(8, result[7]); +} + TEST_F(DistributorHostInfoReporterTest, generate_example_json) { Fixture f; - std::unordered_map<uint16_t, uint32_t> minReplica; + MinReplicaStats minReplica; minReplica[0] = 2; minReplica[5] = 9; f.minReplicaProvider.minReplica = minReplica; @@ -175,7 +197,7 @@ TEST_F(DistributorHostInfoReporterTest, no_report_generated_if_disabled) { Fixture f; f.reporter.enableReporting(false); - std::unordered_map<uint16_t, uint32_t> minReplica; + MinReplicaStats minReplica; minReplica[0] = 2; minReplica[5] = 9; f.minReplicaProvider.minReplica = minReplica; @@ -210,5 +232,41 @@ TEST_F(DistributorHostInfoReporterTest, bucket_spaces_stats_are_reported) { } } +TEST_F(DistributorHostInfoReporterTest, merge_per_node_bucket_spaces_stats) { + + PerNodeBucketSpacesStats stats_a; + stats_a[3]["default"] = BucketSpaceStats(3, 2); + stats_a[3]["global"] = BucketSpaceStats(5, 4); + stats_a[5]["default"] = BucketSpaceStats(7, 6); + stats_a[5]["global"] = BucketSpaceStats(9, 8); + + PerNodeBucketSpacesStats stats_b; + stats_b[5]["default"] = BucketSpaceStats(11, 10); + stats_b[5]["global"] = BucketSpaceStats(13, 12); + stats_b[7]["default"] = BucketSpaceStats(15, 14); + + PerNodeBucketSpacesStats result; + merge_per_node_bucket_spaces_stats(result, stats_a); + merge_per_node_bucket_spaces_stats(result, stats_b); + + PerNodeBucketSpacesStats exp; + exp[3]["default"] = BucketSpaceStats(3, 2); + exp[3]["global"] = BucketSpaceStats(5, 4); + exp[5]["default"] = BucketSpaceStats(7+11, 6+10); + exp[5]["global"] = BucketSpaceStats(9+13, 8+12); + exp[7]["default"] = BucketSpaceStats(15, 14); + + EXPECT_EQ(exp, result); } +TEST_F(DistributorHostInfoReporterTest, merge_bucket_space_stats_maintains_valid_flag) { + BucketSpaceStats stats_a(5, 3); + BucketSpaceStats stats_b; + + stats_a.merge(stats_b); + EXPECT_FALSE(stats_a.valid()); + EXPECT_EQ(5, stats_a.bucketsTotal()); + EXPECT_EQ(3, stats_a.bucketsPending()); +} + +} diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 58dc2430041..1bf3809b135 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/distributor/maintenancemocks.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/distributor_bucket_space_repo.h> @@ -209,4 +210,88 @@ TEST_F(SimpleMaintenanceScannerTest, per_node_maintenance_stats_are_tracked) { } } +TEST_F(SimpleMaintenanceScannerTest, merge_node_maintenance_stats) { + + NodeMaintenanceStats stats_a; + stats_a.movingOut = 1; + stats_a.syncing = 2; + stats_a.copyingIn = 3; + stats_a.copyingOut = 4; + stats_a.total = 5; + + NodeMaintenanceStats stats_b; + stats_b.movingOut = 10; + stats_b.syncing = 20; + stats_b.copyingIn = 30; + stats_b.copyingOut = 40; + stats_b.total = 50; + + NodeMaintenanceStats result; + result.merge(stats_a); + result.merge(stats_b); + + NodeMaintenanceStats exp; + exp.movingOut = 11; + exp.syncing = 22; + exp.copyingIn = 33; + exp.copyingOut = 44; + exp.total = 55; + EXPECT_EQ(exp, result); +} + +TEST_F(SimpleMaintenanceScannerTest, merge_pending_maintenance_stats) { + auto default_space = document::FixedBucketSpaces::default_space(); + auto global_space = document::FixedBucketSpaces::global_space(); + + PendingStats stats_a; + stats_a.global.pending[MaintenanceOperation::DELETE_BUCKET] = 1; + stats_a.global.pending[MaintenanceOperation::MERGE_BUCKET] = 2; + stats_a.global.pending[MaintenanceOperation::SPLIT_BUCKET] = 3; + stats_a.global.pending[MaintenanceOperation::JOIN_BUCKET] = 4; + stats_a.global.pending[MaintenanceOperation::SET_BUCKET_STATE] = 5; + stats_a.global.pending[MaintenanceOperation::GARBAGE_COLLECTION] = 6; + stats_a.perNodeStats.incMovingOut(3, default_space); + stats_a.perNodeStats.incSyncing(3, global_space); + stats_a.perNodeStats.incCopyingIn(5, default_space); + stats_a.perNodeStats.incCopyingOut(5, global_space); + stats_a.perNodeStats.incTotal(5, default_space); + + PendingStats stats_b; + stats_b.global.pending[MaintenanceOperation::DELETE_BUCKET] = 10; + stats_b.global.pending[MaintenanceOperation::MERGE_BUCKET] = 20; + stats_b.global.pending[MaintenanceOperation::SPLIT_BUCKET] = 30; + stats_b.global.pending[MaintenanceOperation::JOIN_BUCKET] = 40; + stats_b.global.pending[MaintenanceOperation::SET_BUCKET_STATE] = 50; + stats_b.global.pending[MaintenanceOperation::GARBAGE_COLLECTION] = 60; + stats_b.perNodeStats.incMovingOut(7, default_space); + stats_b.perNodeStats.incSyncing(7, global_space); + stats_b.perNodeStats.incCopyingIn(5, default_space); + stats_b.perNodeStats.incCopyingOut(5, global_space); + stats_b.perNodeStats.incTotal(5, default_space); + + PendingStats result; + result.merge(stats_a); + result.merge(stats_b); + + PendingStats exp; + exp.global.pending[MaintenanceOperation::DELETE_BUCKET] = 11; + exp.global.pending[MaintenanceOperation::MERGE_BUCKET] = 22; + exp.global.pending[MaintenanceOperation::SPLIT_BUCKET] = 33; + exp.global.pending[MaintenanceOperation::JOIN_BUCKET] = 44; + exp.global.pending[MaintenanceOperation::SET_BUCKET_STATE] = 55; + exp.global.pending[MaintenanceOperation::GARBAGE_COLLECTION] = 66; + exp.perNodeStats.incMovingOut(3, default_space); + exp.perNodeStats.incSyncing(3, global_space); + exp.perNodeStats.incCopyingIn(5, default_space); + exp.perNodeStats.incCopyingIn(5, default_space); + exp.perNodeStats.incCopyingOut(5, global_space); + exp.perNodeStats.incCopyingOut(5, global_space); + exp.perNodeStats.incTotal(5, default_space); + exp.perNodeStats.incTotal(5, default_space); + exp.perNodeStats.incMovingOut(7, default_space); + exp.perNodeStats.incSyncing(7, global_space); + EXPECT_EQ(exp.global, result.global); + EXPECT_EQ(exp.perNodeStats, result.perNodeStats); +} + } diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 7b048e9f109..494a100f478 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(storage_distributor bucket_space_distribution_configs.cpp bucket_space_distribution_context.cpp bucket_space_state_map.cpp + bucket_spaces_stats_provider.cpp bucketdbupdater.cpp bucketgctimecalculator.cpp bucketlistmerger.cpp @@ -29,6 +30,7 @@ vespa_add_library(storage_distributor idealstatemanager.cpp idealstatemetricsset.cpp messagetracker.cpp + min_replica_provider.cpp multi_threaded_stripe_access_guard.cpp nodeinfo.cpp operation_routing_snapshot.cpp diff --git a/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.cpp b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.cpp new file mode 100644 index 00000000000..2b12d437aaa --- /dev/null +++ b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.cpp @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucket_spaces_stats_provider.h" + +namespace storage::distributor { + +std::ostream& +operator<<(std::ostream& out, const BucketSpaceStats& stats) +{ + out << "{valid=" << stats.valid() << ", bucketsTotal=" << stats.bucketsTotal() << ", bucketsPending=" << stats.bucketsPending() << "}"; + return out; +} + +void +merge_bucket_spaces_stats(BucketSpacesStatsProvider::BucketSpacesStats& dest, + const BucketSpacesStatsProvider::BucketSpacesStats& src) +{ + for (const auto& entry : src) { + const auto& bucket_space_name = entry.first; + auto itr = dest.find(bucket_space_name); + if (itr != dest.end()) { + itr->second.merge(entry.second); + } else { + // We need to explicitly handle this case to avoid creating an empty BucketSpaceStats that is not valid. + dest[bucket_space_name] = entry.second; + } + } +} + +void +merge_per_node_bucket_spaces_stats(BucketSpacesStatsProvider::PerNodeBucketSpacesStats& dest, + const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& src) +{ + for (const auto& entry : src) { + auto node_index = entry.first; + merge_bucket_spaces_stats(dest[node_index], entry.second); + } +} + +} diff --git a/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h index 3d7b60f4471..c8ba04ed1ab 100644 --- a/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h +++ b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/string.h> #include <map> +#include <ostream> #include <unordered_map> namespace storage::distributor { @@ -32,8 +33,22 @@ public: bool valid() const noexcept { return _valid; } size_t bucketsTotal() const noexcept { return _bucketsTotal; } size_t bucketsPending() const noexcept { return _bucketsPending; } + + bool operator==(const BucketSpaceStats& rhs) const { + return (_valid == rhs._valid) && + (_bucketsTotal == rhs._bucketsTotal) && + (_bucketsPending == rhs._bucketsPending); + } + + void merge(const BucketSpaceStats& rhs) { + _valid = _valid && rhs._valid; + _bucketsTotal += rhs._bucketsTotal; + _bucketsPending += rhs._bucketsPending; + } }; +std::ostream& operator<<(std::ostream& out, const BucketSpaceStats& stats); + /** * Interface that provides snapshots of bucket spaces statistics per content node. */ @@ -48,4 +63,10 @@ public: virtual PerNodeBucketSpacesStats getBucketSpacesStats() const = 0; }; +void merge_bucket_spaces_stats(BucketSpacesStatsProvider::BucketSpacesStats& dest, + const BucketSpacesStatsProvider::BucketSpacesStats& src); + +void merge_per_node_bucket_spaces_stats(BucketSpacesStatsProvider::PerNodeBucketSpacesStats& dest, + const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& src); + } diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 7ac15b3929c..6d911e56651 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -521,32 +521,41 @@ Distributor::propagateDefaultDistribution( std::unordered_map<uint16_t, uint32_t> Distributor::getMinReplica() const { - // TODO STRIPE merged snapshot from all stripes if (_use_legacy_mode) { return _stripe->getMinReplica(); } else { - return first_stripe().getMinReplica(); + std::unordered_map<uint16_t, uint32_t> result; + for (const auto& stripe : _stripes) { + merge_min_replica_stats(result, stripe->getMinReplica()); + } + return result; } } BucketSpacesStatsProvider::PerNodeBucketSpacesStats Distributor::getBucketSpacesStats() const { - // TODO STRIPE merged snapshot from all stripes if (_use_legacy_mode) { return _stripe->getBucketSpacesStats(); } else { - return first_stripe().getBucketSpacesStats(); + BucketSpacesStatsProvider::PerNodeBucketSpacesStats result; + for (const auto& stripe : _stripes) { + merge_per_node_bucket_spaces_stats(result, stripe->getBucketSpacesStats()); + } + return result; } } SimpleMaintenanceScanner::PendingMaintenanceStats Distributor::pending_maintenance_stats() const { - // TODO STRIPE merged snapshot from all stripes if (_use_legacy_mode) { return _stripe->pending_maintenance_stats(); } else { - return first_stripe().pending_maintenance_stats(); + SimpleMaintenanceScanner::PendingMaintenanceStats result; + for (const auto& stripe : _stripes) { + result.merge(stripe->pending_maintenance_stats()); + } + return result; } } diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp index b954ef93c76..4e7f7d9d89d 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp @@ -7,6 +7,39 @@ namespace storage::distributor { const NodeMaintenanceStats NodeMaintenanceStatsTracker::_emptyNodeMaintenanceStats; +void +NodeMaintenanceStats::merge(const NodeMaintenanceStats& rhs) +{ + movingOut += rhs.movingOut; + syncing += rhs.syncing; + copyingIn += rhs.copyingIn; + copyingOut += rhs.copyingOut; + total += rhs.total; +} + +namespace { + +void +merge_bucket_spaces_stats(NodeMaintenanceStatsTracker::BucketSpacesStats& dest, + const NodeMaintenanceStatsTracker::BucketSpacesStats& src) +{ + for (const auto& entry : src) { + auto bucket_space = entry.first; + dest[bucket_space].merge(entry.second); + } +} + +} + +void +NodeMaintenanceStatsTracker::merge(const NodeMaintenanceStatsTracker& rhs) +{ + for (const auto& entry : rhs._stats) { + auto node_index = entry.first; + merge_bucket_spaces_stats(_stats[node_index], entry.second); + } +} + std::ostream& operator<<(std::ostream& os, const NodeMaintenanceStats& stats) { diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h index faf253fc84c..6399e53089b 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h @@ -37,6 +37,8 @@ struct NodeMaintenanceStats bool operator!=(const NodeMaintenanceStats& other) const noexcept { return !(*this == other); } + + void merge(const NodeMaintenanceStats& rhs); }; std::ostream& operator<<(std::ostream&, const NodeMaintenanceStats&); @@ -93,6 +95,11 @@ public: const PerNodeStats& perNodeStats() const { return _stats; } + + bool operator==(const NodeMaintenanceStatsTracker& rhs) const { + return _stats == rhs._stats; + } + void merge(const NodeMaintenanceStatsTracker& rhs); }; } // storage::distributor diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index 15a57c1e7ee..2bfce9569cc 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -19,6 +19,28 @@ SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucke SimpleMaintenanceScanner::~SimpleMaintenanceScanner() = default; +bool +SimpleMaintenanceScanner::GlobalMaintenanceStats::operator==(const GlobalMaintenanceStats& rhs) const +{ + return pending == rhs.pending; +} + +void +SimpleMaintenanceScanner::GlobalMaintenanceStats::merge(const GlobalMaintenanceStats& rhs) +{ + assert(pending.size() == rhs.pending.size()); + for (size_t i = 0; i < pending.size(); ++i) { + pending[i] += rhs.pending[i]; + } +} + +void +SimpleMaintenanceScanner::PendingMaintenanceStats::merge(const PendingMaintenanceStats& rhs) +{ + global.merge(rhs.global); + perNodeStats.merge(rhs.perNodeStats); +} + SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(const PendingMaintenanceStats &) = default; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index 254b3244171..69e63fd4c65 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -18,6 +18,9 @@ public: GlobalMaintenanceStats() : pending(MaintenanceOperation::OPERATION_COUNT) { } + + bool operator==(const GlobalMaintenanceStats& rhs) const; + void merge(const GlobalMaintenanceStats& rhs); }; struct PendingMaintenanceStats { PendingMaintenanceStats(); @@ -26,6 +29,8 @@ public: ~PendingMaintenanceStats(); GlobalMaintenanceStats global; NodeMaintenanceStatsTracker perNodeStats; + + void merge(const PendingMaintenanceStats& rhs); }; private: BucketPriorityDatabase& _bucketPriorityDb; diff --git a/storage/src/vespa/storage/distributor/min_replica_provider.cpp b/storage/src/vespa/storage/distributor/min_replica_provider.cpp new file mode 100644 index 00000000000..7fcc977fadf --- /dev/null +++ b/storage/src/vespa/storage/distributor/min_replica_provider.cpp @@ -0,0 +1,17 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "min_replica_provider.h" + +namespace storage::distributor { + +void +merge_min_replica_stats(std::unordered_map<uint16_t, uint32_t>& dest, + const std::unordered_map<uint16_t, uint32_t>& src) +{ + for (const auto& entry : src) { + auto node_index = entry.first; + dest[node_index] += entry.second; + } +} + +} diff --git a/storage/src/vespa/storage/distributor/min_replica_provider.h b/storage/src/vespa/storage/distributor/min_replica_provider.h index 6d644f4e9d4..ba946cd5a7f 100644 --- a/storage/src/vespa/storage/distributor/min_replica_provider.h +++ b/storage/src/vespa/storage/distributor/min_replica_provider.h @@ -4,8 +4,7 @@ #include <stdint.h> #include <unordered_map> -namespace storage { -namespace distributor { +namespace storage::distributor { class MinReplicaProvider { @@ -21,5 +20,8 @@ public: virtual std::unordered_map<uint16_t, uint32_t> getMinReplica() const = 0; }; -} // distributor -} // storage +void merge_min_replica_stats(std::unordered_map<uint16_t, uint32_t>& dest, + const std::unordered_map<uint16_t, uint32_t>& src); + +} + |