Aggregate metrics from distributor stripes.

5 files changed, 86 insertions, 6 deletions

diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt
index 494a100f478..231361d72d6 100644
--- a/storage/src/vespa/storage/distributor/CMakeLists.txt
+++ b/storage/src/vespa/storage/distributor/CMakeLists.txt
@@ -23,6 +23,7 @@ vespa_add_library(storage_distributor
     distributor_stripe_component.cpp
     distributor_stripe_pool.cpp
     distributor_stripe_thread.cpp
+    distributor_total_metrics.cpp
     distributormessagesender.cpp
     distributormetricsset.cpp
     externaloperationhandler.cpp
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 6d911e56651..e6353e769d8 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -9,7 +9,7 @@
 #include "distributor_stripe.h"
 #include "distributor_stripe_pool.h"
 #include "distributor_stripe_thread.h"
-#include "distributormetricsset.h"
+#include "distributor_total_metrics.h"
 #include "idealstatemetricsset.h"
 #include "multi_threaded_stripe_access_guard.h"
 #include "operation_sequencer.h"
@@ -60,10 +60,11 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
       framework::StatusReporter("distributor", "Distributor"),
       _comp_reg(compReg),
       _metrics(std::make_shared<DistributorMetricSet>()),
+      _total_metrics((num_distributor_stripes == 0) ? std::shared_ptr<DistributorTotalMetrics>() : std::make_shared<DistributorTotalMetrics>(num_distributor_stripes)),
       _messageSender(messageSender),
       _use_legacy_mode(num_distributor_stripes == 0),
       _n_stripe_bits(0),
-      _stripe(std::make_unique<DistributorStripe>(compReg, *_metrics, node_identity, threadPool,
+      _stripe(std::make_unique<DistributorStripe>(compReg, _use_legacy_mode ? *_metrics : _total_metrics->stripe(0), node_identity, threadPool,
                                                   doneInitHandler, *this, *this, _use_legacy_mode)),
       _stripe_pool(stripe_pool),
       _stripes(),
@@ -91,7 +92,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
       _next_distribution(),
       _current_internal_config_generation(_component.internal_config_generation())
 {
-    _component.registerMetric(*_metrics);
+    _component.registerMetric(_use_legacy_mode ? *_metrics : *_total_metrics);
     _component.registerMetricUpdateHook(_metricUpdateHook, framework::SecondTime(0));
     if (!_use_legacy_mode) {
         assert(num_distributor_stripes == adjusted_num_stripes(num_distributor_stripes));
@@ -105,7 +106,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
                                                                *_stripe_accessor);
         _stripes.emplace_back(std::move(_stripe));
         for (size_t i = 1; i < num_distributor_stripes; ++i) {
-            _stripes.emplace_back(std::make_unique<DistributorStripe>(compReg, *_metrics, node_identity, threadPool,
+            _stripes.emplace_back(std::make_unique<DistributorStripe>(compReg, _total_metrics->stripe(i), node_identity, threadPool,
                                                                       doneInitHandler, *this, *this, _use_legacy_mode, i));
         }
         _stripe_scan_stats.resize(num_distributor_stripes);
@@ -124,6 +125,12 @@ Distributor::~Distributor()
     closeNextLink();
 }
 
+DistributorMetricSet&
+Distributor::getMetrics()
+{
+    return _use_legacy_mode ? *_metrics : _total_metrics->bucket_db_updater_metrics();
+}
+
 // TODO STRIPE remove
 DistributorStripe&
 Distributor::first_stripe() noexcept {
@@ -567,7 +574,10 @@ Distributor::propagateInternalScanMetricsToExternal()
     if (_use_legacy_mode) {
         _stripe->propagateInternalScanMetricsToExternal();
     } else {
-        first_stripe().propagateInternalScanMetricsToExternal();
+        for (auto &stripe : _stripes) {
+            stripe->propagateInternalScanMetricsToExternal();
+        }
+        _total_metrics->aggregate();
     }
 }
 
diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h
index 6f9654b78b8..d9910383a93 100644
--- a/storage/src/vespa/storage/distributor/distributor.h
+++ b/storage/src/vespa/storage/distributor/distributor.h
@@ -44,6 +44,7 @@ class DistributorBucketSpaceRepo;
 class DistributorStatus;
 class DistributorStripe;
 class DistributorStripePool;
+class DistributorTotalMetrics;
 class StripeAccessor;
 class OperationSequencer;
 class OwnershipTransferSafeTimePointCalculator;
@@ -78,7 +79,7 @@ public:
     void sendUp(const std::shared_ptr<api::StorageMessage>&) override;
     void sendDown(const std::shared_ptr<api::StorageMessage>&) override;
 
-    DistributorMetricSet& getMetrics() { return *_metrics; }
+    DistributorMetricSet& getMetrics();
 
     // Implements DistributorInterface and DistributorMessageSender.
     DistributorMetricSet& metrics() override { return getMetrics(); }
@@ -202,6 +203,7 @@ private:
 
     DistributorComponentRegister&         _comp_reg;
     std::shared_ptr<DistributorMetricSet> _metrics;
+    std::shared_ptr<DistributorTotalMetrics> _total_metrics;
     ChainedMessageSender*                 _messageSender;
     const bool                            _use_legacy_mode;
     // TODO STRIPE multiple stripes...! This is for proof of concept of wiring.
diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp
new file mode 100644
index 00000000000..543712cc4d2
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp
@@ -0,0 +1,40 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "distributor_total_metrics.h"
+
+namespace storage::distributor {
+
+DistributorTotalMetrics::DistributorTotalMetrics(uint32_t num_distributor_stripes)
+    : DistributorMetricSet(),
+      _stripes_metrics(),
+      _bucket_db_updater_metrics()
+{
+    _stripes_metrics.reserve(num_distributor_stripes);
+    for (uint32_t i = 0; i < num_distributor_stripes; ++i) {
+        _stripes_metrics.emplace_back(std::make_shared<DistributorMetricSet>());
+    }
+}
+
+DistributorTotalMetrics::~DistributorTotalMetrics() = default;
+
+void
+DistributorTotalMetrics::aggregate()
+{
+    DistributorMetricSet::reset();
+    _bucket_db_updater_metrics.addToPart(*this);
+    for (auto &stripe_metrics : _stripes_metrics) {
+        stripe_metrics->addToPart(*this);
+    }
+}
+
+void
+DistributorTotalMetrics::reset()
+{
+    DistributorMetricSet::reset();
+    _bucket_db_updater_metrics.reset();
+    for (auto &stripe_metrics : _stripes_metrics) {
+        stripe_metrics->reset();
+    }
+}
+
+}
diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.h b/storage/src/vespa/storage/distributor/distributor_total_metrics.h
new file mode 100644
index 00000000000..14116af3d3b
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.h
@@ -0,0 +1,27 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distributormetricsset.h"
+
+namespace storage::distributor {
+
+/*
+ * Class presenting total metrics (as a DistributorMetricSet) to the
+ * metric framework, while managing a DistributorMetricSet for each
+ * stripe and an extra one for the top level bucket db updater.
+ */
+class DistributorTotalMetrics : public DistributorMetricSet
+{
+    std::vector<std::shared_ptr<DistributorMetricSet>> _stripes_metrics;
+    DistributorMetricSet _bucket_db_updater_metrics;
+public:
+    explicit DistributorTotalMetrics(uint32_t num_distributor_stripes);
+    ~DistributorTotalMetrics() override;
+    void aggregate();
+    void reset() override;
+    DistributorMetricSet& stripe(uint32_t stripe_index) { return *_stripes_metrics[stripe_index]; }
+    DistributorMetricSet& bucket_db_updater_metrics() { return _bucket_db_updater_metrics; }
+};
+
+}