Merge pull request #12810 from vespa-engine/vekterli/add-distributor-bucket-db-memory-usage-metrics

Add distributor bucket db memory usage metrics

20 files changed, 187 insertions, 52 deletions

diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt
index 6cf1eadd6f7..6f854fed7c6 100644
--- a/metrics/CMakeLists.txt
+++ b/metrics/CMakeLists.txt
@@ -9,6 +9,7 @@ vespa_define_module(
 
     LIBS
     src/vespa/metrics
+    src/vespa/metrics/common
 
     TESTS
     src/tests
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt
index 96156dc84b0..0d7eeba3601 100644
--- a/metrics/src/vespa/metrics/CMakeLists.txt
+++ b/metrics/src/vespa/metrics/CMakeLists.txt
@@ -20,6 +20,7 @@ vespa_add_library(metrics
     valuemetric.cpp
     valuemetricvalues.cpp
     xmlwriter.cpp
+    $<TARGET_OBJECTS:metrics_common>
 
     INSTALL lib64
     DEPENDS
diff --git a/metrics/src/vespa/metrics/common/CMakeLists.txt b/metrics/src/vespa/metrics/common/CMakeLists.txt
new file mode 100644
index 00000000000..50183655dad
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(metrics_common OBJECT
+    SOURCES
+    memory_usage_metrics.cpp
+    DEPENDS
+)
diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp
new file mode 100644
index 00000000000..0c38e567749
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "memory_usage_metrics.h"
+#include <vespa/vespalib/util/memoryusage.h>
+
+namespace metrics {
+
+MemoryUsageMetrics::MemoryUsageMetrics(metrics::MetricSet* parent)
+    : MetricSet("memory_usage", {}, "The memory usage for a given component", parent),
+      _allocated_bytes("allocated_bytes", {}, "The number of allocated bytes", this),
+      _used_bytes("used_bytes", {}, "The number of used bytes (<= allocated_bytes)", this),
+      _dead_bytes("dead_bytes", {}, "The number of dead bytes (<= used_bytes)", this),
+      _on_hold_bytes("onhold_bytes", {}, "The number of bytes on hold", this)
+{
+}
+
+MemoryUsageMetrics::~MemoryUsageMetrics() = default;
+
+void
+MemoryUsageMetrics::update(const vespalib::MemoryUsage& usage)
+{
+    _allocated_bytes.set(usage.allocatedBytes());
+    _used_bytes.set(usage.usedBytes());
+    _dead_bytes.set(usage.deadBytes());
+    _on_hold_bytes.set(usage.allocatedBytesOnHold());
+}
+
+}
diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.h b/metrics/src/vespa/metrics/common/memory_usage_metrics.h
new file mode 100644
index 00000000000..7030db8c163
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.h
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+
+namespace vespalib { class MemoryUsage; }
+
+namespace metrics {
+
+/**
+ * Metric set for memory usage metrics.
+ */
+class MemoryUsageMetrics : public metrics::MetricSet {
+    metrics::LongValueMetric _allocated_bytes;
+    metrics::LongValueMetric _used_bytes;
+    metrics::LongValueMetric _dead_bytes;
+    metrics::LongValueMetric _on_hold_bytes;
+
+public:
+    explicit MemoryUsageMetrics(metrics::MetricSet* parent);
+    ~MemoryUsageMetrics() override;
+    void update(const vespalib::MemoryUsage& usage);
+};
+
+} // namespace metrics
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
index 6077a6ddd87..8a1dd7ea101 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
@@ -12,7 +12,6 @@ vespa_add_library(searchcore_proton_metrics STATIC
     job_tracker.cpp
     job_tracked_flush_target.cpp
     job_tracked_flush_task.cpp
-    memory_usage_metrics.cpp
     metrics_engine.cpp
     resource_usage_metrics.cpp
     sessionmanager_metrics.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp
deleted file mode 100644
index 1f687d14969..00000000000
--- a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memory_usage_metrics.h"
-#include <vespa/vespalib/util/memoryusage.h>
-
-namespace proton {
-
-MemoryUsageMetrics::MemoryUsageMetrics(metrics::MetricSet *parent)
-    : metrics::MetricSet("memory_usage", {}, "The memory usage for a given component", parent),
-      _allocatedBytes("allocated_bytes", {}, "The number of allocated bytes", this),
-      _usedBytes("used_bytes", {}, "The number of used bytes (<= allocatedbytes)", this),
-      _deadBytes("dead_bytes", {}, "The number of dead bytes (<= usedbytes)", this),
-      _onHoldBytes("onhold_bytes", {}, "The number of bytes on hold", this)
-{
-}
-
-MemoryUsageMetrics::~MemoryUsageMetrics() {}
-
-void
-MemoryUsageMetrics::update(const vespalib::MemoryUsage &usage)
-{
-    _allocatedBytes.set(usage.allocatedBytes());
-    _usedBytes.set(usage.usedBytes());
-    _deadBytes.set(usage.deadBytes());
-    _onHoldBytes.set(usage.allocatedBytesOnHold());
-}
-
-}
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
index 89177c3a359..e82e55848a5 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
@@ -2,27 +2,12 @@
 
 #pragma once
 
-#include <vespa/metrics/metrics.h>
+#include <vespa/metrics/common/memory_usage_metrics.h>
 
 namespace vespalib { class MemoryUsage; }
 
 namespace proton {
 
-/**
- * Metric set for memory usage metrics.
- */
-class MemoryUsageMetrics : public metrics::MetricSet
-{
-private:
-    metrics::LongValueMetric _allocatedBytes;
-    metrics::LongValueMetric _usedBytes;
-    metrics::LongValueMetric _deadBytes;
-    metrics::LongValueMetric _onHoldBytes;
-
-public:
-    MemoryUsageMetrics(metrics::MetricSet *parent);
-    ~MemoryUsageMetrics();
-    void update(const vespalib::MemoryUsage &usage);
-};
+using MemoryUsageMetrics = metrics::MemoryUsageMetrics;
 
 } // namespace proton
diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
index 1008d3ee4f2..e0c3cf161bb 100644
--- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
+++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/storage/distributor/distributormetricsset.h>
 #include <vespa/storage/distributor/idealstatemetricsset.h>
 #include <vespa/storage/config/config-stor-distributormanager.h>
+#include <vespa/vespalib/util/memoryusage.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <string>
 #include <sstream>
@@ -100,6 +101,40 @@ TEST_F(BucketDBMetricUpdaterTest, doc_and_byte_counts_are_updated) {
     EXPECT_EQ(34, dms.bytesStored.getLast());
 }
 
+TEST_F(BucketDBMetricUpdaterTest, bucket_db_memory_usage_metrics_are_updated) {
+    BucketDBMetricUpdater metric_updater;
+    IdealStateMetricSet ims;
+    DistributorMetricSet dms(_loadTypes);
+
+    vespalib::MemoryUsage mem_usage;
+    mem_usage.incAllocatedBytes(1000);
+    mem_usage.incDeadBytes(700);
+    metric_updater.update_db_memory_usage(mem_usage, true);
+
+    mem_usage.incAllocatedBytes(500);
+    mem_usage.incDeadBytes(100);
+    metric_updater.update_db_memory_usage(mem_usage, false);
+
+    metric_updater.completeRound(false);
+    metric_updater.getLastCompleteStats().propagateMetrics(ims, dms);
+
+    auto* m = dms.mutable_dbs.memory_usage.getMetric("allocated_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 1000);
+
+    m = dms.mutable_dbs.memory_usage.getMetric("dead_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 700);
+
+    m = dms.read_only_dbs.memory_usage.getMetric("allocated_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 1500);
+
+    m = dms.read_only_dbs.memory_usage.getMetric("dead_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 800);
+}
+
 TEST_F(BucketDBMetricUpdaterTest, buckets_with_too_few_and_too_many_copies) {
     BucketDBMetricUpdater metricUpdater;
     IdealStateMetricSet ims;
diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
index 66d44a655e0..85f5c8c5be9 100644
--- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
+++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
@@ -526,6 +526,12 @@ void BTreeBucketDatabase::print(std::ostream& out, bool verbose,
     (void)indent;
 }
 
+vespalib::MemoryUsage BTreeBucketDatabase::memory_usage() const noexcept {
+    auto mem_usage = _tree.getMemoryUsage();
+    mem_usage.merge(_store.getMemoryUsage());
+    return mem_usage;
+}
+
 BTreeBucketDatabase::ReadGuardImpl::ReadGuardImpl(const BTreeBucketDatabase& db)
     : _db(&db),
       _guard(_db->_generation_handler.takeGuard()),
diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
index 1f2b25814a8..8898b0c395a 100644
--- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
+++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
@@ -91,6 +91,8 @@ public:
     std::unique_ptr<ReadGuard> acquire_read_guard() const override {
         return std::make_unique<ReadGuardImpl>(*this);
     }
+
+    vespalib::MemoryUsage memory_usage() const noexcept override;
 };
 
 }
diff --git a/storage/src/vespa/storage/bucketdb/bucketdatabase.h b/storage/src/vespa/storage/bucketdb/bucketdatabase.h
index 46aaaa997d9..2dbcdd194ef 100644
--- a/storage/src/vespa/storage/bucketdb/bucketdatabase.h
+++ b/storage/src/vespa/storage/bucketdb/bucketdatabase.h
@@ -7,6 +7,7 @@
 #include <vespa/vespalib/util/printable.h>
 #include <vespa/storage/bucketdb/bucketinfo.h>
 #include <vespa/document/bucket/bucketid.h>
+#include <vespa/vespalib/util/memoryusage.h>
 
 namespace storage {
 
@@ -250,6 +251,8 @@ public:
     virtual std::unique_ptr<ReadGuard> acquire_read_guard() const {
         return std::unique_ptr<ReadGuard>();
     }
+
+    [[nodiscard]] virtual vespalib::MemoryUsage memory_usage() const noexcept = 0;
 };
 
 template <typename BucketInfoType>
diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
index 463e4a4b8ce..edb808da294 100644
--- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
+++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
@@ -594,4 +594,30 @@ void MapBucketDatabase::ReadGuardImpl::find_parents_and_self(const document::Buc
     _db->getParents(bucket, entries);
 }
 
+namespace {
+
+template <typename T>
+size_t allocated_by_vec(const T& vec) noexcept {
+    return (vec.capacity() * sizeof(typename T::value_type));
+}
+
+template <typename T>
+size_t used_by_vec(const T& vec) noexcept {
+    return (vec.size() * sizeof(typename T::value_type));
+}
+
+}
+
+vespalib::MemoryUsage MapBucketDatabase::memory_usage() const noexcept {
+    // We don't have a concept of hold lists here, nor do we know the exact size of the
+    // entries on our free list (these wrap a secondary replica vector allocation).
+    // So we fudge the numbers a bit, returning a lower bound approximation only.
+    // That's OK since this is a legacy database that's on the way out anyway.
+    vespalib::MemoryUsage mem_usage;
+    mem_usage.incAllocatedBytes(allocated_by_vec(_values) + allocated_by_vec(_db));
+    mem_usage.incUsedBytes(used_by_vec(_values) + used_by_vec(_db));
+    mem_usage.incDeadBytes(allocated_by_vec(_free) + allocated_by_vec(_freeValues));
+    return mem_usage;
+}
+
 } // storage
diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
index 9fe5e2d7740..e41b797a321 100644
--- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
+++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
@@ -30,6 +30,7 @@ public:
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
     std::unique_ptr<ReadGuard> acquire_read_guard() const override;
+    vespalib::MemoryUsage memory_usage() const noexcept override;
 private:
     struct E {
         E() : value(-1), e_0(-1), e_1(-1) {};
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
index c211e775326..51eda0f948b 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
@@ -132,6 +132,8 @@ BucketDBMetricUpdater::Stats::propagateMetrics(
 {
     distributorMetrics.docsStored.set(_docCount);
     distributorMetrics.bytesStored.set(_byteCount);
+    distributorMetrics.mutable_dbs.memory_usage.update(_mutable_db_mem_usage);
+    distributorMetrics.read_only_dbs.memory_usage.update(_read_only_db_mem_usage);
 
     idealStateMetrics.buckets_toofewcopies.set(_tooFewCopies);
     idealStateMetrics.buckets_toomanycopies.set(_tooManyCopies);
@@ -145,4 +147,10 @@ BucketDBMetricUpdater::reset()
     resetStats();
 }
 
+void BucketDBMetricUpdater::update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db) {
+    auto& target = (is_mutable_db ? _workingStats._mutable_db_mem_usage
+                                  : _workingStats._read_only_db_mem_usage);
+    target.merge(mem_usage);
+}
+
 } // storage::distributor
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
index 7ef8479866f..766307f49c2 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
@@ -4,7 +4,7 @@
 
 #include <vespa/storage/bucketdb/bucketdatabase.h>
 #include <vespa/storage/config/config-stor-distributormanager.h>
-
+#include <vespa/vespalib/util/memoryusage.h>
 #include <unordered_map>
 
 namespace storage::distributor {
@@ -22,10 +22,12 @@ public:
         uint64_t _tooManyCopies;
         uint64_t _noTrusted;
         uint64_t _totalBuckets;
+        vespalib::MemoryUsage _mutable_db_mem_usage;
+        vespalib::MemoryUsage _read_only_db_mem_usage;
 
         Stats();
         Stats(const Stats &rhs);
-        ~Stats() { }
+        ~Stats() = default;
 
         Stats &operator=(const Stats &rhs) = default;
 
@@ -63,7 +65,6 @@ private:
 
 public:
     BucketDBMetricUpdater();
-
     ~BucketDBMetricUpdater();
 
     void setMinimumReplicaCountingMode(ReplicaCountingMode mode) noexcept {
@@ -98,6 +99,8 @@ public:
         return _lastCompleteStats;
     }
 
+    void update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db);
+
 private:
     void updateMinReplicationStats(const BucketDatabase::Entry& entry, uint32_t trustedCopies);
 
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 988af43a7be..3cca2847671 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -13,6 +13,7 @@
 #include <vespa/storage/common/global_bucket_space_distribution_converter.h>
 #include <vespa/storageframework/generic/status/xmlstatusreporter.h>
 #include <vespa/document/bucket/fixed_bucket_spaces.h>
+#include <vespa/vespalib/util/memoryusage.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor-main");
@@ -768,6 +769,16 @@ Distributor::updateInternalMetricsForCompletedScan()
         _must_send_updated_host_info = true;
     }
     _bucketSpacesStats = std::move(new_space_stats);
+    update_bucket_db_memory_usage_stats();
+}
+
+void Distributor::update_bucket_db_memory_usage_stats() {
+    for (auto& space : *_bucketSpaceRepo) {
+        _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), true);
+    }
+    for (auto& space : *_readOnlyBucketSpaceRepo) {
+        _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), false);
+    }
 }
 
 void
diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h
index ac6e306a4fb..36e34592cba 100644
--- a/storage/src/vespa/storage/distributor/distributor.h
+++ b/storage/src/vespa/storage/distributor/distributor.h
@@ -226,6 +226,7 @@ private:
      * Takes metric lock.
      */
     void updateInternalMetricsForCompletedScan();
+    void update_bucket_db_memory_usage_stats();
     void scanAllBuckets();
     MaintenanceScanner::ScanResult scanNextBucket();
     void enableNextConfig();
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
index 8266aeb29cd..70ab5229311 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
@@ -2,11 +2,19 @@
 #include "distributormetricsset.h"
 #include <vespa/metrics/loadmetric.hpp>
 #include <vespa/metrics/summetric.hpp>
+#include <vespa/vespalib/util/memoryusage.h>
 
 namespace storage::distributor {
 
 using metrics::MetricSet;
 
+BucketDbMetrics::BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner)
+    : metrics::MetricSet("bucket_db", {{"bucket_db_type", db_type}}, "", owner),
+      memory_usage(this)
+{}
+
+BucketDbMetrics::~BucketDbMetrics() = default;
+
 DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt)
     : MetricSet("distributor", {{"distributor"}}, ""),
       puts(lt, PersistenceOperationMetricSet("puts"), this),
@@ -41,7 +49,9 @@ DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt)
       bytesStored("bytesstored",
               {{"logdefault"},{"yamasdefault"}},
               "Number of bytes stored in all buckets controlled by "
-              "this distributor", this)
+              "this distributor", this),
+      mutable_dbs("mutable", this),
+      read_only_dbs("read_only", this)
 {
     docsStored.logOnlyIfSet();
     bytesStored.logOnlyIfSet();
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h
index d9c0711fd14..ce4025d8311 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.h
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.h
@@ -5,12 +5,21 @@
 #include "update_metric_set.h"
 #include "visitormetricsset.h"
 #include <vespa/metrics/metrics.h>
+#include <vespa/metrics/common/memory_usage_metrics.h>
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
 
+namespace vespalib { class MemoryUsage; }
+
 namespace storage::distributor {
 
-class DistributorMetricSet : public metrics::MetricSet
-{
+struct BucketDbMetrics : metrics::MetricSet {
+    BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner);
+    ~BucketDbMetrics() override;
+
+    metrics::MemoryUsageMetrics memory_usage;
+};
+
+class DistributorMetricSet : public metrics::MetricSet {
 public:
     metrics::LoadMetric<PersistenceOperationMetricSet> puts;
     metrics::LoadMetric<UpdateMetricSet> updates;
@@ -29,6 +38,8 @@ public:
     metrics::DoubleAverageMetric recoveryModeTime;
     metrics::LongValueMetric docsStored;
     metrics::LongValueMetric bytesStored;
+    BucketDbMetrics mutable_dbs;
+    BucketDbMetrics read_only_dbs;
 
     explicit DistributorMetricSet(const metrics::LoadTypeSet& lt);
     ~DistributorMetricSet() override;