diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-04-02 12:54:18 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-04-02 13:26:01 +0000 |
commit | 2d3acf264fd3a8661d21d5b3f36bc6d8a65ab895 (patch) | |
tree | cb833f5bbb1a20c80062575e672d88d5cf732eb6 | |
parent | be3c2e7f0f1f3edc07973dccb05b13bf30c75de1 (diff) |
Add memory usage metrics for distributor bucket databases
17 files changed, 185 insertions, 6 deletions
diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 6cf1eadd6f7..6f854fed7c6 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_define_module( LIBS src/vespa/metrics + src/vespa/metrics/common TESTS src/tests diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 96156dc84b0..0d7eeba3601 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_library(metrics valuemetric.cpp valuemetricvalues.cpp xmlwriter.cpp + $<TARGET_OBJECTS:metrics_common> INSTALL lib64 DEPENDS diff --git a/metrics/src/vespa/metrics/common/CMakeLists.txt b/metrics/src/vespa/metrics/common/CMakeLists.txt new file mode 100644 index 00000000000..50183655dad --- /dev/null +++ b/metrics/src/vespa/metrics/common/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(metrics_common OBJECT + SOURCES + memory_usage_metrics.cpp + DEPENDS +) diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp new file mode 100644 index 00000000000..4848c96aea6 --- /dev/null +++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "memory_usage_metrics.h" +#include <vespa/vespalib/util/memoryusage.h> + +namespace metrics { + +MemoryUsageMetrics::MemoryUsageMetrics(metrics::MetricSet* parent) + : MetricSet("memory_usage", {}, "The memory usage for a given component", parent), + _allocated_bytes("allocated_bytes", {}, "The number of allocated bytes", this), + _used_bytes("used_bytes", {}, "The number of used bytes (<= allocatedbytes)", this), + _dead_bytes("dead_bytes", {}, "The number of dead bytes (<= usedbytes)", this), + _on_hold_bytes("onhold_bytes", {}, "The number of bytes on hold", this) +{ +} + +MemoryUsageMetrics::~MemoryUsageMetrics() = default; + +void +MemoryUsageMetrics::update(const vespalib::MemoryUsage& usage) +{ + _allocated_bytes.set(usage.allocatedBytes()); + _used_bytes.set(usage.usedBytes()); + _dead_bytes.set(usage.deadBytes()); + _on_hold_bytes.set(usage.allocatedBytesOnHold()); +} + +} diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.h b/metrics/src/vespa/metrics/common/memory_usage_metrics.h new file mode 100644 index 00000000000..7030db8c163 --- /dev/null +++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/metrics/metrics.h> + +namespace vespalib { class MemoryUsage; } + +namespace metrics { + +/** + * Metric set for memory usage metrics. + */ +class MemoryUsageMetrics : public metrics::MetricSet { + metrics::LongValueMetric _allocated_bytes; + metrics::LongValueMetric _used_bytes; + metrics::LongValueMetric _dead_bytes; + metrics::LongValueMetric _on_hold_bytes; + +public: + explicit MemoryUsageMetrics(metrics::MetricSet* parent); + ~MemoryUsageMetrics() override; + void update(const vespalib::MemoryUsage& usage); +}; + +} // namespace metrics diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp index 1008d3ee4f2..e0c3cf161bb 100644 --- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/config/config-stor-distributormanager.h> +#include <vespa/vespalib/util/memoryusage.h> #include <vespa/vespalib/gtest/gtest.h> #include <string> #include <sstream> @@ -100,6 +101,40 @@ TEST_F(BucketDBMetricUpdaterTest, doc_and_byte_counts_are_updated) { EXPECT_EQ(34, dms.bytesStored.getLast()); } +TEST_F(BucketDBMetricUpdaterTest, bucket_db_memory_usage_metrics_are_updated) { + BucketDBMetricUpdater metric_updater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + vespalib::MemoryUsage mem_usage; + mem_usage.incAllocatedBytes(1000); + mem_usage.incDeadBytes(700); + metric_updater.update_db_memory_usage(mem_usage, true); + + mem_usage.incAllocatedBytes(500); + mem_usage.incDeadBytes(100); + metric_updater.update_db_memory_usage(mem_usage, false); + + metric_updater.completeRound(false); + metric_updater.getLastCompleteStats().propagateMetrics(ims, dms); + + auto* m = dms.mutable_dbs.memory_usage.getMetric("allocated_bytes"); + ASSERT_TRUE(m != nullptr); + EXPECT_EQ(m->getLongValue("last"), 1000); + + m = dms.mutable_dbs.memory_usage.getMetric("dead_bytes"); + ASSERT_TRUE(m != nullptr); + EXPECT_EQ(m->getLongValue("last"), 700); + + m = dms.read_only_dbs.memory_usage.getMetric("allocated_bytes"); + ASSERT_TRUE(m != nullptr); + EXPECT_EQ(m->getLongValue("last"), 1500); + + m = dms.read_only_dbs.memory_usage.getMetric("dead_bytes"); + ASSERT_TRUE(m != nullptr); + EXPECT_EQ(m->getLongValue("last"), 800); +} + TEST_F(BucketDBMetricUpdaterTest, buckets_with_too_few_and_too_many_copies) { BucketDBMetricUpdater metricUpdater; IdealStateMetricSet ims; diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index 66d44a655e0..85f5c8c5be9 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -526,6 +526,12 @@ void BTreeBucketDatabase::print(std::ostream& out, bool verbose, (void)indent; } +vespalib::MemoryUsage BTreeBucketDatabase::memory_usage() const noexcept { + auto mem_usage = _tree.getMemoryUsage(); + mem_usage.merge(_store.getMemoryUsage()); + return mem_usage; +} + BTreeBucketDatabase::ReadGuardImpl::ReadGuardImpl(const BTreeBucketDatabase& db) : _db(&db), _guard(_db->_generation_handler.takeGuard()), diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h index 1f2b25814a8..8898b0c395a 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h @@ -91,6 +91,8 @@ public: std::unique_ptr<ReadGuard> acquire_read_guard() const override { return std::make_unique<ReadGuardImpl>(*this); } + + vespalib::MemoryUsage memory_usage() const noexcept override; }; } diff --git a/storage/src/vespa/storage/bucketdb/bucketdatabase.h b/storage/src/vespa/storage/bucketdb/bucketdatabase.h index 46aaaa997d9..2dbcdd194ef 100644 --- a/storage/src/vespa/storage/bucketdb/bucketdatabase.h +++ b/storage/src/vespa/storage/bucketdb/bucketdatabase.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/storage/bucketdb/bucketinfo.h> #include <vespa/document/bucket/bucketid.h> +#include <vespa/vespalib/util/memoryusage.h> namespace storage { @@ -250,6 +251,8 @@ public: virtual std::unique_ptr<ReadGuard> acquire_read_guard() const { return std::unique_ptr<ReadGuard>(); } + + [[nodiscard]] virtual vespalib::MemoryUsage memory_usage() const noexcept = 0; }; template <typename BucketInfoType> diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp index 463e4a4b8ce..edb808da294 100644 --- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp @@ -594,4 +594,30 @@ void MapBucketDatabase::ReadGuardImpl::find_parents_and_self(const document::Buc _db->getParents(bucket, entries); } +namespace { + +template <typename T> +size_t allocated_by_vec(const T& vec) noexcept { + return (vec.capacity() * sizeof(typename T::value_type)); +} + +template <typename T> +size_t used_by_vec(const T& vec) noexcept { + return (vec.size() * sizeof(typename T::value_type)); +} + +} + +vespalib::MemoryUsage MapBucketDatabase::memory_usage() const noexcept { + // We don't have a concept of hold lists here, nor do we know the exact size of the + // entries on our free list (these wrap a secondary replica vector allocation). + // So we fudge the numbers a bit, returning a lower bound approximation only. + // That's OK since this is a legacy database that's on the way out anyway. + vespalib::MemoryUsage mem_usage; + mem_usage.incAllocatedBytes(allocated_by_vec(_values) + allocated_by_vec(_db)); + mem_usage.incUsedBytes(used_by_vec(_values) + used_by_vec(_db)); + mem_usage.incDeadBytes(allocated_by_vec(_free) + allocated_by_vec(_freeValues)); + return mem_usage; +} + } // storage diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h index 9fe5e2d7740..e41b797a321 100644 --- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h @@ -30,6 +30,7 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; std::unique_ptr<ReadGuard> acquire_read_guard() const override; + vespalib::MemoryUsage memory_usage() const noexcept override; private: struct E { E() : value(-1), e_0(-1), e_1(-1) {}; diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index c211e775326..51eda0f948b 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -132,6 +132,8 @@ BucketDBMetricUpdater::Stats::propagateMetrics( { distributorMetrics.docsStored.set(_docCount); distributorMetrics.bytesStored.set(_byteCount); + distributorMetrics.mutable_dbs.memory_usage.update(_mutable_db_mem_usage); + distributorMetrics.read_only_dbs.memory_usage.update(_read_only_db_mem_usage); idealStateMetrics.buckets_toofewcopies.set(_tooFewCopies); idealStateMetrics.buckets_toomanycopies.set(_tooManyCopies); @@ -145,4 +147,10 @@ BucketDBMetricUpdater::reset() resetStats(); } +void BucketDBMetricUpdater::update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db) { + auto& target = (is_mutable_db ? _workingStats._mutable_db_mem_usage + : _workingStats._read_only_db_mem_usage); + target.merge(mem_usage); +} + } // storage::distributor diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 7ef8479866f..766307f49c2 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -4,7 +4,7 @@ #include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/config/config-stor-distributormanager.h> - +#include <vespa/vespalib/util/memoryusage.h> #include <unordered_map> namespace storage::distributor { @@ -22,10 +22,12 @@ public: uint64_t _tooManyCopies; uint64_t _noTrusted; uint64_t _totalBuckets; + vespalib::MemoryUsage _mutable_db_mem_usage; + vespalib::MemoryUsage _read_only_db_mem_usage; Stats(); Stats(const Stats &rhs); - ~Stats() { } + ~Stats() = default; Stats &operator=(const Stats &rhs) = default; @@ -63,7 +65,6 @@ private: public: BucketDBMetricUpdater(); - ~BucketDBMetricUpdater(); void setMinimumReplicaCountingMode(ReplicaCountingMode mode) noexcept { @@ -98,6 +99,8 @@ public: return _lastCompleteStats; } + void update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db); + private: void updateMinReplicationStats(const BucketDatabase::Entry& entry, uint32_t trustedCopies); diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 988af43a7be..3cca2847671 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -13,6 +13,7 @@ #include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/memoryusage.h> #include <vespa/log/log.h> LOG_SETUP(".distributor-main"); @@ -768,6 +769,16 @@ Distributor::updateInternalMetricsForCompletedScan() _must_send_updated_host_info = true; } _bucketSpacesStats = std::move(new_space_stats); + update_bucket_db_memory_usage_stats(); +} + +void Distributor::update_bucket_db_memory_usage_stats() { + for (auto& space : *_bucketSpaceRepo) { + _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), true); + } + for (auto& space : *_readOnlyBucketSpaceRepo) { + _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), false); + } } void diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index ac6e306a4fb..36e34592cba 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -226,6 +226,7 @@ private: * Takes metric lock. */ void updateInternalMetricsForCompletedScan(); + void update_bucket_db_memory_usage_stats(); void scanAllBuckets(); MaintenanceScanner::ScanResult scanNextBucket(); void enableNextConfig(); diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp index 8266aeb29cd..c1b034781c4 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -2,11 +2,19 @@ #include "distributormetricsset.h" #include <vespa/metrics/loadmetric.hpp> #include <vespa/metrics/summetric.hpp> +#include <vespa/vespalib/util/memoryusage.h> namespace storage::distributor { using metrics::MetricSet; +BucketDbMetrics::BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner) + : metrics::MetricSet("bucket_db", {{"type", db_type}}, "", owner), + memory_usage(this) +{} + +BucketDbMetrics::~BucketDbMetrics() = default; + DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt) : MetricSet("distributor", {{"distributor"}}, ""), puts(lt, PersistenceOperationMetricSet("puts"), this), @@ -41,7 +49,9 @@ DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt) bytesStored("bytesstored", {{"logdefault"},{"yamasdefault"}}, "Number of bytes stored in all buckets controlled by " - "this distributor", this) + "this distributor", this), + mutable_dbs("mutable", this), + read_only_dbs("read_only", this) { docsStored.logOnlyIfSet(); bytesStored.logOnlyIfSet(); diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h index d9c0711fd14..ce4025d8311 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.h +++ b/storage/src/vespa/storage/distributor/distributormetricsset.h @@ -5,12 +5,21 @@ #include "update_metric_set.h" #include "visitormetricsset.h" #include <vespa/metrics/metrics.h> +#include <vespa/metrics/common/memory_usage_metrics.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> +namespace vespalib { class MemoryUsage; } + namespace storage::distributor { -class DistributorMetricSet : public metrics::MetricSet -{ +struct BucketDbMetrics : metrics::MetricSet { + BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner); + ~BucketDbMetrics() override; + + metrics::MemoryUsageMetrics memory_usage; +}; + +class DistributorMetricSet : public metrics::MetricSet { public: metrics::LoadMetric<PersistenceOperationMetricSet> puts; metrics::LoadMetric<UpdateMetricSet> updates; @@ -29,6 +38,8 @@ public: metrics::DoubleAverageMetric recoveryModeTime; metrics::LongValueMetric docsStored; metrics::LongValueMetric bytesStored; + BucketDbMetrics mutable_dbs; + BucketDbMetrics read_only_dbs; explicit DistributorMetricSet(const metrics::LoadTypeSet& lt); ~DistributorMetricSet() override; |