diff options
38 files changed, 715 insertions, 597 deletions
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt index 166f8499725..8ee8a93df47 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(memfilepersistence_mapper OBJECT SOURCES buffer.cpp memfilemapper.cpp + serializationmetrics.cpp memfile_v1_serializer.cpp memfile_v1_verifier.cpp locationreadplanner.cpp diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp new file mode 100644 index 00000000000..366b0ae33e0 --- /dev/null +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializationmetrics.h" + +namespace storage { +namespace memfile { + +SerializationWriteMetrics::SerializationWriteMetrics(const std::string& name, MetricSet& owner) + : MetricSet(name, "", + "Write metrics for memfile persistence engine", + &owner), + headerLatency("header_latency", "", + "Time spent writing a single contiguous header location " + "on the disk.", this), + headerSize("header_size", "", + "Average size of contiguous header disk writes", this), + bodyLatency("body_latency", "", + "Time spent writing a single contiguous body location " + "on the disk.", this), + bodySize("body_size", "", + "Average size of contiguous body disk writes", this), + metaLatency("meta_latency", "", + "Time spent writing file header and slot metadata", this), + metaSize("meta_size", "", + "Size of file header and metadata writes", this), + totalLatency("total_latency", "", + "Total time spent performing slot file writing", this) +{ } + +SerializationWriteMetrics::~SerializationWriteMetrics() { } + +SerializationMetrics::SerializationMetrics(const std::string& name, MetricSet* owner) + : MetricSet(name, "", + "(De-)serialization I/O metrics for memfile " + "persistence engine", owner), + initialMetaReadLatency( + "initial_meta_read_latency", "", + "Time spent doing the initial read of " + "the file header and most (or all) of metadata", + this), + tooLargeMetaReadLatency( + "too_large_meta_read_latency", "", + "Time spent doing additional read for " + "metadata too large to be covered by initial " + "read", this), + totalLoadFileLatency( + "total_load_file_latency", "", + "Total time spent initially loading a " + "file from disk", this), + verifyLatency( + "verify_latency", "", + "Time spent performing file verification", this), + deleteFileLatency( + "delete_file_latency", "", + "Time spent deleting a file from disk", this), + headerReadLatency( + "header_read_latency", "", + "Time spent reading a single contiguous header location " + "on the disk (may span many document blobs)", this), + headerReadSize( + "header_read_size", "", + "Size of contiguous header disk location reads", this), + bodyReadLatency( + "body_read_latency", "", + "Time spent reading a single contiguous body location " + "on the disk (may span many document blobs)", this), + bodyReadSize( + "body_read_size", "", + "Size of contiguous body disk location reads", this), + cacheUpdateAndImplicitVerifyLatency( + "cache_update_and_implicit_verify_latency", "", + "Time spent updating memory cache structures and verifying " + "read data blocks for corruptions", this), + fullRewritesDueToDownsizingFile( + "full_rewrites_due_to_downsizing_file", "", + "Number of times a file was rewritten fully because the " + "original file had too low fill rate", this), + fullRewritesDueToTooSmallFile( + "full_rewrites_due_to_too_small_file", "", + "Number of times a file was rewritten fully because the " + "original file did not have sufficient free space for a " + "partial write", this), + partialWrite("partialwrite", *this), + fullWrite("fullwrite", *this) +{ } + +SerializationMetrics::~SerializationMetrics() { } + +} // memfile +} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h index 0eac46a1065..ac55b546b9e 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h @@ -17,28 +17,8 @@ public: metrics::LongAverageMetric metaSize; metrics::LongAverageMetric totalLatency; - SerializationWriteMetrics(const std::string& name, metrics::MetricSet& owner) - : metrics::MetricSet(name, "", - "Write metrics for memfile persistence engine", - &owner), - headerLatency("header_latency", "", - "Time spent writing a single contiguous header location " - "on the disk.", this), - headerSize("header_size", "", - "Average size of contiguous header disk writes", this), - bodyLatency("body_latency", "", - "Time spent writing a single contiguous body location " - "on the disk.", this), - bodySize("body_size", "", - "Average size of contiguous body disk writes", this), - metaLatency("meta_latency", "", - "Time spent writing file header and slot metadata", this), - metaSize("meta_size", "", - "Size of file header and metadata writes", this), - totalLatency("total_latency", "", - "Total time spent performing slot file writing", this) - { - } + SerializationWriteMetrics(const std::string& name, MetricSet& owner); + ~SerializationWriteMetrics(); }; class SerializationMetrics : public metrics::MetricSet @@ -59,62 +39,8 @@ public: SerializationWriteMetrics partialWrite; SerializationWriteMetrics fullWrite; - SerializationMetrics(const std::string& name, - metrics::MetricSet* owner = 0) - : metrics::MetricSet(name, "", - "(De-)serialization I/O metrics for memfile " - "persistence engine", owner), - initialMetaReadLatency( - "initial_meta_read_latency", "", - "Time spent doing the initial read of " - "the file header and most (or all) of metadata", - this), - tooLargeMetaReadLatency( - "too_large_meta_read_latency", "", - "Time spent doing additional read for " - "metadata too large to be covered by initial " - "read", this), - totalLoadFileLatency( - "total_load_file_latency", "", - "Total time spent initially loading a " - "file from disk", this), - verifyLatency( - "verify_latency", "", - "Time spent performing file verification", this), - deleteFileLatency( - "delete_file_latency", "", - "Time spent deleting a file from disk", this), - headerReadLatency( - "header_read_latency", "", - "Time spent reading a single contiguous header location " - "on the disk (may span many document blobs)", this), - headerReadSize( - "header_read_size", "", - "Size of contiguous header disk location reads", this), - bodyReadLatency( - "body_read_latency", "", - "Time spent reading a single contiguous body location " - "on the disk (may span many document blobs)", this), - bodyReadSize( - "body_read_size", "", - "Size of contiguous body disk location reads", this), - cacheUpdateAndImplicitVerifyLatency( - "cache_update_and_implicit_verify_latency", "", - "Time spent updating memory cache structures and verifying " - "read data blocks for corruptions", this), - fullRewritesDueToDownsizingFile( - "full_rewrites_due_to_downsizing_file", "", - "Number of times a file was rewritten fully because the " - "original file had too low fill rate", this), - fullRewritesDueToTooSmallFile( - "full_rewrites_due_to_too_small_file", "", - "Number of times a file was rewritten fully because the " - "original file did not have sufficient free space for a " - "partial write", this), - partialWrite("partialwrite", *this), - fullWrite("fullwrite", *this) - { - } + SerializationMetrics(const std::string& name, MetricSet* owner = 0); + ~SerializationMetrics(); }; } // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt index e30807d99b2..885b0edb8dc 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(memfilepersistence_spi OBJECT SOURCES memfilepersistence.cpp memfilepersistenceprovider.cpp + memfilepersistenceprovidermetrics.cpp operationhandler.cpp iteratorhandler.cpp joinoperationhandler.cpp diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp new file mode 100644 index 00000000000..e3fa4337fd5 --- /dev/null +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "memfilepersistenceprovidermetrics.h" +#include <vespa/metrics/summetric.hpp> +#include <vespa/storageframework/generic/component/component.h> + +namespace storage { +namespace memfile { + +using metrics::MetricSet; + +MemFilePersistenceThreadMetrics::MemFilePersistenceThreadMetrics(const std::string& name, MetricSet& owner) + : MetricSet(name, "partofsum thread", "Metrics for a worker thread using memfile persistence provider", &owner), + headerOnlyGets("headeronlygets", "", "Number of gets that only read header", this), + headerOnlyUpdates("headeronlyupdates", "", "Number of updates that only wrote header", this), + serialization("serialization", this) +{ } + +MemFilePersistenceThreadMetrics::~MemFilePersistenceThreadMetrics() { } + +MemFilePersistenceCacheMetrics::MemFilePersistenceCacheMetrics(MetricSet& owner) + : MetricSet("cache", "", "Metrics for the VDS persistence cache", &owner), + files("files", "", "Number of files cached", this), + meta("meta", "", "Bytes of file metadata cached", this), + header("header", "", "Bytes of file header parts cached", this), + body("body", "", "Bytes of file body parts cached", this), + hits("hits", "", "Number of times a bucket was attempted fetched " + "from the cache and it was already present", this), + misses("misses", "", "Number of times a bucket was attempted fetched " + "from the cache and it could not be found, requiring a load", this), + meta_evictions("meta_evictions", "", "Bucket meta data evictions", this), + header_evictions("header_evictions", "", "Bucket header (and " + "implicitly body, if present) data evictions", this), + body_evictions("body_evictions", "", "Bucket body data evictions", this) +{ } + +MemFilePersistenceCacheMetrics::~MemFilePersistenceCacheMetrics() { } + +MemFilePersistenceMetrics::MemFilePersistenceMetrics(framework::Component& component) + : MetricSet("memfilepersistence", "", "Metrics for the VDS persistence layer"), + _component(component), + _cache(*this) +{ } + +MemFilePersistenceMetrics::~MemFilePersistenceMetrics() { } + +MemFilePersistenceThreadMetrics* +MemFilePersistenceMetrics::addThreadMetrics() { + metrics::MetricLockGuard metricLock(_component.getMetricManagerLock()); + vespalib::LockGuard guard(_threadMetricsLock); + + if (!_sumMetric.get()) { + _sumMetric.reset(new metrics::SumMetric<MemFilePersistenceThreadMetrics> + ("allthreads", "sum", "", this)); + } + + std::string name = vespalib::make_string("thread_%zu", _threadMetrics.size()); + auto metrics = std::make_unique<MemFilePersistenceThreadMetrics>(name, *this); + _threadMetrics.emplace_back(std::move(metrics)); + _sumMetric->addMetricToSum(*metrics); + return metrics.get(); +} + +} +} + +template class metrics::SumMetric<storage::memfile::MemFilePersistenceThreadMetrics>; diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h index 70c711e81fd..6c6e1d882d1 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h @@ -5,6 +5,9 @@ #include <vespa/memfilepersistence/mapper/serializationmetrics.h> namespace storage { + +namespace framework { class Component; } + namespace memfile { class MemFilePersistenceThreadMetrics : public metrics::MetricSet @@ -14,17 +17,8 @@ public: metrics::LongCountMetric headerOnlyUpdates; SerializationMetrics serialization; - MemFilePersistenceThreadMetrics(const std::string& name, metrics::MetricSet& owner) - : metrics::MetricSet(name, "partofsum thread", - "Metrics for a worker thread using memfile persistence " - "provider", &owner), - headerOnlyGets("headeronlygets", "", - "Number of gets that only read header", this), - headerOnlyUpdates("headeronlyupdates", "", - "Number of updates that only wrote header", this), - serialization("serialization", this) - { - } + MemFilePersistenceThreadMetrics(const std::string& name, metrics::MetricSet& owner); + ~MemFilePersistenceThreadMetrics(); }; class MemFilePersistenceCacheMetrics : public metrics::MetricSet @@ -40,22 +34,8 @@ public: metrics::LongCountMetric header_evictions; metrics::LongCountMetric body_evictions; - MemFilePersistenceCacheMetrics(metrics::MetricSet& owner) - : metrics::MetricSet("cache", "", - "Metrics for the VDS persistence cache", &owner), - files("files", "", "Number of files cached", this), - meta("meta", "", "Bytes of file metadata cached", this), - header("header", "", "Bytes of file header parts cached", this), - body("body", "", "Bytes of file body parts cached", this), - hits("hits", "", "Number of times a bucket was attempted fetched " - "from the cache and it was already present", this), - misses("misses", "", "Number of times a bucket was attempted fetched " - "from the cache and it could not be found, requiring a load", this), - meta_evictions("meta_evictions", "", "Bucket meta data evictions", this), - header_evictions("header_evictions", "", "Bucket header (and " - "implicitly body, if present) data evictions", this), - body_evictions("body_evictions", "", "Bucket body data evictions", this) - {} + MemFilePersistenceCacheMetrics(metrics::MetricSet& owner); + ~MemFilePersistenceCacheMetrics(); }; class MemFilePersistenceMetrics : public metrics::MetricSet @@ -64,38 +44,14 @@ class MemFilePersistenceMetrics : public metrics::MetricSet public: vespalib::Lock _threadMetricsLock; - std::list<vespalib::LinkedPtr<MemFilePersistenceThreadMetrics> > _threadMetrics; + std::list<std::unique_ptr<MemFilePersistenceThreadMetrics> > _threadMetrics; std::unique_ptr<metrics::SumMetric<MemFilePersistenceThreadMetrics> > _sumMetric; MemFilePersistenceCacheMetrics _cache; - MemFilePersistenceMetrics(framework::Component& component) - : metrics::MetricSet("memfilepersistence", "", - "Metrics for the VDS persistence layer"), - _component(component), - _cache(*this) - { - } - - MemFilePersistenceThreadMetrics* addThreadMetrics() { - metrics::MetricLockGuard metricLock(_component.getMetricManagerLock()); - vespalib::LockGuard guard(_threadMetricsLock); - - if (!_sumMetric.get()) { - _sumMetric.reset(new metrics::SumMetric<MemFilePersistenceThreadMetrics> - ("allthreads", "sum", "", this)); - } - - std::string name = vespalib::make_string("thread_%zu", _threadMetrics.size()); - - MemFilePersistenceThreadMetrics* metrics = - new MemFilePersistenceThreadMetrics(name, *this); - - _threadMetrics.push_back(vespalib::LinkedPtr<MemFilePersistenceThreadMetrics>( - metrics)); - _sumMetric->addMetricToSum(*metrics); - return metrics; - } + MemFilePersistenceMetrics(framework::Component& component); + ~MemFilePersistenceMetrics(); + MemFilePersistenceThreadMetrics* addThreadMetrics(); }; } diff --git a/metrics/src/tests/loadmetrictest.cpp b/metrics/src/tests/loadmetrictest.cpp index 3585f4adf32..ff4658d6bb2 100644 --- a/metrics/src/tests/loadmetrictest.cpp +++ b/metrics/src/tests/loadmetrictest.cpp @@ -62,7 +62,7 @@ namespace { { } - Metric* clone(std::vector<Metric::LP>& ownerList, + MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, MetricSet* owner, bool includeUnused = false) const @@ -71,7 +71,9 @@ namespace { return MetricSet::clone(ownerList, copyType, owner, includeUnused); } - return (new MyMetricSet(owner))->assignValues(*this); + MyMetricSet * myset = new MyMetricSet(owner); + myset->assignValues(*this); + return myset; } }; } diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index d3349d2e91d..31471cc40ed 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -3,7 +3,6 @@ #include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/metrics/metrics.h> -#include <vespa/metrics/summetric.hpp> #include <vespa/metrics/xmlwriter.h> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index 626b53f7d90..5683363ab42 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -67,7 +67,7 @@ struct SubSubMetricSet : public MetricSet { averageSum.addMetricToSum(average2); } - virtual Metric* clone(std::vector<Metric::LP>& ownerList, + virtual MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const { @@ -119,7 +119,7 @@ struct SubMetricSet : public MetricSet { setSum.addMetricToSum(set2); } - virtual Metric* clone(std::vector<Metric::LP>& ownerList, + virtual MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const { diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp index cb069f27300..f164bf03ee1 100644 --- a/metrics/src/tests/stresstest.cpp +++ b/metrics/src/tests/stresstest.cpp @@ -43,15 +43,16 @@ namespace { _valueSum.addMetricToSum(_value2); } - Metric* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, MetricSet* owner, bool includeUnused) const { if (copyType != CLONE) { return MetricSet::clone(ownerList, copyType, owner, includeUnused); } - return (new InnerMetricSet(getName().c_str(), _loadTypes, owner)) - ->assignValues(*this); + InnerMetricSet * myset = new InnerMetricSet(getName().c_str(), _loadTypes, owner); + myset->assignValues(*this); + return myset; } }; struct OuterMetricSet : public MetricSet { diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp index 3b5812f363f..0343174e0b3 100644 --- a/metrics/src/tests/summetrictest.cpp +++ b/metrics/src/tests/summetrictest.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> #include <vespa/metrics/metrics.h> -#include <vespa/metrics/summetric.hpp> #include <vespa/vdstestlib/cppunit/macros.h> namespace metrics { diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h index 65281fb5b3a..e09168f54db 100644 --- a/metrics/src/vespa/metrics/countmetric.h +++ b/metrics/src/vespa/metrics/countmetric.h @@ -66,8 +66,9 @@ public: CountMetric(const String& name, Tags dimensions, const String& description, MetricSet* owner = 0); - CountMetric(const CountMetric<T, SumOnAdd>& other, - CopyType, MetricSet* owner); + CountMetric(const CountMetric<T, SumOnAdd>& other, CopyType, MetricSet* owner); + + ~CountMetric(); virtual MetricValueClass::UP getValues() const { return MetricValueClass::UP(new Values(_values.getValues())); } diff --git a/metrics/src/vespa/metrics/countmetric.hpp b/metrics/src/vespa/metrics/countmetric.hpp index f2f7b953a31..885da17e718 100644 --- a/metrics/src/vespa/metrics/countmetric.hpp +++ b/metrics/src/vespa/metrics/countmetric.hpp @@ -34,6 +34,9 @@ CountMetric<T, SumOnAdd>::CountMetric(const CountMetric<T, SumOnAdd>& other, } template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>::~CountMetric() { } + +template <typename T, bool SumOnAdd> CountMetric<T, SumOnAdd>& CountMetric<T, SumOnAdd>::operator+=(const CountMetric<T, SumOnAdd>& other) { diff --git a/metrics/src/vespa/metrics/loadmetric.h b/metrics/src/vespa/metrics/loadmetric.h index c141c2d9e61..e23973d1681 100644 --- a/metrics/src/vespa/metrics/loadmetric.h +++ b/metrics/src/vespa/metrics/loadmetric.h @@ -64,7 +64,8 @@ public: * the load metric alters this data in supplied metric) */ LoadMetric(const LoadMetric<MetricType>& other, MetricSet* owner); - Metric* clone(std::vector<Metric::LP>& ownerList, + ~LoadMetric(); + MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, MetricSet* owner, bool includeUnused = false) const override; diff --git a/metrics/src/vespa/metrics/loadmetric.hpp b/metrics/src/vespa/metrics/loadmetric.hpp index 68a0a4981a7..ea456ece92c 100644 --- a/metrics/src/vespa/metrics/loadmetric.hpp +++ b/metrics/src/vespa/metrics/loadmetric.hpp @@ -53,7 +53,10 @@ LoadMetric<MetricType>::LoadMetric(const LoadMetric<MetricType>& other, MetricSe } template<typename MetricType> -Metric* +LoadMetric<MetricType>::~LoadMetric() { } + +template<typename MetricType> +MetricSet* LoadMetric<MetricType>::clone(std::vector<Metric::LP>& ownerList, CopyType copyType, MetricSet* owner, bool includeUnused) const diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp index 01ab46d1093..dbb58882670 100644 --- a/metrics/src/vespa/metrics/metric.cpp +++ b/metrics/src/vespa/metrics/metric.cpp @@ -92,6 +92,8 @@ Metric::Metric(const Metric& other, MetricSet* owner) registerWithOwnerIfRequired(owner); } +Metric::~Metric() { } + bool Metric::tagsSpecifyAtLeastOneDimension(const Tags& tags) const { diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h index 40b267652fc..850d712ebf5 100644 --- a/metrics/src/vespa/metrics/metric.h +++ b/metrics/src/vespa/metrics/metric.h @@ -122,7 +122,7 @@ public: MetricSet* owner = 0); Metric(const Metric& other, MetricSet* owner); - virtual ~Metric() {} + ~Metric(); const String& getName() const { return _name; } /** diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index b915786afcf..df7316610a0 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -340,8 +340,9 @@ MetricManager::checkMetricsAltered(const MetricLockGuard & guard) // When calling this function, the metric lock is already taken. The thread // monitor lock might be taken. void -MetricManager::handleMetricsAltered(const MetricLockGuard &) +MetricManager::handleMetricsAltered(const MetricLockGuard & guard) { + (void) guard; if (_config.get() == NULL) { LOG(info, "_config is NULL -> very odd indeed."); return; diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 594fa5c43f5..ffa73c5eff1 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -303,7 +303,7 @@ private: time_t updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule); void updateSnapshotMetrics(const MetricLockGuard & guard); - void handleMetricsAltered(const MetricLockGuard &); + void handleMetricsAltered(const MetricLockGuard & guard); typedef std::pair<uint32_t, std::string> SnapSpec; static std::vector<SnapSpec> createSnapshotPeriods( diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp index 4fc925f0218..96bd87c7102 100644 --- a/metrics/src/vespa/metrics/metricset.cpp +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -54,6 +54,8 @@ MetricSet::MetricSet(const MetricSet& other, } } +MetricSet::~MetricSet() { } + const Metric* MetricSet::getMetricInternal(const String& name) const { diff --git a/metrics/src/vespa/metrics/metricset.h b/metrics/src/vespa/metrics/metricset.h index 444d7491e7c..47a29deda34 100644 --- a/metrics/src/vespa/metrics/metricset.h +++ b/metrics/src/vespa/metrics/metricset.h @@ -26,9 +26,6 @@ class MetricSet : public Metric // If so, the name of the metric is used as dimension value. public: - typedef std::unique_ptr<MetricSet> UP; - typedef std::shared_ptr<MetricSet> SP; - MetricSet(const String& name, const String& tags, const String& description, MetricSet* owner = 0, const std::string& dimensionKey = ""); @@ -38,7 +35,7 @@ public: MetricSet(const MetricSet&, std::vector<Metric::LP>& ownerList, CopyType, MetricSet* owner = 0, bool includeUnused = false); - virtual ~MetricSet() {} + ~MetricSet(); // If no path, this metric is not registered within another bool isTopSet() const { return _owner == 0; } @@ -55,9 +52,9 @@ public: void registerMetric(Metric& m); void unregisterMetric(Metric& m); - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType type, MetricSet* owner, - bool includeUnused = false) const + virtual MetricSet* clone(std::vector<Metric::LP>& ownerList, + CopyType type, MetricSet* owner, + bool includeUnused = false) const { return new MetricSet(*this, ownerList, type, owner, includeUnused); } void reset(); diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp index afb4aa305d4..fee30760f91 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.cpp +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -49,7 +49,7 @@ MetricSnapshot::recreateSnapshot(const MetricSet& metrics, bool copyUnset) std::vector<Metric::LP> newMetrics; Metric* m = metrics.clone(newMetrics, Metric::INACTIVE, 0, copyUnset); assert(m->isMetricSet()); - MetricSet::UP newSnapshot(static_cast<MetricSet*>(m)); + std::unique_ptr<MetricSet> newSnapshot(static_cast<MetricSet*>(m)); newSnapshot->reset(); _snapshot->addToSnapshot(*newSnapshot, newMetrics); _snapshot = std::move(newSnapshot); diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h index ba4cfe9df53..c10ec204a84 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.h +++ b/metrics/src/vespa/metrics/metricsnapshot.h @@ -26,7 +26,7 @@ class MetricSnapshot // If set to 0, use _fromTime + _period. time_t _toTime; // Keeps the metrics set view of the snapshot - MetricSet::UP _snapshot; + std::unique_ptr<MetricSet> _snapshot; // Snapshots must own their own metrics mutable std::vector<Metric::LP> _metrics; diff --git a/metrics/src/vespa/metrics/summetric.cpp b/metrics/src/vespa/metrics/summetric.cpp index a1658bf0781..bb75e9dc36c 100644 --- a/metrics/src/vespa/metrics/summetric.cpp +++ b/metrics/src/vespa/metrics/summetric.cpp @@ -11,6 +11,7 @@ template class SumMetric<ValueMetric<int64_t, int64_t, true>>; template class SumMetric<ValueMetric<double, double, false>>; template class SumMetric<ValueMetric<double, double, true>>; template class SumMetric<CountMetric<uint64_t, true>>; +template class SumMetric<MetricSet>; } // metrics diff --git a/metrics/src/vespa/metrics/summetric.h b/metrics/src/vespa/metrics/summetric.h index f40495a0334..cb213cc36c1 100644 --- a/metrics/src/vespa/metrics/summetric.h +++ b/metrics/src/vespa/metrics/summetric.h @@ -27,14 +27,14 @@ public: { private: std::vector<Metric::LP> _startValueChildren; - std::unique_ptr<AddendMetric> _startValue; + Metric::UP _startValue; public: typedef std::shared_ptr<StartValue> SP; StartValue(const AddendMetric &metric) : _startValueChildren(), _startValue(metric.clone(_startValueChildren, CLONE, 0, false)) {} - const AddendMetric &getStartValue() const { return *_startValue; } + const AddendMetric &getStartValue() const { return static_cast<const AddendMetric &>(*_startValue); } }; private: @@ -44,6 +44,7 @@ private: public: SumMetric(const String& name, const String& tags, const String& description, MetricSet* owner = 0); SumMetric(const SumMetric<AddendMetric>& other, std::vector<Metric::LP>& ownerList, MetricSet* owner = 0); + ~SumMetric(); virtual Metric* clone( std::vector<Metric::LP>&, CopyType, MetricSet* owner, bool includeUnused = false) const; diff --git a/metrics/src/vespa/metrics/summetric.hpp b/metrics/src/vespa/metrics/summetric.hpp index 59739779901..b881bb43f9a 100644 --- a/metrics/src/vespa/metrics/summetric.hpp +++ b/metrics/src/vespa/metrics/summetric.hpp @@ -93,6 +93,9 @@ SumMetric<AddendMetric>::SumMetric(const SumMetric<AddendMetric>& other, } template<typename AddendMetric> +SumMetric<AddendMetric>::~SumMetric() { } + +template<typename AddendMetric> Metric* SumMetric<AddendMetric>::clone(std::vector<Metric::LP>& ownerList, CopyType copyType, MetricSet* owner, diff --git a/metrics/src/vespa/metrics/valuemetric.h b/metrics/src/vespa/metrics/valuemetric.h index 81edca7ac16..48fa1005127 100644 --- a/metrics/src/vespa/metrics/valuemetric.h +++ b/metrics/src/vespa/metrics/valuemetric.h @@ -88,13 +88,15 @@ public: ValueMetric(const String& name, Tags dimensions, const String& description, MetricSet* owner = 0); + ~ValueMetric(); + virtual MetricValueClass::UP getValues() const { return MetricValueClass::UP(new Values(_values.getValues())); } void unsetOnZeroValue() { _values.setFlag(UNSET_ON_ZERO_VALUE); } void logOnlyIfSet() { _values.removeFlag(LOG_IF_UNSET); } - virtual ValueMetric<AvgVal, TotVal, SumOnAdd>* clone( + virtual ValueMetric * clone( std::vector<Metric::LP>&, CopyType type, MetricSet* owner, bool /*includeUnused*/) const { return new ValueMetric<AvgVal,TotVal,SumOnAdd>(*this, type, owner); } diff --git a/metrics/src/vespa/metrics/valuemetric.hpp b/metrics/src/vespa/metrics/valuemetric.hpp index 0a196ec420f..96a59e9f347 100644 --- a/metrics/src/vespa/metrics/valuemetric.hpp +++ b/metrics/src/vespa/metrics/valuemetric.hpp @@ -38,6 +38,9 @@ ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric( } template<typename AvgVal, typename TotVal, bool SumOnAdd> +ValueMetric<AvgVal, TotVal, SumOnAdd>::~ValueMetric() { } + +template<typename AvgVal, typename TotVal, bool SumOnAdd> void ValueMetric<AvgVal, TotVal, SumOnAdd>::inc(AvgVal incVal) { if (!checkFinite(incVal, std::is_floating_point<AvgVal>())) { diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 7fedc230cf8..5e8e0b106c8 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -2,6 +2,9 @@ vespa_add_library(storage_distributor SOURCES distributor.cpp + distributormetricsset.cpp + visitormetricsset.cpp + idealstatemetricsset.cpp operationowner.cpp distributorcomponent.cpp statechecker.cpp diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp new file mode 100644 index 00000000000..b57ac81909e --- /dev/null +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "distributormetricsset.h" +#include <vespa/metrics/loadmetric.hpp> +#include <vespa/metrics/summetric.hpp> + +namespace storage { + +using metrics::MetricSet; + +PersistenceFailuresMetricSet::PersistenceFailuresMetricSet(MetricSet* owner) + : MetricSet("failures", "", "Detailed failure statistics", owner), + sum("total", "logdefault yamasdefault", "Sum of all failures", this), + notready("notready", "", "The number of operations discarded because distributor was not ready", this), + notconnected("notconnected", "", "The number of operations discarded because there were no available storage nodes to send to", this), + wrongdistributor("wrongdistributor", "", "The number of operations discarded because they were sent to the wrong distributor", this), + safe_time_not_reached("safe_time_not_reached", "", + "The number of operations that were transiently" + " failed due to them arriving before the safe " + "time point for bucket ownership handovers has " + "passed", this), + storagefailure("storagefailure", "", "The number of operations that failed in storage", this), + timeout("timeout", "", "The number of operations that failed because the operation timed out towards storage", this), + busy("busy", "", "The number of messages from storage that failed because the storage node was busy", this), + notfound("notfound", "", "The number of operations that failed because the document did not exist", this) +{ + sum.addMetricToSum(notready); + sum.addMetricToSum(notconnected); + sum.addMetricToSum(wrongdistributor); + sum.addMetricToSum(safe_time_not_reached); + sum.addMetricToSum(storagefailure); + sum.addMetricToSum(timeout); + sum.addMetricToSum(busy); + sum.addMetricToSum(notfound); +} +PersistenceFailuresMetricSet::~PersistenceFailuresMetricSet() { } + +MetricSet * +PersistenceFailuresMetricSet::clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (PersistenceFailuresMetricSet*) + (new PersistenceFailuresMetricSet(owner))->assignValues(*this); +} + +PersistenceOperationMetricSet::PersistenceOperationMetricSet(const std::string& name, MetricSet* owner) + : MetricSet(name, "", vespalib::make_string("Statistics for the %s command", name.c_str()), owner, "operationtype"), + latency("latency", "yamasdefault", vespalib::make_string("The average latency of %s operations", name.c_str()), this), + ok("ok", "logdefault yamasdefault", vespalib::make_string("The number of successful %s operations performed", name.c_str()), this), + failures(this) +{ } + +PersistenceOperationMetricSet::~PersistenceOperationMetricSet() { } + +MetricSet * +PersistenceOperationMetricSet::clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (PersistenceOperationMetricSet*) + (new PersistenceOperationMetricSet(getName(), owner)) + ->assignValues(*this); +} + +DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt) + : MetricSet("distributor", "distributor", ""), + puts(lt, *&PersistenceOperationMetricSet("puts"), this), + updates(lt, *&PersistenceOperationMetricSet("updates"), this), + update_puts(lt, *&PersistenceOperationMetricSet("update_puts"), this), + update_gets(lt, *&PersistenceOperationMetricSet("update_gets"), this), + removes(lt, *&PersistenceOperationMetricSet("removes"), this), + removelocations(lt, *&PersistenceOperationMetricSet("removelocations"), this), + gets(lt, *&PersistenceOperationMetricSet("gets"), this), + stats(lt, *&PersistenceOperationMetricSet("stats"), this), + multioperations(lt, *&PersistenceOperationMetricSet("multioperations"), this), + recoveryModeTime("recoverymodeschedulingtime", "", + "Time spent scheduling operations in recovery mode " + "after receiving new cluster state", this), + docsStored("docsstored", "logdefault yamasdefault", + "Number of documents stored in all buckets controlled by " + "this distributor", this), + bytesStored("bytesstored", "logdefault yamasdefault", + "Number of bytes stored in all buckets controlled by " + "this distributor", this) +{ + docsStored.logOnlyIfSet(); + bytesStored.logOnlyIfSet(); +} + +DistributorMetricSet::~DistributorMetricSet() { } + +} + +template class metrics::LoadMetric<storage::PersistenceOperationMetricSet>; +template class metrics::SumMetric<storage::PersistenceOperationMetricSet>; diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h index f83d34e4dd8..392a1068f2c 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.h +++ b/storage/src/vespa/storage/distributor/distributormetricsset.h @@ -9,31 +9,8 @@ namespace storage { class PersistenceFailuresMetricSet : public metrics::MetricSet { public: - PersistenceFailuresMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("failures", "", "Detailed failure statistics", owner), - sum("total", "logdefault yamasdefault", "Sum of all failures", this), - notready("notready", "", "The number of operations discarded because distributor was not ready", this), - notconnected("notconnected", "", "The number of operations discarded because there were no available storage nodes to send to", this), - wrongdistributor("wrongdistributor", "", "The number of operations discarded because they were sent to the wrong distributor", this), - safe_time_not_reached("safe_time_not_reached", "", - "The number of operations that were transiently" - " failed due to them arriving before the safe " - "time point for bucket ownership handovers has " - "passed", this), - storagefailure("storagefailure", "", "The number of operations that failed in storage", this), - timeout("timeout", "", "The number of operations that failed because the operation timed out towards storage", this), - busy("busy", "", "The number of messages from storage that failed because the storage node was busy", this), - notfound("notfound", "", "The number of operations that failed because the document did not exist", this) - { - sum.addMetricToSum(notready); - sum.addMetricToSum(notconnected); - sum.addMetricToSum(wrongdistributor); - sum.addMetricToSum(safe_time_not_reached); - sum.addMetricToSum(storagefailure); - sum.addMetricToSum(timeout); - sum.addMetricToSum(busy); - sum.addMetricToSum(notfound); - } + PersistenceFailuresMetricSet(metrics::MetricSet* owner); + ~PersistenceFailuresMetricSet(); metrics::SumMetric<metrics::LongCountMetric> sum; metrics::LongCountMetric notready; @@ -45,17 +22,8 @@ public: metrics::LongCountMetric busy; metrics::LongCountMetric notfound; - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); - } - return (PersistenceFailuresMetricSet*) - (new PersistenceFailuresMetricSet(owner))->assignValues(*this); - } + MetricSet * clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + metrics::MetricSet* owner, bool includeUnused) const; PersistenceFailuresMetricSet* operator&() { return this; } }; @@ -66,43 +34,12 @@ public: metrics::LongCountMetric ok; PersistenceFailuresMetricSet failures; - PersistenceOperationMetricSet(const std::string& name, - metrics::MetricSet* owner = 0) - : metrics::MetricSet(name, - "", - vespalib::make_string("Statistics for the %s command", - name.c_str()), - owner, - "operationtype"), - latency("latency", - "yamasdefault", - vespalib::make_string("The average latency of %s operations", - name.c_str()), - this), - ok("ok", - "logdefault yamasdefault", - vespalib::make_string("The number of successful %s operations performed", - name.c_str()), - this), - failures(this) - { - } + PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr); + ~PersistenceOperationMetricSet(); - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); - } - return (PersistenceOperationMetricSet*) - (new PersistenceOperationMetricSet(getName(), owner)) - ->assignValues(*this); - } + MetricSet * clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + metrics::MetricSet* owner, bool includeUnused) const override; PersistenceOperationMetricSet* operator&() { return this; } - - }; class DistributorMetricSet : public metrics::MetricSet @@ -121,31 +58,8 @@ public: metrics::LongValueMetric docsStored; metrics::LongValueMetric bytesStored; - DistributorMetricSet(const metrics::LoadTypeSet& lt) - : metrics::MetricSet("distributor", "distributor", ""), - puts(lt, *&PersistenceOperationMetricSet("puts"), this), - updates(lt, *&PersistenceOperationMetricSet("updates"), this), - update_puts(lt, *&PersistenceOperationMetricSet("update_puts"), this), - update_gets(lt, *&PersistenceOperationMetricSet("update_gets"), this), - removes(lt, *&PersistenceOperationMetricSet("removes"), this), - removelocations(lt, *&PersistenceOperationMetricSet("removelocations"), this), - gets(lt, *&PersistenceOperationMetricSet("gets"), this), - stats(lt, *&PersistenceOperationMetricSet("stats"), this), - multioperations(lt, *&PersistenceOperationMetricSet("multioperations"), this), - recoveryModeTime("recoverymodeschedulingtime", "", - "Time spent scheduling operations in recovery mode " - "after receiving new cluster state", this), - docsStored("docsstored", "logdefault yamasdefault", - "Number of documents stored in all buckets controlled by " - "this distributor", this), - bytesStored("bytesstored", "logdefault yamasdefault", - "Number of bytes stored in all buckets controlled by " - "this distributor", this) - { - docsStored.logOnlyIfSet(); - bytesStored.logOnlyIfSet(); - } - + DistributorMetricSet(const metrics::LoadTypeSet& lt); + ~DistributorMetricSet(); }; } diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp new file mode 100644 index 00000000000..2f8bdf1f13d --- /dev/null +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "idealstatemetricsset.h" +namespace storage { + +namespace distributor { + +OperationMetricSet::OperationMetricSet(const std::string& name, const std::string& tags, const std::string& description, MetricSet* owner) + : MetricSet(name, tags, description, owner), + pending("pending", "logdefault yamasdefault", "The number of operations pending", this), + ok("done_ok", "logdefault yamasdefault", "The number of operations successfully performed", this), + failed("done_failed", "logdefault yamasdefault", "The number of operations that failed", this) +{ } + +OperationMetricSet::~OperationMetricSet() { } + +void +IdealStateMetricSet::createOperationMetrics() { + typedef IdealStateOperation ISO; + operations.resize(ISO::OPERATION_COUNT); + operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("delete_bucket", "logdefault yamasdefault", + "Operations to delete excess buckets on storage nodes", this)); + operations[ISO::MERGE_BUCKET] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("merge_bucket", "logdefault yamasdefault", + "Operations to merge buckets that are out of sync", this)); + operations[ISO::SPLIT_BUCKET] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("split_bucket", "logdefault yamasdefault", + "Operations to split buckets that are larger than the configured size", this)); + operations[ISO::JOIN_BUCKET] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("join_bucket", "logdefault yamasdefault", + "Operations to join buckets that in sum are smaller than the configured size", this)); + operations[ISO::SET_BUCKET_STATE] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("set_bucket_state", + "logdefault yamasdefault", + "Operations to set active/ready state for bucket copies", this)); + operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>( + new OperationMetricSet("garbage_collection", + "logdefault yamasdefault", + "Operations to garbage collect data from buckets", this)); +} + +IdealStateMetricSet::IdealStateMetricSet() + : MetricSet("idealstate", "idealstate", "Statistics for ideal state generation"), + idealstate_diff("idealstate_diff", "logdefault yamasdefault", + "A number representing the current difference from the ideal " + "state. This is a number that decreases steadily as the system " + "is getting closer to the ideal state", this), + buckets_toofewcopies("buckets_toofewcopies", "logdefault yamasdefault", + "The number of buckets the distributor controls that have less " + "than the desired redundancy", this), + buckets_toomanycopies("buckets_toomanycopies", "logdefault yamasdefault", + "The number of buckets the distributor controls that have more " + "than the desired redundancy", this), + buckets("buckets", "logdefault yamasdefault", "The number of buckets the distributor controls", this), + buckets_notrusted("buckets_notrusted", "logdefault yamasdefault", + "The number of buckets that have no trusted copies.", this), + buckets_rechecking("buckets_rechecking", "logdefault yamasdefault", + "The number of buckets that we are rechecking for " + "ideal state operations", this), + startOperationsLatency("start_operations_latency", "", "Time used in startOperations()", this), + nodesPerMerge("nodes_per_merge", "", "The number of nodes involved in a single merge operation.", this) +{ + createOperationMetrics(); +} + +IdealStateMetricSet::~IdealStateMetricSet() { } + +void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) { + for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { + operations[i]->pending.set(newMetrics[i]); + } + + idealstate_diff.set( + operations[IdealStateOperation::DELETE_BUCKET]->pending.getLast() + + operations[IdealStateOperation::MERGE_BUCKET]->pending.getLast() * 10 + + operations[IdealStateOperation::SPLIT_BUCKET]->pending.getLast() * 4 + + operations[IdealStateOperation::JOIN_BUCKET]->pending.getLast() * 2 + + operations[IdealStateOperation::SET_BUCKET_STATE]->pending.getLast()); +} + +} + +} + diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 85d19e18d3e..09189bc80c3 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -15,20 +15,8 @@ public: metrics::LongCountMetric ok; metrics::LongCountMetric failed; - OperationMetricSet(const std::string& name, const std::string& tags, const std::string& description, metrics::MetricSet* owner) - : metrics::MetricSet(name, tags, description, owner), - pending("pending", - "logdefault yamasdefault", - "The number of operations pending", this), - ok("done_ok", - "logdefault yamasdefault", - "The number of operations successfully performed", this), - failed("done_failed", - "logdefault yamasdefault", - "The number of operations that failed", this) - { - } - + OperationMetricSet(const std::string& name, const std::string& tags, const std::string& description, MetricSet* owner); + ~OperationMetricSet(); }; class IdealStateMetricSet : public metrics::MetricSet @@ -44,72 +32,12 @@ public: metrics::LongAverageMetric startOperationsLatency; metrics::DoubleAverageMetric nodesPerMerge; - void createOperationMetrics() { - typedef IdealStateOperation ISO; - operations.resize(ISO::OPERATION_COUNT); - operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("delete_bucket", "logdefault yamasdefault", - "Operations to delete excess buckets on storage nodes", this)); - operations[ISO::MERGE_BUCKET] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("merge_bucket", "logdefault yamasdefault", - "Operations to merge buckets that are out of sync", this)); - operations[ISO::SPLIT_BUCKET] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("split_bucket", "logdefault yamasdefault", - "Operations to split buckets that are larger than the configured size", this)); - operations[ISO::JOIN_BUCKET] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("join_bucket", "logdefault yamasdefault", - "Operations to join buckets that in sum are smaller than the configured size", this)); - operations[ISO::SET_BUCKET_STATE] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("set_bucket_state", - "logdefault yamasdefault", - "Operations to set active/ready state for bucket copies", this)); - operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("garbage_collection", - "logdefault yamasdefault", - "Operations to garbage collect data from buckets", this)); - } - - IdealStateMetricSet() - : metrics::MetricSet("idealstate", "idealstate", - "Statistics for ideal state generation"), - idealstate_diff("idealstate_diff", "logdefault yamasdefault", - "A number representing the current difference from the ideal " - "state. This is a number that decreases steadily as the system " - "is getting closer to the ideal state", this), - buckets_toofewcopies("buckets_toofewcopies", "logdefault yamasdefault", - "The number of buckets the distributor controls that have less " - "than the desired redundancy", this), - buckets_toomanycopies("buckets_toomanycopies", "logdefault yamasdefault", - "The number of buckets the distributor controls that have more " - "than the desired redundancy", this), - buckets("buckets", "logdefault yamasdefault", - "The number of buckets the distributor controls", this), - buckets_notrusted("buckets_notrusted", "logdefault yamasdefault", - "The number of buckets that have no trusted copies.", this), - buckets_rechecking("buckets_rechecking", "logdefault yamasdefault", - "The number of buckets that we are rechecking for " - "ideal state operations", this), - startOperationsLatency("start_operations_latency", "", - "Time used in startOperations()", this), - nodesPerMerge("nodes_per_merge", "", - "The number of nodes involved in a single merge operation.", - this) - { - createOperationMetrics(); - } + void createOperationMetrics(); - void setPendingOperations(const std::vector<uint64_t>& newMetrics) { - for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { - operations[i]->pending.set(newMetrics[i]); - } + IdealStateMetricSet(); + ~IdealStateMetricSet(); - idealstate_diff.set( - operations[IdealStateOperation::DELETE_BUCKET]->pending.getLast() + - operations[IdealStateOperation::MERGE_BUCKET]->pending.getLast() * 10 + - operations[IdealStateOperation::SPLIT_BUCKET]->pending.getLast() * 4 + - operations[IdealStateOperation::JOIN_BUCKET]->pending.getLast() * 2 + - operations[IdealStateOperation::SET_BUCKET_STATE]->pending.getLast()); - } + void setPendingOperations(const std::vector<uint64_t>& newMetrics); }; } diff --git a/storage/src/vespa/storage/distributor/visitormetricsset.cpp b/storage/src/vespa/storage/distributor/visitormetricsset.cpp new file mode 100644 index 00000000000..ae8aeeb8463 --- /dev/null +++ b/storage/src/vespa/storage/distributor/visitormetricsset.cpp @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "visitormetricsset.h" +#include <vespa/metrics/loadmetric.hpp> +#include <vespa/metrics/summetric.hpp> + +namespace storage { + +using metrics::MetricSet; + +VisitorMetricSet::VisitorMetricSet(MetricSet* owner) + : MetricSet("visitor", "visitor", "", owner), + latency("latency", "", "Latency of visitor (in ms)", this), + failed("failed", "", "Number of visitors that failed or were aborted by the user", this) +{ } + +VisitorMetricSet::~VisitorMetricSet() { } + +MetricSet * +VisitorMetricSet::clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (VisitorMetricSet*) (new VisitorMetricSet(owner))->assignValues(*this); +} + +} + +template class metrics::LoadMetric<storage::VisitorMetricSet>; +template class metrics::SumMetric<storage::VisitorMetricSet>; diff --git a/storage/src/vespa/storage/distributor/visitormetricsset.h b/storage/src/vespa/storage/distributor/visitormetricsset.h index 122682b9e04..53981ec3834 100644 --- a/storage/src/vespa/storage/distributor/visitormetricsset.h +++ b/storage/src/vespa/storage/distributor/visitormetricsset.h @@ -10,24 +10,11 @@ struct VisitorMetricSet : public metrics::MetricSet { metrics::DoubleAverageMetric latency; metrics::LongCountMetric failed; - VisitorMetricSet(metrics::MetricSet* owner = 0) - : metrics::MetricSet("visitor", "visitor", "", owner), - latency("latency", "", "Latency of visitor (in ms)", this), - failed("failed", "", "Number of visitors that failed or were aborted by the user", this) - { - } - - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); - } - return (VisitorMetricSet*) - (new VisitorMetricSet(owner))->assignValues(*this); - } + VisitorMetricSet(MetricSet* owner = 0); + ~VisitorMetricSet(); + + MetricSet * clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; VisitorMetricSet* operator&() { return this; } }; diff --git a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt index 13d3afd9df0..8a03a1fc170 100644 --- a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_filestorpersistence OBJECT SOURCES + filestormetrics.cpp filestormanager.cpp filestorhandler.cpp filestorhandlerimpl.cpp diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp new file mode 100644 index 00000000000..73e187f7f1e --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "filestormetrics.h" +#include <vespa/metrics/loadmetric.hpp> +#include <vespa/metrics/summetric.hpp> + +namespace storage { + +using metrics::MetricSet; +using metrics::LoadTypeSet; + +FileStorThreadMetrics::Op::Op(const std::string& id, const std::string name, MetricSet* owner) + : MetricSet(id, id, name + " load in filestor thread", owner, "operationtype"), + _name(name), + count("count", "yamasdefault", "Number of requests processed.", this), + latency("latency", "yamasdefault", "Latency of successful requests.", this), + failed("failed", "yamasdefault", "Number of failed requests.", this) +{ } + +FileStorThreadMetrics::Op::~Op() { } + +MetricSet * +FileStorThreadMetrics::Op::clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (Op*) (new Op(getName(), _name, owner))->assignValues(*this); +} + +FileStorThreadMetrics::OpWithNotFound::OpWithNotFound(const std::string& id, const std::string name, MetricSet* owner) + : Op(id, name, owner), + notFound("not_found", "", "Number of requests that could not be " + "completed due to source document not found.", this) +{ } + +FileStorThreadMetrics::OpWithNotFound::~OpWithNotFound() { } + +MetricSet * +FileStorThreadMetrics::OpWithNotFound::clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (OpWithNotFound*) + (new OpWithNotFound(getName(), _name, owner)) + ->assignValues(*this); +} + +FileStorThreadMetrics::Update::Update(MetricSet* owner) + : OpWithNotFound("update", "Update", owner), + latencyRead("latency_read", "", "Latency of the source read in the request.", this) +{ } + +FileStorThreadMetrics::Update::~Update() { } + +MetricSet * +FileStorThreadMetrics::Update::clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (Update*) (new Update(owner))->assignValues(*this); +} + +FileStorThreadMetrics::Visitor::Visitor(MetricSet* owner) + : Op("visit", "Visit", owner), + documentsPerIterate("docs", "", "Number of entries read per iterate call", this) +{ } + +FileStorThreadMetrics::Visitor::~Visitor() { } + +MetricSet * +FileStorThreadMetrics::Visitor::clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (Visitor*) (new Visitor(owner))->assignValues(*this); +} + +FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std::string& desc, const LoadTypeSet& lt) + : MetricSet(name, "filestor partofsum thread", desc, NULL, "thread"), + operations("operations", "", "Number of operations processed.", this), + failedOperations("failedoperations", "", "Number of operations throwing exceptions.", this), + put(lt, *&Op("put", "Put"), this), + get(lt, *&OpWithNotFound("get", "Get"), this), + remove(lt, *&OpWithNotFound("remove", "Remove"), this), + removeLocation(lt, *&Op("remove_location", "Remove location"), this), + statBucket(lt, *&Op("stat_bucket", "Stat bucket"), this), + update(lt, *&Update(), this), + revert(lt, *&OpWithNotFound("revert", "Revert"), this), + createIterator("createiterator", "", this), + visit(lt, *&Visitor(), this), + multiOp(lt, *&Op("multioperations", "The number of multioperations that have been created"), this), + createBuckets("createbuckets", "Number of buckets that has been created.", this), + deleteBuckets("deletebuckets", "Number of buckets that has been deleted.", this), + repairs("bucketverified", "Number of times buckets have been checked.", this), + repairFixed("bucketfixed", "", "Number of times bucket has been fixed because of corruption", this), + recheckBucketInfo("recheckbucketinfo", + "Number of times bucket info has been explicitly " + "rechecked due to buckets being marked modified by " + "the persistence provider", + this), + splitBuckets("splitbuckets", "Number of times buckets have been split.", this), + joinBuckets("joinbuckets", "Number of times buckets have been joined.", this), + setBucketStates("setbucketstates", "Number of times buckets have been activated or deactivated.", this), + movedBuckets("movedbuckets", "Number of buckets moved between disks", this), + readBucketList("readbucketlist", "Number of read bucket list requests", this), + readBucketInfo("readbucketinfo", "Number of read bucket info requests", this), + internalJoin("internaljoin", "Number of joins to join buckets on multiple disks during " + "storage initialization.", this), + mergeBuckets("mergebuckets", "Number of times buckets have been merged.", this), + getBucketDiff("getbucketdiff", "Number of getbucketdiff commands that have been processed.", this), + applyBucketDiff("applybucketdiff", "Number of applybucketdiff commands that have been processed.", this), + bytesMerged("bytesmerged", "", "Total number of bytes merged into this node.", this), + getBucketDiffReply("getbucketdiffreply", "", "Number of getbucketdiff replies that have been processed.", this), + applyBucketDiffReply("applybucketdiffreply", "", "Number of applybucketdiff replies that have been processed.", this), + mergeLatencyTotal("mergelatencytotal", "", + "Latency of total merge operation, from master node receives " + "it, until merge is complete and master node replies.", this), + mergeMetadataReadLatency("mergemetadatareadlatency", "", + "Latency of time used in a merge step to check metadata of " + "current node to see what data it has.", this), + mergeDataReadLatency("mergedatareadlatency", "", + "Latency of time used in a merge step to read data other " + "nodes need.", this), + mergeDataWriteLatency("mergedatawritelatency", "", + "Latency of time used in a merge step to write data needed to " + "current node.", this), + mergeAverageDataReceivedNeeded("mergeavgdatareceivedneeded", "", "Amount of data transferred from previous node " + "in chain that we needed to apply locally.", this), + batchingSize("batchingsize", "", "Number of operations batched per bucket (only counts " + "batches of size > 1)", this) +{ } + +FileStorThreadMetrics::~FileStorThreadMetrics() { } + +FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, + const std::string& description, + const LoadTypeSet& loadTypes, + MetricSet* owner) + : MetricSet(name, "partofsum disk", description, owner, "disk"), + sum("allthreads", "sum", "", this), + queueSize("queuesize", "", "Size of input message queue.", this), + averageQueueWaitingTime(loadTypes, metrics::LongAverageMetric( + "averagequeuewait", "", + "Average time an operation spends in input queue."), this), + pendingMerges("pendingmerge", "", "Number of buckets currently being merged.", this), + waitingForLockHitRate("waitingforlockrate", "", + "Amount of times a filestor thread has needed to wait for " + "lock to take next message in queue.", this), + lockWaitTime("lockwaittime", "", "Amount of time waiting used waiting for lock.", this) +{ + pendingMerges.unsetOnZeroValue(); + waitingForLockHitRate.unsetOnZeroValue(); +} + +FileStorDiskMetrics::~FileStorDiskMetrics() { } + +void +FileStorDiskMetrics::initDiskMetrics(const LoadTypeSet& loadTypes, uint32_t threadsPerDisk) +{ + threads.clear(); + threads.resize(threadsPerDisk); + for (uint32_t i=0; i<threadsPerDisk; ++i) { + std::ostringstream desc; + std::ostringstream name; + name << "thread" << i; + desc << "Thread " << i << '/' << threadsPerDisk; + threads[i] = std::shared_ptr<FileStorThreadMetrics>(new FileStorThreadMetrics(name.str(), desc.str(), loadTypes)); + registerMetric(*threads[i]); + sum.addMetricToSum(*threads[i]); + } +} + +FileStorMetrics::FileStorMetrics(const LoadTypeSet&) + : MetricSet("filestor", "filestor", ""), + sum("alldisks", "sum", "", this), + directoryEvents("directoryevents", "", "Number of directory events received.", this), + partitionEvents("partitionevents", "", "Number of partition events received.", this), + diskEvents("diskevents", "", "Number of disk events received.", this) +{ } + +FileStorMetrics::~FileStorMetrics() { } + +void FileStorMetrics::initDiskMetrics(uint16_t numDisks, const LoadTypeSet& loadTypes, uint32_t threadsPerDisk) +{ + if (!disks.empty()) { + throw vespalib::IllegalStateException("Can't initialize disks twice", VESPA_STRLOC); + } + disks.clear(); + disks.resize(numDisks); + for (uint32_t i=0; i<numDisks; ++i) { + // Currently FileStorHandlerImpl expects metrics to exist for + // disks that are not in use too. + std::ostringstream desc; + std::ostringstream name; + name << "disk_" << i; + desc << "Disk " << i; + disks[i] = FileStorDiskMetrics::SP(new FileStorDiskMetrics( name.str(), desc.str(), loadTypes, this)); + sum.addMetricToSum(*disks[i]); + disks[i]->initDiskMetrics(loadTypes, threadsPerDisk); + } +} + +} + +template class metrics::LoadMetric<storage::FileStorThreadMetrics::Op>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::OpWithNotFound>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::Update>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::Visitor>; +template class metrics::SumMetric<storage::FileStorThreadMetrics::Op>; +template class metrics::SumMetric<storage::FileStorThreadMetrics::OpWithNotFound>; +template class metrics::SumMetric<storage::FileStorThreadMetrics::Update>; +template class metrics::SumMetric<storage::FileStorThreadMetrics::Visitor>; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index f9c8f25a5dc..5d4b9f618e3 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -25,114 +25,42 @@ struct FileStorThreadMetrics : public metrics::MetricSet metrics::LongAverageMetric latency; metrics::LongCountMetric failed; - Op(const std::string& id, const std::string name, - metrics::MetricSet* owner = 0) - : MetricSet(id, - id, - name + " load in filestor thread", - owner, - "operationtype"), - _name(name), - count("count", - "yamasdefault", - "Number of requests processed.", - this), - latency("latency", - "yamasdefault", - "Latency of successful requests.", - this), - failed("failed", - "yamasdefault", - "Number of failed requests.", - this) - { - } + Op(const std::string& id, const std::string name, MetricSet* owner = 0); + ~Op(); - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone( - ownerList, INACTIVE, owner, includeUnused); - } - return (Op*) (new Op(getName(), _name, owner))->assignValues(*this); - } + MetricSet * clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; Op* operator&() { return this; } }; struct OpWithNotFound : public Op { metrics::LongCountMetric notFound; - OpWithNotFound(const std::string& id, const std::string name, - metrics::MetricSet* owner = 0) - : Op(id, name, owner), - notFound("not_found", "", "Number of requests that could not be " - "completed due to source document not found.", this) - { - } - - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone( - ownerList, INACTIVE, owner, includeUnused); - } - return (OpWithNotFound*) - (new OpWithNotFound(getName(), _name, owner)) - ->assignValues(*this); - } + OpWithNotFound(const std::string& id, const std::string name, metrics::MetricSet* owner = 0); + ~OpWithNotFound(); + MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; OpWithNotFound* operator&() { return this; } }; struct Update : public OpWithNotFound { metrics::LongAverageMetric latencyRead; - Update(metrics::MetricSet* owner = 0) - : OpWithNotFound("update", "Update", owner), - latencyRead("latency_read", "", "Latency of the source read in " - "the request.", this) - { - } + Update(MetricSet* owner = 0); + ~Update(); - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone( - ownerList, INACTIVE, owner, includeUnused); - } - return (Update*) (new Update(owner))->assignValues(*this); - } + MetricSet* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; Update* operator&() { return this; } }; struct Visitor : public Op { metrics::LongAverageMetric documentsPerIterate; - Visitor(metrics::MetricSet* owner = 0) - : Op("visit", "Visit", owner), - documentsPerIterate("docs", "", "Number of entries read per iterate call", - this) - { - } + Visitor(MetricSet* owner = 0); + ~Visitor(); - virtual Metric* clone(std::vector<Metric::LP>& ownerList, - CopyType copyType, - metrics::MetricSet* owner, - bool includeUnused) const - { - if (copyType == INACTIVE) { - return MetricSet::clone( - ownerList, INACTIVE, owner, includeUnused); - } - return (Visitor*) (new Visitor(owner))->assignValues(*this); - } + MetricSet * clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; Visitor* operator&() { return this; } }; @@ -174,89 +102,8 @@ struct FileStorThreadMetrics : public metrics::MetricSet metrics::DoubleAverageMetric mergeAverageDataReceivedNeeded; metrics::LongAverageMetric batchingSize; - FileStorThreadMetrics(const std::string& name, const std::string& desc, - const metrics::LoadTypeSet& lt) - : metrics::MetricSet(name, "filestor partofsum thread", desc, NULL, "thread"), - operations("operations", "", - "Number of operations processed.", this), - failedOperations("failedoperations", "", - "Number of operations throwing exceptions.", this), - - put(lt, *&Op("put", "Put"), this), - get(lt, *&OpWithNotFound("get", "Get"), this), - remove(lt, *&OpWithNotFound("remove", "Remove"), this), - removeLocation(lt, *&Op("remove_location", "Remove location"), this), - statBucket(lt, *&Op("stat_bucket", "Stat bucket"), this), - update(lt, *&Update(), this), - revert(lt, *&OpWithNotFound("revert", "Revert"), this), - createIterator("createiterator", "", this), - visit(lt, *&Visitor(), this), - multiOp(lt, *&Op("multioperations", - "The number of multioperations that have been created"), this), - createBuckets("createbuckets", - "Number of buckets that has been created.", this), - deleteBuckets("deletebuckets", - "Number of buckets that has been deleted.", this), - repairs("bucketverified", "Number of times buckets have been checked.", this), - repairFixed("bucketfixed", "", - "Number of times bucket has been fixed because of " - "corruption", this), - recheckBucketInfo("recheckbucketinfo", - "Number of times bucket info has been explicitly " - "rechecked due to buckets being marked modified by " - "the persistence provider", - this), - splitBuckets("splitbuckets", - "Number of times buckets have been split.", this), - joinBuckets("joinbuckets", - "Number of times buckets have been joined.", this), - setBucketStates("setbucketstates", - "Number of times buckets have been activated or deactivated.", this), - movedBuckets("movedbuckets", - "Number of buckets moved between disks", this), - readBucketList("readbucketlist", - "Number of read bucket list requests", this), - readBucketInfo("readbucketinfo", - "Number of read bucket info requests", this), - internalJoin("internaljoin", - "Number of joins to join buckets on multiple disks during " - "storage initialization.", this), - mergeBuckets("mergebuckets", - "Number of times buckets have been merged.", this), - getBucketDiff("getbucketdiff", - "Number of getbucketdiff commands that have been processed.", this), - applyBucketDiff("applybucketdiff", - "Number of applybucketdiff commands that have been processed.", this), - bytesMerged("bytesmerged", "", - "Total number of bytes merged into this node.", this), - getBucketDiffReply("getbucketdiffreply", "", - "Number of getbucketdiff replies that have been processed.", this), - applyBucketDiffReply("applybucketdiffreply", "", - "Number of applybucketdiff replies that have been processed.", this), - mergeLatencyTotal("mergelatencytotal", "", - "Latency of total merge operation, from master node receives " - "it, until merge is complete and master node replies.", this), - mergeMetadataReadLatency("mergemetadatareadlatency", "", - "Latency of time used in a merge step to check metadata of " - "current node to see what data it has.", this), - mergeDataReadLatency("mergedatareadlatency", "", - "Latency of time used in a merge step to read data other " - "nodes need.", this), - mergeDataWriteLatency("mergedatawritelatency", "", - "Latency of time used in a merge step to write data needed to " - "current node.", this), - mergeAverageDataReceivedNeeded("mergeavgdatareceivedneeded", - "", - "Amount of data transferred from previous node " - "in chain that " - "we needed to apply locally.", this), - batchingSize("batchingsize", - "", - "Number of operations batched per bucket (only counts " - "batches of size > 1)", this) - { - } - + FileStorThreadMetrics(const std::string& name, const std::string& desc, const metrics::LoadTypeSet& lt); + ~FileStorThreadMetrics(); }; class FileStorDiskMetrics : public metrics::MetricSet @@ -272,46 +119,11 @@ public: metrics::DoubleAverageMetric waitingForLockHitRate; metrics::LongAverageMetric lockWaitTime; - FileStorDiskMetrics(const std::string& name, - const std::string& description, - const metrics::LoadTypeSet& loadTypes, - metrics::MetricSet* owner) - : MetricSet(name, "partofsum disk", description, owner, "disk"), - sum("allthreads", "sum", "", this), - queueSize("queuesize", "", "Size of input message queue.", this), - averageQueueWaitingTime(loadTypes, metrics::LongAverageMetric( - "averagequeuewait", "", - "Average time an operation spends in input queue."), this), - pendingMerges("pendingmerge", "", - "Number of buckets currently being merged.", this), - waitingForLockHitRate("waitingforlockrate", "", - "Amount of times a filestor thread has needed to wait for " - "lock to take next message in queue.", this), - lockWaitTime("lockwaittime", "", - "Amount of time waiting used waiting for lock.", this) - { - pendingMerges.unsetOnZeroValue(); - waitingForLockHitRate.unsetOnZeroValue(); - } + FileStorDiskMetrics(const std::string& name, const std::string& description, + const metrics::LoadTypeSet& loadTypes, MetricSet* owner); + ~FileStorDiskMetrics(); - void initDiskMetrics(const metrics::LoadTypeSet& loadTypes, - uint32_t threadsPerDisk) - { - threads.clear(); - threads.resize(threadsPerDisk); - for (uint32_t i=0; i<threadsPerDisk; ++i) { - std::ostringstream desc; - std::ostringstream name; - name << "thread" << i; - desc << "Thread " << i << '/' << threadsPerDisk; - threads[i] - = std::shared_ptr<FileStorThreadMetrics>( - new FileStorThreadMetrics(name.str(), desc.str(), - loadTypes)); - registerMetric(*threads[i]); - sum.addMetricToSum(*threads[i]); - } - } + void initDiskMetrics(const metrics::LoadTypeSet& loadTypes, uint32_t threadsPerDisk); }; struct FileStorMetrics : public metrics::MetricSet @@ -322,41 +134,10 @@ struct FileStorMetrics : public metrics::MetricSet metrics::LongCountMetric partitionEvents; metrics::LongCountMetric diskEvents; - FileStorMetrics(const metrics::LoadTypeSet&) - : metrics::MetricSet("filestor", "filestor", ""), - sum("alldisks", "sum", "", this), - directoryEvents("directoryevents", "", - "Number of directory events received.", this), - partitionEvents("partitionevents", "", - "Number of partition events received.", this), - diskEvents("diskevents", "", - "Number of disk events received.", this) - { - } + FileStorMetrics(const metrics::LoadTypeSet&); + ~FileStorMetrics(); - void initDiskMetrics(uint16_t numDisks, - const metrics::LoadTypeSet& loadTypes, - uint32_t threadsPerDisk) - { - if (!disks.empty()) { - throw vespalib::IllegalStateException( - "Can't initialize disks twice", VESPA_STRLOC); - } - disks.clear(); - disks.resize(numDisks); - for (uint32_t i=0; i<numDisks; ++i) { - // Currently FileStorHandlerImpl expects metrics to exist for - // disks that are not in use too. - std::ostringstream desc; - std::ostringstream name; - name << "disk_" << i; - desc << "Disk " << i; - disks[i] = FileStorDiskMetrics::SP(new FileStorDiskMetrics( - name.str(), desc.str(), loadTypes, this)); - sum.addMetricToSum(*disks[i]); - disks[i]->initDiskMetrics(loadTypes, threadsPerDisk); - } - } + void initDiskMetrics(uint16_t numDisks, const metrics::LoadTypeSet& loadTypes, uint32_t threadsPerDisk); }; } |