diff options
Diffstat (limited to 'memfilepersistence')
6 files changed, 173 insertions, 133 deletions
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt index 166f8499725..8ee8a93df47 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(memfilepersistence_mapper OBJECT SOURCES buffer.cpp memfilemapper.cpp + serializationmetrics.cpp memfile_v1_serializer.cpp memfile_v1_verifier.cpp locationreadplanner.cpp diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp new file mode 100644 index 00000000000..366b0ae33e0 --- /dev/null +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializationmetrics.h" + +namespace storage { +namespace memfile { + +SerializationWriteMetrics::SerializationWriteMetrics(const std::string& name, MetricSet& owner) + : MetricSet(name, "", + "Write metrics for memfile persistence engine", + &owner), + headerLatency("header_latency", "", + "Time spent writing a single contiguous header location " + "on the disk.", this), + headerSize("header_size", "", + "Average size of contiguous header disk writes", this), + bodyLatency("body_latency", "", + "Time spent writing a single contiguous body location " + "on the disk.", this), + bodySize("body_size", "", + "Average size of contiguous body disk writes", this), + metaLatency("meta_latency", "", + "Time spent writing file header and slot metadata", this), + metaSize("meta_size", "", + "Size of file header and metadata writes", this), + totalLatency("total_latency", "", + "Total time spent performing slot file writing", this) +{ } + +SerializationWriteMetrics::~SerializationWriteMetrics() { } + +SerializationMetrics::SerializationMetrics(const std::string& name, MetricSet* owner) + : MetricSet(name, "", + "(De-)serialization I/O metrics for memfile " + "persistence engine", owner), + initialMetaReadLatency( + "initial_meta_read_latency", "", + "Time spent doing the initial read of " + "the file header and most (or all) of metadata", + this), + tooLargeMetaReadLatency( + "too_large_meta_read_latency", "", + "Time spent doing additional read for " + "metadata too large to be covered by initial " + "read", this), + totalLoadFileLatency( + "total_load_file_latency", "", + "Total time spent initially loading a " + "file from disk", this), + verifyLatency( + "verify_latency", "", + "Time spent performing file verification", this), + deleteFileLatency( + "delete_file_latency", "", + "Time spent deleting a file from disk", this), + headerReadLatency( + "header_read_latency", "", + "Time spent reading a single contiguous header location " + "on the disk (may span many document blobs)", this), + headerReadSize( + "header_read_size", "", + "Size of contiguous header disk location reads", this), + bodyReadLatency( + "body_read_latency", "", + "Time spent reading a single contiguous body location " + "on the disk (may span many document blobs)", this), + bodyReadSize( + "body_read_size", "", + "Size of contiguous body disk location reads", this), + cacheUpdateAndImplicitVerifyLatency( + "cache_update_and_implicit_verify_latency", "", + "Time spent updating memory cache structures and verifying " + "read data blocks for corruptions", this), + fullRewritesDueToDownsizingFile( + "full_rewrites_due_to_downsizing_file", "", + "Number of times a file was rewritten fully because the " + "original file had too low fill rate", this), + fullRewritesDueToTooSmallFile( + "full_rewrites_due_to_too_small_file", "", + "Number of times a file was rewritten fully because the " + "original file did not have sufficient free space for a " + "partial write", this), + partialWrite("partialwrite", *this), + fullWrite("fullwrite", *this) +{ } + +SerializationMetrics::~SerializationMetrics() { } + +} // memfile +} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h index 0eac46a1065..ac55b546b9e 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h @@ -17,28 +17,8 @@ public: metrics::LongAverageMetric metaSize; metrics::LongAverageMetric totalLatency; - SerializationWriteMetrics(const std::string& name, metrics::MetricSet& owner) - : metrics::MetricSet(name, "", - "Write metrics for memfile persistence engine", - &owner), - headerLatency("header_latency", "", - "Time spent writing a single contiguous header location " - "on the disk.", this), - headerSize("header_size", "", - "Average size of contiguous header disk writes", this), - bodyLatency("body_latency", "", - "Time spent writing a single contiguous body location " - "on the disk.", this), - bodySize("body_size", "", - "Average size of contiguous body disk writes", this), - metaLatency("meta_latency", "", - "Time spent writing file header and slot metadata", this), - metaSize("meta_size", "", - "Size of file header and metadata writes", this), - totalLatency("total_latency", "", - "Total time spent performing slot file writing", this) - { - } + SerializationWriteMetrics(const std::string& name, MetricSet& owner); + ~SerializationWriteMetrics(); }; class SerializationMetrics : public metrics::MetricSet @@ -59,62 +39,8 @@ public: SerializationWriteMetrics partialWrite; SerializationWriteMetrics fullWrite; - SerializationMetrics(const std::string& name, - metrics::MetricSet* owner = 0) - : metrics::MetricSet(name, "", - "(De-)serialization I/O metrics for memfile " - "persistence engine", owner), - initialMetaReadLatency( - "initial_meta_read_latency", "", - "Time spent doing the initial read of " - "the file header and most (or all) of metadata", - this), - tooLargeMetaReadLatency( - "too_large_meta_read_latency", "", - "Time spent doing additional read for " - "metadata too large to be covered by initial " - "read", this), - totalLoadFileLatency( - "total_load_file_latency", "", - "Total time spent initially loading a " - "file from disk", this), - verifyLatency( - "verify_latency", "", - "Time spent performing file verification", this), - deleteFileLatency( - "delete_file_latency", "", - "Time spent deleting a file from disk", this), - headerReadLatency( - "header_read_latency", "", - "Time spent reading a single contiguous header location " - "on the disk (may span many document blobs)", this), - headerReadSize( - "header_read_size", "", - "Size of contiguous header disk location reads", this), - bodyReadLatency( - "body_read_latency", "", - "Time spent reading a single contiguous body location " - "on the disk (may span many document blobs)", this), - bodyReadSize( - "body_read_size", "", - "Size of contiguous body disk location reads", this), - cacheUpdateAndImplicitVerifyLatency( - "cache_update_and_implicit_verify_latency", "", - "Time spent updating memory cache structures and verifying " - "read data blocks for corruptions", this), - fullRewritesDueToDownsizingFile( - "full_rewrites_due_to_downsizing_file", "", - "Number of times a file was rewritten fully because the " - "original file had too low fill rate", this), - fullRewritesDueToTooSmallFile( - "full_rewrites_due_to_too_small_file", "", - "Number of times a file was rewritten fully because the " - "original file did not have sufficient free space for a " - "partial write", this), - partialWrite("partialwrite", *this), - fullWrite("fullwrite", *this) - { - } + SerializationMetrics(const std::string& name, MetricSet* owner = 0); + ~SerializationMetrics(); }; } // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt index e30807d99b2..885b0edb8dc 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(memfilepersistence_spi OBJECT SOURCES memfilepersistence.cpp memfilepersistenceprovider.cpp + memfilepersistenceprovidermetrics.cpp operationhandler.cpp iteratorhandler.cpp joinoperationhandler.cpp diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp new file mode 100644 index 00000000000..e3fa4337fd5 --- /dev/null +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "memfilepersistenceprovidermetrics.h" +#include <vespa/metrics/summetric.hpp> +#include <vespa/storageframework/generic/component/component.h> + +namespace storage { +namespace memfile { + +using metrics::MetricSet; + +MemFilePersistenceThreadMetrics::MemFilePersistenceThreadMetrics(const std::string& name, MetricSet& owner) + : MetricSet(name, "partofsum thread", "Metrics for a worker thread using memfile persistence provider", &owner), + headerOnlyGets("headeronlygets", "", "Number of gets that only read header", this), + headerOnlyUpdates("headeronlyupdates", "", "Number of updates that only wrote header", this), + serialization("serialization", this) +{ } + +MemFilePersistenceThreadMetrics::~MemFilePersistenceThreadMetrics() { } + +MemFilePersistenceCacheMetrics::MemFilePersistenceCacheMetrics(MetricSet& owner) + : MetricSet("cache", "", "Metrics for the VDS persistence cache", &owner), + files("files", "", "Number of files cached", this), + meta("meta", "", "Bytes of file metadata cached", this), + header("header", "", "Bytes of file header parts cached", this), + body("body", "", "Bytes of file body parts cached", this), + hits("hits", "", "Number of times a bucket was attempted fetched " + "from the cache and it was already present", this), + misses("misses", "", "Number of times a bucket was attempted fetched " + "from the cache and it could not be found, requiring a load", this), + meta_evictions("meta_evictions", "", "Bucket meta data evictions", this), + header_evictions("header_evictions", "", "Bucket header (and " + "implicitly body, if present) data evictions", this), + body_evictions("body_evictions", "", "Bucket body data evictions", this) +{ } + +MemFilePersistenceCacheMetrics::~MemFilePersistenceCacheMetrics() { } + +MemFilePersistenceMetrics::MemFilePersistenceMetrics(framework::Component& component) + : MetricSet("memfilepersistence", "", "Metrics for the VDS persistence layer"), + _component(component), + _cache(*this) +{ } + +MemFilePersistenceMetrics::~MemFilePersistenceMetrics() { } + +MemFilePersistenceThreadMetrics* +MemFilePersistenceMetrics::addThreadMetrics() { + metrics::MetricLockGuard metricLock(_component.getMetricManagerLock()); + vespalib::LockGuard guard(_threadMetricsLock); + + if (!_sumMetric.get()) { + _sumMetric.reset(new metrics::SumMetric<MemFilePersistenceThreadMetrics> + ("allthreads", "sum", "", this)); + } + + std::string name = vespalib::make_string("thread_%zu", _threadMetrics.size()); + auto metrics = std::make_unique<MemFilePersistenceThreadMetrics>(name, *this); + _threadMetrics.emplace_back(std::move(metrics)); + _sumMetric->addMetricToSum(*metrics); + return metrics.get(); +} + +} +} + +template class metrics::SumMetric<storage::memfile::MemFilePersistenceThreadMetrics>; diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h index 70c711e81fd..6c6e1d882d1 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h @@ -5,6 +5,9 @@ #include <vespa/memfilepersistence/mapper/serializationmetrics.h> namespace storage { + +namespace framework { class Component; } + namespace memfile { class MemFilePersistenceThreadMetrics : public metrics::MetricSet @@ -14,17 +17,8 @@ public: metrics::LongCountMetric headerOnlyUpdates; SerializationMetrics serialization; - MemFilePersistenceThreadMetrics(const std::string& name, metrics::MetricSet& owner) - : metrics::MetricSet(name, "partofsum thread", - "Metrics for a worker thread using memfile persistence " - "provider", &owner), - headerOnlyGets("headeronlygets", "", - "Number of gets that only read header", this), - headerOnlyUpdates("headeronlyupdates", "", - "Number of updates that only wrote header", this), - serialization("serialization", this) - { - } + MemFilePersistenceThreadMetrics(const std::string& name, metrics::MetricSet& owner); + ~MemFilePersistenceThreadMetrics(); }; class MemFilePersistenceCacheMetrics : public metrics::MetricSet @@ -40,22 +34,8 @@ public: metrics::LongCountMetric header_evictions; metrics::LongCountMetric body_evictions; - MemFilePersistenceCacheMetrics(metrics::MetricSet& owner) - : metrics::MetricSet("cache", "", - "Metrics for the VDS persistence cache", &owner), - files("files", "", "Number of files cached", this), - meta("meta", "", "Bytes of file metadata cached", this), - header("header", "", "Bytes of file header parts cached", this), - body("body", "", "Bytes of file body parts cached", this), - hits("hits", "", "Number of times a bucket was attempted fetched " - "from the cache and it was already present", this), - misses("misses", "", "Number of times a bucket was attempted fetched " - "from the cache and it could not be found, requiring a load", this), - meta_evictions("meta_evictions", "", "Bucket meta data evictions", this), - header_evictions("header_evictions", "", "Bucket header (and " - "implicitly body, if present) data evictions", this), - body_evictions("body_evictions", "", "Bucket body data evictions", this) - {} + MemFilePersistenceCacheMetrics(metrics::MetricSet& owner); + ~MemFilePersistenceCacheMetrics(); }; class MemFilePersistenceMetrics : public metrics::MetricSet @@ -64,38 +44,14 @@ class MemFilePersistenceMetrics : public metrics::MetricSet public: vespalib::Lock _threadMetricsLock; - std::list<vespalib::LinkedPtr<MemFilePersistenceThreadMetrics> > _threadMetrics; + std::list<std::unique_ptr<MemFilePersistenceThreadMetrics> > _threadMetrics; std::unique_ptr<metrics::SumMetric<MemFilePersistenceThreadMetrics> > _sumMetric; MemFilePersistenceCacheMetrics _cache; - MemFilePersistenceMetrics(framework::Component& component) - : metrics::MetricSet("memfilepersistence", "", - "Metrics for the VDS persistence layer"), - _component(component), - _cache(*this) - { - } - - MemFilePersistenceThreadMetrics* addThreadMetrics() { - metrics::MetricLockGuard metricLock(_component.getMetricManagerLock()); - vespalib::LockGuard guard(_threadMetricsLock); - - if (!_sumMetric.get()) { - _sumMetric.reset(new metrics::SumMetric<MemFilePersistenceThreadMetrics> - ("allthreads", "sum", "", this)); - } - - std::string name = vespalib::make_string("thread_%zu", _threadMetrics.size()); - - MemFilePersistenceThreadMetrics* metrics = - new MemFilePersistenceThreadMetrics(name, *this); - - _threadMetrics.push_back(vespalib::LinkedPtr<MemFilePersistenceThreadMetrics>( - metrics)); - _sumMetric->addMetricToSum(*metrics); - return metrics; - } + MemFilePersistenceMetrics(framework::Component& component); + ~MemFilePersistenceMetrics(); + MemFilePersistenceThreadMetrics* addThreadMetrics(); }; } |