diff options
author | Tor Brede Vekterli <vekterli@oath.com> | 2018-12-13 16:20:22 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@oath.com> | 2018-12-18 13:06:01 +0000 |
commit | c695946680c4df612adb3e5249bfea1a9bdf9cc3 (patch) | |
tree | 23cbf13cf95ad7d0d951c9b2754b18a708c51c4f /storage/src | |
parent | b488e40f4ed89d73341ccf76bc96cba7a79a94f6 (diff) |
Add TLS statistics to vespalib and expose as metrics via storageserver
Now without unused expiry time extraction.
Diffstat (limited to 'storage/src')
6 files changed, 123 insertions, 8 deletions
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 2df3d3a9606..73873e78032 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver storagemetricsset.cpp storagenode.cpp storagenodecontext.cpp + tls_statistics_metrics_wrapper.cpp INSTALL lib64 DEPENDS storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index 4ea9a9f9296..f0e64f0dfd1 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this), highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this), veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this) -{ } -MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {} +{} + +MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) : metrics::MetricSet("document_serialization", {{"docserialization"}}, @@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe "Number of times we reserialized a document because the " "compression it had in cache did not match what was configured", this) -{ } -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { } +{} + +DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, @@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this) -{ } -StorageMetricSet::~StorageMetricSet() { } + documentSerialization(this), + tls_metrics(this) +{} + +StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { document::SerializableArray::Statistics stats( @@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() { stats._serializedUncompressed); documentSerialization.inputWronglySerialized.set( stats._inputWronglySerialized); + + // Delta snapshotting is destructive, so if an explicit snapshot is triggered + // (instead of just regular periodic snapshots), some events will effectively + // be erased from history. This will no longer be a problem once we move to a + // metrics system built around absolute (rather than derived) values. + tls_metrics.update_metrics_with_snapshot_delta(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index 40b70821bcd..e9378010540 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -2,6 +2,8 @@ #pragma once +#include "tls_statistics_metrics_wrapper.h" + #include <vespa/metrics/metrics.h> namespace storage { @@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse_visiting; DocumentSerializationMetricSet documentSerialization; + TlsStatisticsMetricsWrapper tls_metrics; + StorageMetricSet(); ~StorageMetricSet(); void updateMetrics(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index d159b6e5bdd..64c0970cf9d 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -7,6 +7,7 @@ #include "statereporter.h" #include "storagemetricsset.h" #include "storagenodecontext.h" +#include "tls_statistics_metrics_wrapper.h" #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> @@ -159,7 +160,7 @@ StorageNode::initialize() _context.getComponentRegister().setPriorityConfig(*_priorityConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - _metrics.reset(new StorageMetricSet); + _metrics = std::make_shared<StorageMetricSet>(); _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode")); _component->registerMetric(*_metrics); if (!_context.getComponentRegister().hasMetricManager()) { diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp new file mode 100644 index 00000000000..707e1c84036 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tls_statistics_metrics_wrapper.h" + +namespace storage { + +TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner) + : metrics::MetricSet("network", {}, "Network connection metrics", owner), + insecure_client_connections_established("insecure_client_connections_established", {}, + "Number of insecure (plaintext) client connections established", this), + insecure_server_connections_accepted("insecure_server_connections_accepted", {}, + "Number of insecure (plaintext) server connections accepted", this), + tls_client_connections_established("tls_client_connections_established", {}, + "Number of secure mTLS client connections established", this), + tls_server_connections_accepted("tls_server_connections_accepted", {}, + "Number of secure mTLS server connections accepted", this), + tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or " + "server connection attempts that failed during TLS handshaking", this), + peer_authorization_failures("peer_authorization_failures", {}, + "Number of TLS connection attempts failed due to bad or missing " + "peer certificate credentials", this), + tls_connections_broken("tls_connections_broken", {}, "Number of TLS " + "connections broken due to failures during frame encoding or decoding", this), + failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times " + "background reloading of TLS config has failed", this), + last_client_stats_snapshot(), + last_server_stats_snapshot(), + last_config_stats_snapshot() +{} + +TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default; + +void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() { + auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot(); + auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot(); + auto server_delta = server_current.subtract(last_server_stats_snapshot); + auto client_delta = client_current.subtract(last_client_stats_snapshot); + + insecure_client_connections_established.set(client_delta.insecure_connections); + insecure_server_connections_accepted.set(server_delta.insecure_connections); + tls_client_connections_established.set(client_delta.tls_connections); + tls_server_connections_accepted.set(server_delta.tls_connections); + // We have underlying stats for both server and client here, but for the + // moment we just aggregate them up into combined metrics. Can be trivially + // split up into separate metrics later if deemed useful. + tls_handshakes_failed.set(client_delta.failed_tls_handshakes + + server_delta.failed_tls_handshakes); + peer_authorization_failures.set(client_delta.invalid_peer_credentials + + server_delta.invalid_peer_credentials); + tls_connections_broken.set(client_delta.broken_tls_connections + + server_delta.broken_tls_connections); + + auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot(); + auto config_delta = config_current.subtract(last_config_stats_snapshot); + + failed_tls_config_reloads.set(config_delta.failed_config_reloads); + + last_server_stats_snapshot = server_current; + last_client_stats_snapshot = client_current; + last_config_stats_snapshot = config_current; +} + +} diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h new file mode 100644 index 00000000000..9784aaddd1c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/metrics/metrics.h> +#include <vespa/vespalib/net/tls/statistics.h> + +#include <chrono> + +namespace storage { + +// Simple wrapper around low-level vespalib network statistics which +// converts the monotonically increasing counters to deltas during +// periodic metric snapshotting. +class TlsStatisticsMetricsWrapper : public metrics::MetricSet { + metrics::LongCountMetric insecure_client_connections_established; + metrics::LongCountMetric insecure_server_connections_accepted; + metrics::LongCountMetric tls_client_connections_established; + metrics::LongCountMetric tls_server_connections_accepted; + metrics::LongCountMetric tls_handshakes_failed; + metrics::LongCountMetric peer_authorization_failures; + metrics::LongCountMetric tls_connections_broken; + + metrics::LongCountMetric failed_tls_config_reloads; + + vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot; + vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot; + vespalib::net::tls::ConfigStatistics::Snapshot last_config_stats_snapshot; + +public: + explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner); + ~TlsStatisticsMetricsWrapper() override; + + void update_metrics_with_snapshot_delta(); +}; + +} |