From 0bfa7d73757596e4c402002dd21399df1b0b00e9 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 13 Dec 2018 16:20:22 +0000 Subject: Add TLS statistics to vespalib and expose as metrics via storageserver Also add functionality for extracting "notAfter" expiration time from current certificate, which may later be added as an expiry metric. --- .../src/vespa/storage/storageserver/CMakeLists.txt | 1 + .../storage/storageserver/storagemetricsset.cpp | 24 ++++++--- .../storage/storageserver/storagemetricsset.h | 4 ++ .../vespa/storage/storageserver/storagenode.cpp | 3 +- .../tls_statistics_metrics_wrapper.cpp | 63 ++++++++++++++++++++++ .../storageserver/tls_statistics_metrics_wrapper.h | 36 +++++++++++++ 6 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp create mode 100644 storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h (limited to 'storage') diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 2df3d3a9606..73873e78032 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver storagemetricsset.cpp storagenode.cpp storagenodecontext.cpp + tls_statistics_metrics_wrapper.cpp INSTALL lib64 DEPENDS storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index 4ea9a9f9296..f0e64f0dfd1 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this), highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this), veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this) -{ } -MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {} +{} + +MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) : metrics::MetricSet("document_serialization", {{"docserialization"}}, @@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe "Number of times we reserialized a document because the " "compression it had in cache did not match what was configured", this) -{ } -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { } +{} + +DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, @@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this) -{ } -StorageMetricSet::~StorageMetricSet() { } + documentSerialization(this), + tls_metrics(this) +{} + +StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { document::SerializableArray::Statistics stats( @@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() { stats._serializedUncompressed); documentSerialization.inputWronglySerialized.set( stats._inputWronglySerialized); + + // Delta snapshotting is destructive, so if an explicit snapshot is triggered + // (instead of just regular periodic snapshots), some events will effectively + // be erased from history. This will no longer be a problem once we move to a + // metrics system built around absolute (rather than derived) values. + tls_metrics.update_metrics_with_snapshot_delta(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index 40b70821bcd..e9378010540 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -2,6 +2,8 @@ #pragma once +#include "tls_statistics_metrics_wrapper.h" + #include namespace storage { @@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse_visiting; DocumentSerializationMetricSet documentSerialization; + TlsStatisticsMetricsWrapper tls_metrics; + StorageMetricSet(); ~StorageMetricSet(); void updateMetrics(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index d159b6e5bdd..64c0970cf9d 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -7,6 +7,7 @@ #include "statereporter.h" #include "storagemetricsset.h" #include "storagenodecontext.h" +#include "tls_statistics_metrics_wrapper.h" #include #include @@ -159,7 +160,7 @@ StorageNode::initialize() _context.getComponentRegister().setPriorityConfig(*_priorityConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - _metrics.reset(new StorageMetricSet); + _metrics = std::make_shared(); _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode")); _component->registerMetric(*_metrics); if (!_context.getComponentRegister().hasMetricManager()) { diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp new file mode 100644 index 00000000000..707e1c84036 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tls_statistics_metrics_wrapper.h" + +namespace storage { + +TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner) + : metrics::MetricSet("network", {}, "Network connection metrics", owner), + insecure_client_connections_established("insecure_client_connections_established", {}, + "Number of insecure (plaintext) client connections established", this), + insecure_server_connections_accepted("insecure_server_connections_accepted", {}, + "Number of insecure (plaintext) server connections accepted", this), + tls_client_connections_established("tls_client_connections_established", {}, + "Number of secure mTLS client connections established", this), + tls_server_connections_accepted("tls_server_connections_accepted", {}, + "Number of secure mTLS server connections accepted", this), + tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or " + "server connection attempts that failed during TLS handshaking", this), + peer_authorization_failures("peer_authorization_failures", {}, + "Number of TLS connection attempts failed due to bad or missing " + "peer certificate credentials", this), + tls_connections_broken("tls_connections_broken", {}, "Number of TLS " + "connections broken due to failures during frame encoding or decoding", this), + failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times " + "background reloading of TLS config has failed", this), + last_client_stats_snapshot(), + last_server_stats_snapshot(), + last_config_stats_snapshot() +{} + +TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default; + +void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() { + auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot(); + auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot(); + auto server_delta = server_current.subtract(last_server_stats_snapshot); + auto client_delta = client_current.subtract(last_client_stats_snapshot); + + insecure_client_connections_established.set(client_delta.insecure_connections); + insecure_server_connections_accepted.set(server_delta.insecure_connections); + tls_client_connections_established.set(client_delta.tls_connections); + tls_server_connections_accepted.set(server_delta.tls_connections); + // We have underlying stats for both server and client here, but for the + // moment we just aggregate them up into combined metrics. Can be trivially + // split up into separate metrics later if deemed useful. + tls_handshakes_failed.set(client_delta.failed_tls_handshakes + + server_delta.failed_tls_handshakes); + peer_authorization_failures.set(client_delta.invalid_peer_credentials + + server_delta.invalid_peer_credentials); + tls_connections_broken.set(client_delta.broken_tls_connections + + server_delta.broken_tls_connections); + + auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot(); + auto config_delta = config_current.subtract(last_config_stats_snapshot); + + failed_tls_config_reloads.set(config_delta.failed_config_reloads); + + last_server_stats_snapshot = server_current; + last_client_stats_snapshot = client_current; + last_config_stats_snapshot = config_current; +} + +} diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h new file mode 100644 index 00000000000..9784aaddd1c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include +#include + +#include + +namespace storage { + +// Simple wrapper around low-level vespalib network statistics which +// converts the monotonically increasing counters to deltas during +// periodic metric snapshotting. +class TlsStatisticsMetricsWrapper : public metrics::MetricSet { + metrics::LongCountMetric insecure_client_connections_established; + metrics::LongCountMetric insecure_server_connections_accepted; + metrics::LongCountMetric tls_client_connections_established; + metrics::LongCountMetric tls_server_connections_accepted; + metrics::LongCountMetric tls_handshakes_failed; + metrics::LongCountMetric peer_authorization_failures; + metrics::LongCountMetric tls_connections_broken; + + metrics::LongCountMetric failed_tls_config_reloads; + + vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot; + vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot; + vespalib::net::tls::ConfigStatistics::Snapshot last_config_stats_snapshot; + +public: + explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner); + ~TlsStatisticsMetricsWrapper() override; + + void update_metrics_with_snapshot_delta(); +}; + +} -- cgit v1.2.3