Add TLS statistics to vespalib and expose as metrics via storageserver

Now without unused expiry time extraction.

6 files changed, 123 insertions, 8 deletions

diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 2df3d3a9606..73873e78032 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver
     storagemetricsset.cpp
     storagenode.cpp
     storagenodecontext.cpp
+    tls_statistics_metrics_wrapper.cpp
     INSTALL lib64
     DEPENDS
     storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
index 4ea9a9f9296..f0e64f0dfd1 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
@@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner)
       normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this),
       highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this),
       veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this)
-{ }
-MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {}
+{}
+
+MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default;
 
 DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner)
     : metrics::MetricSet("document_serialization", {{"docserialization"}},
@@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe
             "Number of times we reserialized a document because the "
             "compression it had in cache did not match what was configured",
             this)
-{ }
-DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { }
+{}
+
+DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default;
 
 StorageMetricSet::StorageMetricSet()
     : metrics::MetricSet("server", {{"memory"}},
@@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet()
       memoryUse_messages(this),
       memoryUse_visiting("memoryusage_visiting", {{"memory"}},
             "Message use from visiting", this),
-      documentSerialization(this)
-{ }
-StorageMetricSet::~StorageMetricSet() { }
+      documentSerialization(this),
+      tls_metrics(this)
+{}
+
+StorageMetricSet::~StorageMetricSet() = default;
 
 void StorageMetricSet::updateMetrics() {
     document::SerializableArray::Statistics stats(
@@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() {
             stats._serializedUncompressed);
     documentSerialization.inputWronglySerialized.set(
             stats._inputWronglySerialized);
+
+    // Delta snapshotting is destructive, so if an explicit snapshot is triggered
+    // (instead of just regular periodic snapshots), some events will effectively
+    // be erased from history. This will no longer be a problem once we move to a
+    // metrics system built around absolute (rather than derived) values.
+    tls_metrics.update_metrics_with_snapshot_delta();
 }
 
 } // storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h
index 40b70821bcd..e9378010540 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.h
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include "tls_statistics_metrics_wrapper.h"
+
 #include <vespa/metrics/metrics.h>
 
 namespace storage {
@@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet
     metrics::LongValueMetric memoryUse_visiting;
     DocumentSerializationMetricSet documentSerialization;
 
+    TlsStatisticsMetricsWrapper tls_metrics;
+
     StorageMetricSet();
     ~StorageMetricSet();
     void updateMetrics();
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index d159b6e5bdd..64c0970cf9d 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -7,6 +7,7 @@
 #include "statereporter.h"
 #include "storagemetricsset.h"
 #include "storagenodecontext.h"
+#include "tls_statistics_metrics_wrapper.h"
 
 #include <vespa/storage/frameworkimpl/status/statuswebserver.h>
 #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h>
@@ -159,7 +160,7 @@ StorageNode::initialize()
     _context.getComponentRegister().setPriorityConfig(*_priorityConfig);
     _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig);
 
-    _metrics.reset(new StorageMetricSet);
+    _metrics = std::make_shared<StorageMetricSet>();
     _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode"));
     _component->registerMetric(*_metrics);
     if (!_context.getComponentRegister().hasMetricManager()) {
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
new file mode 100644
index 00000000000..707e1c84036
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "tls_statistics_metrics_wrapper.h"
+
+namespace storage {
+
+TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner)
+    : metrics::MetricSet("network", {}, "Network connection metrics", owner),
+      insecure_client_connections_established("insecure_client_connections_established", {},
+              "Number of insecure (plaintext) client connections established", this),
+      insecure_server_connections_accepted("insecure_server_connections_accepted", {},
+              "Number of insecure (plaintext) server connections accepted", this),
+      tls_client_connections_established("tls_client_connections_established", {},
+              "Number of secure mTLS client connections established", this),
+      tls_server_connections_accepted("tls_server_connections_accepted", {},
+              "Number of secure mTLS server connections accepted", this),
+      tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or "
+              "server connection attempts that failed during TLS handshaking", this),
+      peer_authorization_failures("peer_authorization_failures", {},
+              "Number of TLS connection attempts failed due to bad or missing "
+              "peer certificate credentials", this),
+      tls_connections_broken("tls_connections_broken", {}, "Number of TLS "
+              "connections broken due to failures during frame encoding or decoding", this),
+      failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times "
+              "background reloading of TLS config has failed", this),
+      last_client_stats_snapshot(),
+      last_server_stats_snapshot(),
+      last_config_stats_snapshot()
+{}
+
+TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default;
+
+void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() {
+    auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot();
+    auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot();
+    auto server_delta = server_current.subtract(last_server_stats_snapshot);
+    auto client_delta = client_current.subtract(last_client_stats_snapshot);
+
+    insecure_client_connections_established.set(client_delta.insecure_connections);
+    insecure_server_connections_accepted.set(server_delta.insecure_connections);
+    tls_client_connections_established.set(client_delta.tls_connections);
+    tls_server_connections_accepted.set(server_delta.tls_connections);
+    // We have underlying stats for both server and client here, but for the
+    // moment we just aggregate them up into combined metrics. Can be trivially
+    // split up into separate metrics later if deemed useful.
+    tls_handshakes_failed.set(client_delta.failed_tls_handshakes +
+                              server_delta.failed_tls_handshakes);
+    peer_authorization_failures.set(client_delta.invalid_peer_credentials +
+                                    server_delta.invalid_peer_credentials);
+    tls_connections_broken.set(client_delta.broken_tls_connections +
+                               server_delta.broken_tls_connections);
+
+    auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot();
+    auto config_delta = config_current.subtract(last_config_stats_snapshot);
+
+    failed_tls_config_reloads.set(config_delta.failed_config_reloads);
+
+    last_server_stats_snapshot = server_current;
+    last_client_stats_snapshot = client_current;
+    last_config_stats_snapshot = config_current;
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
new file mode 100644
index 00000000000..9784aaddd1c
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
@@ -0,0 +1,36 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/net/tls/statistics.h>
+
+#include <chrono>
+
+namespace storage {
+
+// Simple wrapper around low-level vespalib network statistics which
+// converts the monotonically increasing counters to deltas during
+// periodic metric snapshotting.
+class TlsStatisticsMetricsWrapper : public metrics::MetricSet {
+    metrics::LongCountMetric insecure_client_connections_established;
+    metrics::LongCountMetric insecure_server_connections_accepted;
+    metrics::LongCountMetric tls_client_connections_established;
+    metrics::LongCountMetric tls_server_connections_accepted;
+    metrics::LongCountMetric tls_handshakes_failed;
+    metrics::LongCountMetric peer_authorization_failures;
+    metrics::LongCountMetric tls_connections_broken;
+
+    metrics::LongCountMetric failed_tls_config_reloads;
+
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot;
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot;
+    vespalib::net::tls::ConfigStatistics::Snapshot     last_config_stats_snapshot;
+
+public:
+    explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner);
+    ~TlsStatisticsMetricsWrapper() override;
+
+    void update_metrics_with_snapshot_delta();
+};
+
+}