Add TLS statistics to vespalib and expose as metrics via storageserver

Also add functionality for extracting "notAfter" expiration time from
current certificate, which may later be added as an expiry metric.

6 files changed, 123 insertions, 8 deletions

diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 2df3d3a9606..73873e78032 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver
     storagemetricsset.cpp
     storagenode.cpp
     storagenodecontext.cpp
+    tls_statistics_metrics_wrapper.cpp
     INSTALL lib64
     DEPENDS
     storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
index 4ea9a9f9296..f0e64f0dfd1 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
@@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner)
       normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this),
       highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this),
       veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this)
-{ }
-MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {}
+{}
+
+MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default;
 
 DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner)
     : metrics::MetricSet("document_serialization", {{"docserialization"}},
@@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe
             "Number of times we reserialized a document because the "
             "compression it had in cache did not match what was configured",
             this)
-{ }
-DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { }
+{}
+
+DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default;
 
 StorageMetricSet::StorageMetricSet()
     : metrics::MetricSet("server", {{"memory"}},
@@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet()
       memoryUse_messages(this),
       memoryUse_visiting("memoryusage_visiting", {{"memory"}},
             "Message use from visiting", this),
-      documentSerialization(this)
-{ }
-StorageMetricSet::~StorageMetricSet() { }
+      documentSerialization(this),
+      tls_metrics(this)
+{}
+
+StorageMetricSet::~StorageMetricSet() = default;
 
 void StorageMetricSet::updateMetrics() {
     document::SerializableArray::Statistics stats(
@@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() {
             stats._serializedUncompressed);
     documentSerialization.inputWronglySerialized.set(
             stats._inputWronglySerialized);
+
+    // Delta snapshotting is destructive, so if an explicit snapshot is triggered
+    // (instead of just regular periodic snapshots), some events will effectively
+    // be erased from history. This will no longer be a problem once we move to a
+    // metrics system built around absolute (rather than derived) values.
+    tls_metrics.update_metrics_with_snapshot_delta();
 }
 
 } // storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h
index 40b70821bcd..e9378010540 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.h
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include "tls_statistics_metrics_wrapper.h"
+
 #include <vespa/metrics/metrics.h>
 
 namespace storage {
@@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet
     metrics::LongValueMetric memoryUse_visiting;
     DocumentSerializationMetricSet documentSerialization;
 
+    TlsStatisticsMetricsWrapper tls_metrics;
+
     StorageMetricSet();
     ~StorageMetricSet();
     void updateMetrics();
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index d159b6e5bdd..64c0970cf9d 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -7,6 +7,7 @@
 #include "statereporter.h"
 #include "storagemetricsset.h"
 #include "storagenodecontext.h"
+#include "tls_statistics_metrics_wrapper.h"
 
 #include <vespa/storage/frameworkimpl/status/statuswebserver.h>
 #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h>
@@ -159,7 +160,7 @@ StorageNode::initialize()
     _context.getComponentRegister().setPriorityConfig(*_priorityConfig);
     _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig);
 
-    _metrics.reset(new StorageMetricSet);
+    _metrics = std::make_shared<StorageMetricSet>();
     _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode"));
     _component->registerMetric(*_metrics);
     if (!_context.getComponentRegister().hasMetricManager()) {
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
new file mode 100644
index 00000000000..707e1c84036
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "tls_statistics_metrics_wrapper.h"
+
+namespace storage {
+
+TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner)
+    : metrics::MetricSet("network", {}, "Network connection metrics", owner),
+      insecure_client_connections_established("insecure_client_connections_established", {},
+              "Number of insecure (plaintext) client connections established", this),
+      insecure_server_connections_accepted("insecure_server_connections_accepted", {},
+              "Number of insecure (plaintext) server connections accepted", this),
+      tls_client_connections_established("tls_client_connections_established", {},
+              "Number of secure mTLS client connections established", this),
+      tls_server_connections_accepted("tls_server_connections_accepted", {},
+              "Number of secure mTLS server connections accepted", this),
+      tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or "
+              "server connection attempts that failed during TLS handshaking", this),
+      peer_authorization_failures("peer_authorization_failures", {},
+              "Number of TLS connection attempts failed due to bad or missing "
+              "peer certificate credentials", this),
+      tls_connections_broken("tls_connections_broken", {}, "Number of TLS "
+              "connections broken due to failures during frame encoding or decoding", this),
+      failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times "
+              "background reloading of TLS config has failed", this),
+      last_client_stats_snapshot(),
+      last_server_stats_snapshot(),
+      last_config_stats_snapshot()
+{}
+
+TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default;
+
+void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() {
+    auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot();
+    auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot();
+    auto server_delta = server_current.subtract(last_server_stats_snapshot);
+    auto client_delta = client_current.subtract(last_client_stats_snapshot);
+
+    insecure_client_connections_established.set(client_delta.insecure_connections);
+    insecure_server_connections_accepted.set(server_delta.insecure_connections);
+    tls_client_connections_established.set(client_delta.tls_connections);
+    tls_server_connections_accepted.set(server_delta.tls_connections);
+    // We have underlying stats for both server and client here, but for the
+    // moment we just aggregate them up into combined metrics. Can be trivially
+    // split up into separate metrics later if deemed useful.
+    tls_handshakes_failed.set(client_delta.failed_tls_handshakes +
+                              server_delta.failed_tls_handshakes);
+    peer_authorization_failures.set(client_delta.invalid_peer_credentials +
+                                    server_delta.invalid_peer_credentials);
+    tls_connections_broken.set(client_delta.broken_tls_connections +
+                               server_delta.broken_tls_connections);
+
+    auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot();
+    auto config_delta = config_current.subtract(last_config_stats_snapshot);
+
+    failed_tls_config_reloads.set(config_delta.failed_config_reloads);
+
+    last_server_stats_snapshot = server_current;
+    last_client_stats_snapshot = client_current;
+    last_config_stats_snapshot = config_current;
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
new file mode 100644
index 00000000000..9784aaddd1c
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
@@ -0,0 +1,36 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/net/tls/statistics.h>
+
+#include <chrono>
+
+namespace storage {
+
+// Simple wrapper around low-level vespalib network statistics which
+// converts the monotonically increasing counters to deltas during
+// periodic metric snapshotting.
+class TlsStatisticsMetricsWrapper : public metrics::MetricSet {
+    metrics::LongCountMetric insecure_client_connections_established;
+    metrics::LongCountMetric insecure_server_connections_accepted;
+    metrics::LongCountMetric tls_client_connections_established;
+    metrics::LongCountMetric tls_server_connections_accepted;
+    metrics::LongCountMetric tls_handshakes_failed;
+    metrics::LongCountMetric peer_authorization_failures;
+    metrics::LongCountMetric tls_connections_broken;
+
+    metrics::LongCountMetric failed_tls_config_reloads;
+
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot;
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot;
+    vespalib::net::tls::ConfigStatistics::Snapshot     last_config_stats_snapshot;
+
+public:
+    explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner);
+    ~TlsStatisticsMetricsWrapper() override;
+
+    void update_metrics_with_snapshot_delta();
+};
+
+}