Add TLS statistics to vespalib and expose as metrics via storageserver

Also add functionality for extracting "notAfter" expiration time from
current certificate, which may later be added as an expiry metric.

18 files changed, 389 insertions, 40 deletions

diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 2df3d3a9606..73873e78032 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver
     storagemetricsset.cpp
     storagenode.cpp
     storagenodecontext.cpp
+    tls_statistics_metrics_wrapper.cpp
     INSTALL lib64
     DEPENDS
     storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
index 4ea9a9f9296..f0e64f0dfd1 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
@@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner)
       normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this),
       highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this),
       veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this)
-{ }
-MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {}
+{}
+
+MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default;
 
 DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner)
     : metrics::MetricSet("document_serialization", {{"docserialization"}},
@@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe
             "Number of times we reserialized a document because the "
             "compression it had in cache did not match what was configured",
             this)
-{ }
-DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { }
+{}
+
+DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default;
 
 StorageMetricSet::StorageMetricSet()
     : metrics::MetricSet("server", {{"memory"}},
@@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet()
       memoryUse_messages(this),
       memoryUse_visiting("memoryusage_visiting", {{"memory"}},
             "Message use from visiting", this),
-      documentSerialization(this)
-{ }
-StorageMetricSet::~StorageMetricSet() { }
+      documentSerialization(this),
+      tls_metrics(this)
+{}
+
+StorageMetricSet::~StorageMetricSet() = default;
 
 void StorageMetricSet::updateMetrics() {
     document::SerializableArray::Statistics stats(
@@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() {
             stats._serializedUncompressed);
     documentSerialization.inputWronglySerialized.set(
             stats._inputWronglySerialized);
+
+    // Delta snapshotting is destructive, so if an explicit snapshot is triggered
+    // (instead of just regular periodic snapshots), some events will effectively
+    // be erased from history. This will no longer be a problem once we move to a
+    // metrics system built around absolute (rather than derived) values.
+    tls_metrics.update_metrics_with_snapshot_delta();
 }
 
 } // storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h
index 40b70821bcd..e9378010540 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.h
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include "tls_statistics_metrics_wrapper.h"
+
 #include <vespa/metrics/metrics.h>
 
 namespace storage {
@@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet
     metrics::LongValueMetric memoryUse_visiting;
     DocumentSerializationMetricSet documentSerialization;
 
+    TlsStatisticsMetricsWrapper tls_metrics;
+
     StorageMetricSet();
     ~StorageMetricSet();
     void updateMetrics();
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index d159b6e5bdd..64c0970cf9d 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -7,6 +7,7 @@
 #include "statereporter.h"
 #include "storagemetricsset.h"
 #include "storagenodecontext.h"
+#include "tls_statistics_metrics_wrapper.h"
 
 #include <vespa/storage/frameworkimpl/status/statuswebserver.h>
 #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h>
@@ -159,7 +160,7 @@ StorageNode::initialize()
     _context.getComponentRegister().setPriorityConfig(*_priorityConfig);
     _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig);
 
-    _metrics.reset(new StorageMetricSet);
+    _metrics = std::make_shared<StorageMetricSet>();
     _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode"));
     _component->registerMetric(*_metrics);
     if (!_context.getComponentRegister().hasMetricManager()) {
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
new file mode 100644
index 00000000000..707e1c84036
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "tls_statistics_metrics_wrapper.h"
+
+namespace storage {
+
+TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner)
+    : metrics::MetricSet("network", {}, "Network connection metrics", owner),
+      insecure_client_connections_established("insecure_client_connections_established", {},
+              "Number of insecure (plaintext) client connections established", this),
+      insecure_server_connections_accepted("insecure_server_connections_accepted", {},
+              "Number of insecure (plaintext) server connections accepted", this),
+      tls_client_connections_established("tls_client_connections_established", {},
+              "Number of secure mTLS client connections established", this),
+      tls_server_connections_accepted("tls_server_connections_accepted", {},
+              "Number of secure mTLS server connections accepted", this),
+      tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or "
+              "server connection attempts that failed during TLS handshaking", this),
+      peer_authorization_failures("peer_authorization_failures", {},
+              "Number of TLS connection attempts failed due to bad or missing "
+              "peer certificate credentials", this),
+      tls_connections_broken("tls_connections_broken", {}, "Number of TLS "
+              "connections broken due to failures during frame encoding or decoding", this),
+      failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times "
+              "background reloading of TLS config has failed", this),
+      last_client_stats_snapshot(),
+      last_server_stats_snapshot(),
+      last_config_stats_snapshot()
+{}
+
+TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default;
+
+void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() {
+    auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot();
+    auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot();
+    auto server_delta = server_current.subtract(last_server_stats_snapshot);
+    auto client_delta = client_current.subtract(last_client_stats_snapshot);
+
+    insecure_client_connections_established.set(client_delta.insecure_connections);
+    insecure_server_connections_accepted.set(server_delta.insecure_connections);
+    tls_client_connections_established.set(client_delta.tls_connections);
+    tls_server_connections_accepted.set(server_delta.tls_connections);
+    // We have underlying stats for both server and client here, but for the
+    // moment we just aggregate them up into combined metrics. Can be trivially
+    // split up into separate metrics later if deemed useful.
+    tls_handshakes_failed.set(client_delta.failed_tls_handshakes +
+                              server_delta.failed_tls_handshakes);
+    peer_authorization_failures.set(client_delta.invalid_peer_credentials +
+                                    server_delta.invalid_peer_credentials);
+    tls_connections_broken.set(client_delta.broken_tls_connections +
+                               server_delta.broken_tls_connections);
+
+    auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot();
+    auto config_delta = config_current.subtract(last_config_stats_snapshot);
+
+    failed_tls_config_reloads.set(config_delta.failed_config_reloads);
+
+    last_server_stats_snapshot = server_current;
+    last_client_stats_snapshot = client_current;
+    last_config_stats_snapshot = config_current;
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
new file mode 100644
index 00000000000..9784aaddd1c
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
@@ -0,0 +1,36 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/net/tls/statistics.h>
+
+#include <chrono>
+
+namespace storage {
+
+// Simple wrapper around low-level vespalib network statistics which
+// converts the monotonically increasing counters to deltas during
+// periodic metric snapshotting.
+class TlsStatisticsMetricsWrapper : public metrics::MetricSet {
+    metrics::LongCountMetric insecure_client_connections_established;
+    metrics::LongCountMetric insecure_server_connections_accepted;
+    metrics::LongCountMetric tls_client_connections_established;
+    metrics::LongCountMetric tls_server_connections_accepted;
+    metrics::LongCountMetric tls_handshakes_failed;
+    metrics::LongCountMetric peer_authorization_failures;
+    metrics::LongCountMetric tls_connections_broken;
+
+    metrics::LongCountMetric failed_tls_config_reloads;
+
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot;
+    vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot;
+    vespalib::net::tls::ConfigStatistics::Snapshot     last_config_stats_snapshot;
+
+public:
+    explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner);
+    ~TlsStatisticsMetricsWrapper() override;
+
+    void update_metrics_with_snapshot_delta();
+};
+
+}
diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
index 245368b6a7b..5dc85bc567f 100644
--- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
+++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/transport_security_options_reading.h>
 #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h>
@@ -106,17 +107,11 @@ struct Fixture {
     }
 
     vespalib::string current_cert_chain() const {
-        auto impl = engine->acquire_current_engine();
-        // Leaks implementation details galore, but it's not very likely that we'll use
-        // anything but OpenSSL (or compatible APIs) in practice...
-        auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context());
-        return ctx_impl.transport_security_options().cert_chain_pem();
+        return engine->acquire_current_engine()->tls_context()->transport_security_options().cert_chain_pem();
     }
 
     AuthorizationMode current_authorization_mode() const {
-        auto impl = engine->acquire_current_engine();
-        auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context());
-        return ctx_impl.authorization_mode();
+        return engine->acquire_current_engine()->tls_context()->authorization_mode();
     }
 };
 
@@ -143,4 +138,15 @@ TEST_FF("Authorization mode is propagated to engine", Fixture(50ms, Authorizatio
     EXPECT_EQUAL(AuthorizationMode::LogOnly, f1.current_authorization_mode());
 }
 
+TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), TimeBomb(60)) {
+    auto before = ConfigStatistics::get().snapshot();
+
+    write_file("test_cert.pem.tmp", "Broken file oh no :(");
+    rename("test_cert.pem.tmp", "test_cert.pem", false, false);
+
+    while (ConfigStatistics::get().snapshot().subtract(before).failed_config_reloads == 0) {
+        std::this_thread::sleep_for(10ms);
+    }
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
index 69e0d44147e..f70c5670bc9 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
+++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/data/smart_buffer.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
 #include <vespa/vespalib/net/tls/crypto_codec.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/tls_context.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h>
@@ -619,6 +620,37 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF
     EXPECT_TRUE(f.handshake());
 }
 
+TEST_F("Failure statistics are incremented on authorization failures", CertFixture) {
+    reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce);
+    auto server_before = ConnectionStatistics::get(true).snapshot();
+    auto client_before = ConnectionStatistics::get(false).snapshot();
+    EXPECT_FALSE(f.handshake());
+    auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before);
+    auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before);
+
+    EXPECT_EQUAL(1u, server_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(1u, server_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(0u, server_stats.tls_connections);
+    EXPECT_EQUAL(0u, client_stats.tls_connections);
+}
+
+TEST_F("Success statistics are incremented on OK authorization", CertFixture) {
+    reset_peers_with_server_authz_mode(f, AuthorizationMode::Disable);
+    auto server_before = ConnectionStatistics::get(true).snapshot();
+    auto client_before = ConnectionStatistics::get(false).snapshot();
+    EXPECT_TRUE(f.handshake());
+    auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before);
+    auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before);
+
+    EXPECT_EQUAL(0u, server_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, server_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(0u, client_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(1u, server_stats.tls_connections);
+    EXPECT_EQUAL(1u, client_stats.tls_connections);
+}
+
 // TODO we can't test embedded nulls since the OpenSSL v3 extension APIs
 // take in null terminated strings as arguments... :I
 
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
index b771e35dab1..e291f39a834 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -8,6 +8,7 @@
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
 #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/transport_security_options_reading.h>
 #include <vespa/vespalib/net/tls/tls_crypto_engine.h>
@@ -243,8 +244,9 @@ CryptoEngine::get_default()
 }
 
 CryptoSocket::UP
-NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool)
+NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
 {
+    net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections();
     return std::make_unique<NullCryptoSocket>(std::move(socket));
 }
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
index 2f8c490b911..6dc48db68e4 100644
--- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
@@ -12,6 +12,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT
     peer_policies.cpp
     policy_checking_certificate_verifier.cpp
     protocol_snooping.cpp
+    statistics.cpp
     tls_context.cpp
     tls_crypto_engine.cpp
     tls_crypto_socket.cpp
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
index 9e38efabb2b..5f20280e0e2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "auto_reloading_tls_crypto_engine.h"
+#include "statistics.h"
 #include "tls_context.h"
 #include "tls_crypto_engine.h"
 #include "transport_security_options.h"
@@ -29,6 +30,7 @@ std::shared_ptr<TlsCryptoEngine> try_create_engine_from_tls_config(const vespali
     } catch (std::exception& e) {
         LOG(warning, "Failed to reload TLS config file (%s): '%s'. Old config remains in effect.",
             config_file_path.c_str(), e.what());
+        ConfigStatistics::get().inc_failed_config_reloads();
         return {};
     }
 }
@@ -78,6 +80,7 @@ void AutoReloadingTlsCryptoEngine::run_reload_loop() {
 void AutoReloadingTlsCryptoEngine::try_replace_current_engine() {
     auto new_engine = try_create_engine_from_tls_config(_config_file_path, _authorization_mode);
     if (new_engine) {
+        ConfigStatistics::get().inc_successful_config_reloads();
         std::lock_guard guard(_engine_mutex);
         _current_engine = std::move(new_engine);
     }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
index fee2b19f911..e8b1e32213d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
@@ -2,12 +2,16 @@
 #include "openssl_crypto_codec_impl.h"
 #include "openssl_tls_context_impl.h"
 #include "direct_buffer_bio.h"
+
 #include <vespa/vespalib/net/tls/crypto_codec.h>
 #include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/net/tls/statistics.h>
+
 #include <mutex>
 #include <vector>
 #include <memory>
 #include <stdexcept>
+
 #include <openssl/ssl.h>
 #include <openssl/crypto.h>
 #include <openssl/err.h>
@@ -250,9 +254,11 @@ HandshakeResult OpenSslCryptoCodecImpl::do_handshake_and_consume_peer_input_byte
             return handshake_failed();
         }
         LOG(debug, "SSL_do_handshake() is complete, using protocol %s", SSL_get_version(_ssl.get()));
+        ConnectionStatistics::get(_mode == Mode::Server).inc_tls_connections();
         return handshake_consumed_bytes_and_is_complete(static_cast<size_t>(consumed));
     } else {
         log_ssl_error("SSL_do_handshake()", ssl_result);
+        ConnectionStatistics::get(_mode == Mode::Server).inc_failed_tls_handshakes();
         return handshake_failed();
     }
 }
@@ -277,6 +283,7 @@ EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaint
         const int consumed = ::SSL_write(_ssl.get(), plaintext, to_consume);
         if (consumed < 0) {
             log_ssl_error("SSL_write()", ::SSL_get_error(_ssl.get(), consumed));
+            ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections();
             return encode_failed(); // TODO explicitly detect and log TLS renegotiation error (SSL_ERROR_WANT_READ)?
         } else if (consumed != to_consume) {
             LOG(error, "SSL_write() returned OK but did not consume all requested plaintext");
@@ -340,6 +347,7 @@ DecodeResult OpenSslCryptoCodecImpl::remap_ssl_read_failure_to_decode_result(int
         return decode_peer_has_closed();
     default:
         log_ssl_error("SSL_read()", ssl_error);
+        ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections();
         return decode_failed();
     }
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
index 53ed4d7bdff..b6e095f491d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
@@ -2,6 +2,7 @@
 #include "openssl_typedefs.h"
 #include "openssl_tls_context_impl.h"
 #include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <mutex>
@@ -243,6 +244,7 @@ void OpenSslTlsContextImpl::add_certificate_chain(vespalib::stringref chain_pem)
     if (!own_cert) {
         throw CryptoException("No X509 certificates could be found in provided chain");
     }
+    _expiry_time_point = extract_expiration_time(*own_cert);
     // Ownership of certificate is _not_ transferred, OpenSSL makes internal copy.
     // This is not well documented, but is mentioned by other impls.
     if (::SSL_CTX_use_certificate(_ctx.get(), own_cert.get()) != 1) {
@@ -314,6 +316,19 @@ void OpenSslTlsContextImpl::disable_renegotiation() {
 #endif
 }
 
+std::chrono::system_clock::time_point
+OpenSslTlsContextImpl::extract_expiration_time(::X509& cert) const {
+    ::ASN1_TIME* expiry = X509_get_notAfter(&cert); // Not owned by us.
+    int ndays = 0;
+    int nsecs = 0;
+    if (::ASN1_TIME_diff(&ndays, &nsecs, nullptr, expiry) != 1) {
+        throw CryptoException("ASN1_TIME_diff");
+    }
+    return (std::chrono::system_clock::now()
+            + std::chrono::hours(24 * ndays)
+            + std::chrono::seconds(nsecs));
+}
+
 namespace {
 
 // There's no good reason for entries to contain embedded nulls, aside from
@@ -407,52 +422,55 @@ int OpenSslTlsContextImpl::verify_cb_wrapper(int preverified_ok, ::X509_STORE_CT
     // since we trust the intermediates to have done their job.
     const bool is_peer_cert = (::X509_STORE_CTX_get_error_depth(store_ctx) == 0);
     if (!is_peer_cert) {
-        return 1; // OK for root/intermediate cert.
+        return 1; // OK for root/intermediate cert. Callback will be invoked again for other certs.
     }
     // Fetch the SSL instance associated with the X509_STORE_CTX
     const void* data = ::X509_STORE_CTX_get_ex_data(store_ctx, ::SSL_get_ex_data_X509_STORE_CTX_idx());
-    if (!data) {
-        return 0;
-    }
+    LOG_ASSERT(data != nullptr);
     const auto* ssl = static_cast<const ::SSL*>(data);
     const ::SSL_CTX* ssl_ctx = ::SSL_get_SSL_CTX(ssl);
-    if (!ssl_ctx) {
-        return 0;
-    }
+    LOG_ASSERT(ssl_ctx != nullptr);
     auto* self = static_cast<OpenSslTlsContextImpl*>(SSL_CTX_get_app_data(ssl_ctx));
-    if (!self) {
-        return 0;
+    LOG_ASSERT(self != nullptr);
+
+    if (self->verify_trusted_certificate(store_ctx)) {
+        return 1;
     }
-    const auto authz_mode = self->authorization_mode();
+    ConnectionStatistics::get(SSL_in_accept_init(ssl) != 0).inc_invalid_peer_credentials();
+    return 0;
+}
+
+bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_ctx) {
+    const auto authz_mode = authorization_mode();
     // TODO consider if we want to fill in peer credentials even if authorization is disabled
     if (authz_mode == AuthorizationMode::Disable) {
-        return 1;
+        return true;
     }
     ::X509* cert = ::X509_STORE_CTX_get_current_cert(store_ctx); // _not_ owned by us
     if (!cert) {
         LOG(error, "Got X509_STORE_CTX with preverified_ok == 1 but no current cert");
-        return 0;
+        return false;
     }
     PeerCredentials creds;
     if (!fill_certificate_common_name(cert, creds)) {
-        return 0;
+        return false;
     }
     if (!fill_certificate_subject_alternate_names(cert, creds)) {
-        return 0;
+        return false;
     }
     try {
-        const bool verified_by_cb = self->_cert_verify_callback->verify(creds);
+        const bool verified_by_cb = _cert_verify_callback->verify(creds);
         if (!verified_by_cb) {
             // TODO we should print the peer's remote address too, but that information is
             // not currently available to us here.
             LOG(warning, "Certificate verification failed for %s", to_string(creds).c_str());
-            return (authz_mode == AuthorizationMode::Enforce) ? 0 : 1;
+            return (authz_mode != AuthorizationMode::Enforce);
         }
     } catch (std::exception& e) {
         LOG(error, "Got exception during certificate verification callback: %s", e.what());
-        return 0;
+        return false;
     } // we don't expect any non-std::exception derived exceptions, so let them terminate the process.
-    return 1;
+    return true;
 }
 
 void OpenSslTlsContextImpl::enforce_peer_certificate_verification() {
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
index 757204325b0..92d8d895272 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
@@ -7,6 +7,8 @@
 #include <vespa/vespalib/net/tls/certificate_verification_callback.h>
 #include <vespa/vespalib/stllike/string.h>
 
+#include <chrono>
+
 namespace vespalib::net::tls::impl {
 
 class OpenSslTlsContextImpl : public TlsContext {
@@ -14,6 +16,7 @@ class OpenSslTlsContextImpl : public TlsContext {
     AuthorizationMode _authorization_mode;
     std::shared_ptr<CertificateVerificationCallback> _cert_verify_callback;
     TransportSecurityOptions _redacted_transport_options;
+    std::chrono::system_clock::time_point _expiry_time_point;
 public:
     OpenSslTlsContextImpl(const TransportSecurityOptions& ts_opts,
                           std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
@@ -21,13 +24,13 @@ public:
     ~OpenSslTlsContextImpl() override;
 
     ::SSL_CTX* native_context() const noexcept { return _ctx.get(); }
-    // Transport options this context was created with, but with the private key
-    // information scrubbed away.
-    const TransportSecurityOptions& transport_security_options() const noexcept {
+    const TransportSecurityOptions& transport_security_options() const noexcept override {
         return _redacted_transport_options;
     }
-    // AuthorizationMode this context was created with
-    AuthorizationMode authorization_mode() const noexcept { return _authorization_mode; }
+    AuthorizationMode authorization_mode() const noexcept override { return _authorization_mode; }
+    std::chrono::system_clock::time_point cert_expiration_time() const noexcept override {
+        return _expiry_time_point;
+    }
 private:
     // Note: single use per instance; does _not_ clear existing chain!
     void add_certificate_authorities(stringref ca_pem);
@@ -45,6 +48,10 @@ private:
     void enforce_peer_certificate_verification();
     void set_ssl_ctx_self_reference();
 
+    std::chrono::system_clock::time_point extract_expiration_time(::X509& cert) const;
+
+    bool verify_trusted_certificate(::X509_STORE_CTX* store_ctx);
+
     static int verify_cb_wrapper(int preverified_ok, ::X509_STORE_CTX* store_ctx);
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
index 4b9bc8a30b4..b6969250ae5 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "maybe_tls_crypto_socket.h"
+#include "statistics.h"
 #include "tls_crypto_socket.h"
 #include "protocol_snooping.h"
 #include <vespa/vespalib/data/smart_buffer.h>
@@ -54,6 +55,7 @@ public:
                 self = std::move(tls_socket);
                 return self->handshake();
             } else {
+                net::tls::ConnectionStatistics::get(true).inc_insecure_connections();
                 _factory.reset();
             }
         }
diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.cpp b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp
new file mode 100644
index 00000000000..d11aa60266e
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "statistics.h"
+
+namespace vespalib::net::tls {
+
+ConnectionStatistics ConnectionStatistics::client_stats = {};
+ConnectionStatistics ConnectionStatistics::server_stats = {};
+
+ConfigStatistics ConfigStatistics::instance = {};
+
+ConnectionStatistics::Snapshot ConnectionStatistics::snapshot() const noexcept {
+    Snapshot s;
+    s.insecure_connections       = insecure_connections.load(std::memory_order_relaxed);
+    s.tls_connections            = tls_connections.load(std::memory_order_relaxed);
+    s.failed_tls_handshakes      = failed_tls_handshakes.load(std::memory_order_relaxed);
+    s.invalid_peer_credentials   = invalid_peer_credentials.load(std::memory_order_relaxed);
+    s.broken_tls_connections     = broken_tls_connections.load(std::memory_order_relaxed);
+    return s;
+}
+
+ConnectionStatistics::Snapshot ConnectionStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept {
+    Snapshot s;
+    s.insecure_connections     = insecure_connections     - rhs.insecure_connections;
+    s.tls_connections          = tls_connections          - rhs.tls_connections;
+    s.failed_tls_handshakes    = failed_tls_handshakes    - rhs.failed_tls_handshakes;
+    s.invalid_peer_credentials = invalid_peer_credentials - rhs.invalid_peer_credentials;
+    s.broken_tls_connections   = broken_tls_connections   - rhs.broken_tls_connections;
+    return s;
+}
+
+ConfigStatistics::Snapshot ConfigStatistics::snapshot() const noexcept {
+    Snapshot s;
+    s.successful_config_reloads = successful_config_reloads.load(std::memory_order_relaxed);
+    s.failed_config_reloads     = failed_config_reloads.load(std::memory_order_relaxed);
+    return s;
+}
+
+ConfigStatistics::Snapshot ConfigStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept {
+    Snapshot s;
+    s.successful_config_reloads = successful_config_reloads - rhs.successful_config_reloads;
+    s.failed_config_reloads     = failed_config_reloads     - rhs.failed_config_reloads;
+    return s;
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.h b/vespalib/src/vespa/vespalib/net/tls/statistics.h
new file mode 100644
index 00000000000..1fce4f48b0c
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/statistics.h
@@ -0,0 +1,99 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <atomic>
+#include <stdint.h>
+
+namespace vespalib::net::tls {
+
+/**
+ * Low-level statistics set by connection and credential management code
+ * for TLS and insecure plaintext connections.
+ *
+ * A poor man's substitute for not currently having the ability to natively
+ * export metrics in vespalib. Should be removed in favor of proper metrics
+ * once this is possible.
+ *
+ * Fully thread safe.
+ */
+struct ConnectionStatistics {
+
+    // Number of insecure (legacy) plaintext connections established
+    std::atomic<uint64_t> insecure_connections     = 0;
+    // Number of TLS connections successfully established. Note that
+    // the handshake has to succeed for a connection to be counted here.
+    std::atomic<uint64_t> tls_connections          = 0;
+    // Number of connections that failed during the TLS handshake process.
+    // May be caused by bad certificates, invalid credentials, bad ciphers etc.
+    std::atomic<uint64_t> failed_tls_handshakes    = 0;
+    // Number of connections rejected because the certificate did not have
+    // credentials that matched the requirements given in the TLS config file.
+    std::atomic<uint64_t> invalid_peer_credentials = 0;
+    // Number of connections broken due to errors during TLS encoding or decoding
+    std::atomic<uint64_t> broken_tls_connections   = 0;
+
+    void inc_insecure_connections() noexcept {
+        insecure_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_tls_connections() noexcept {
+        tls_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_failed_tls_handshakes() noexcept {
+        failed_tls_handshakes.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_invalid_peer_credentials() noexcept {
+        invalid_peer_credentials.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_broken_tls_connections() noexcept {
+        broken_tls_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    struct Snapshot {
+        uint64_t insecure_connections     = 0;
+        uint64_t tls_connections          = 0;
+        uint64_t failed_tls_handshakes    = 0;
+        uint64_t invalid_peer_credentials = 0;
+        uint64_t broken_tls_connections   = 0;
+
+        Snapshot subtract(const Snapshot& rhs) const noexcept;
+    };
+
+    // Acquires a snapshot of statistics that is expected to be reasonably up to date.
+    // Thread safe.
+    Snapshot snapshot() const noexcept;
+
+    static ConnectionStatistics client_stats;
+    static ConnectionStatistics server_stats;
+
+    static ConnectionStatistics& get(bool is_server) noexcept {
+        return (is_server ? server_stats : client_stats);
+    }
+};
+
+struct ConfigStatistics {
+    std::atomic<uint64_t> successful_config_reloads = 0;
+    std::atomic<uint64_t> failed_config_reloads     = 0;
+
+    void inc_successful_config_reloads() noexcept {
+        successful_config_reloads.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_failed_config_reloads() noexcept {
+        failed_config_reloads.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    struct Snapshot {
+        uint64_t successful_config_reloads = 0;
+        uint64_t failed_config_reloads     = 0;
+
+        Snapshot subtract(const Snapshot& rhs) const noexcept;
+    };
+
+    // Acquires a snapshot of statistics that is expected to be reasonably up to date.
+    // Thread safe.
+    Snapshot snapshot() const noexcept;
+
+    static ConfigStatistics instance;
+    static ConfigStatistics& get() noexcept { return instance; }
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_context.h b/vespalib/src/vespa/vespalib/net/tls/tls_context.h
index 04538694ee7..26637a955c2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_context.h
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_context.h
@@ -2,6 +2,8 @@
 #pragma once
 
 #include "authorization_mode.h"
+
+#include <chrono>
 #include <memory>
 
 namespace vespalib::net::tls {
@@ -12,6 +14,14 @@ struct CertificateVerificationCallback;
 struct TlsContext {
     virtual ~TlsContext() = default;
 
+    // Transport options this context was created with, but with the private key
+    // information scrubbed away.
+    virtual const TransportSecurityOptions& transport_security_options() const noexcept = 0;
+    // AuthorizationMode this context was created with
+    virtual AuthorizationMode authorization_mode() const noexcept = 0;
+    // Expiration time point of the peer certificate context was created with
+    virtual std::chrono::system_clock::time_point cert_expiration_time() const noexcept = 0;
+
     // Create a TLS context which verifies certificates according to the provided options'
     // CA trust roots AND authorized peer policies
     static std::shared_ptr<TlsContext> create_default_context(const TransportSecurityOptions&,