Merge pull request #7947 from vespa-engine/vekterli/add-low-level-connection-stats-and-metrics

Add TLS statistics to vespalib and expose as metrics via storageserver

12 files changed, 266 insertions, 32 deletions

diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
index 245368b6a7b..5dc85bc567f 100644
--- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
+++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/transport_security_options_reading.h>
 #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h>
@@ -106,17 +107,11 @@ struct Fixture {
     }
 
     vespalib::string current_cert_chain() const {
-        auto impl = engine->acquire_current_engine();
-        // Leaks implementation details galore, but it's not very likely that we'll use
-        // anything but OpenSSL (or compatible APIs) in practice...
-        auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context());
-        return ctx_impl.transport_security_options().cert_chain_pem();
+        return engine->acquire_current_engine()->tls_context()->transport_security_options().cert_chain_pem();
     }
 
     AuthorizationMode current_authorization_mode() const {
-        auto impl = engine->acquire_current_engine();
-        auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context());
-        return ctx_impl.authorization_mode();
+        return engine->acquire_current_engine()->tls_context()->authorization_mode();
     }
 };
 
@@ -143,4 +138,15 @@ TEST_FF("Authorization mode is propagated to engine", Fixture(50ms, Authorizatio
     EXPECT_EQUAL(AuthorizationMode::LogOnly, f1.current_authorization_mode());
 }
 
+TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), TimeBomb(60)) {
+    auto before = ConfigStatistics::get().snapshot();
+
+    write_file("test_cert.pem.tmp", "Broken file oh no :(");
+    rename("test_cert.pem.tmp", "test_cert.pem", false, false);
+
+    while (ConfigStatistics::get().snapshot().subtract(before).failed_config_reloads == 0) {
+        std::this_thread::sleep_for(10ms);
+    }
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
index 69e0d44147e..f70c5670bc9 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
+++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/data/smart_buffer.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
 #include <vespa/vespalib/net/tls/crypto_codec.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/tls_context.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h>
@@ -619,6 +620,37 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF
     EXPECT_TRUE(f.handshake());
 }
 
+TEST_F("Failure statistics are incremented on authorization failures", CertFixture) {
+    reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce);
+    auto server_before = ConnectionStatistics::get(true).snapshot();
+    auto client_before = ConnectionStatistics::get(false).snapshot();
+    EXPECT_FALSE(f.handshake());
+    auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before);
+    auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before);
+
+    EXPECT_EQUAL(1u, server_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(1u, server_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(0u, server_stats.tls_connections);
+    EXPECT_EQUAL(0u, client_stats.tls_connections);
+}
+
+TEST_F("Success statistics are incremented on OK authorization", CertFixture) {
+    reset_peers_with_server_authz_mode(f, AuthorizationMode::Disable);
+    auto server_before = ConnectionStatistics::get(true).snapshot();
+    auto client_before = ConnectionStatistics::get(false).snapshot();
+    EXPECT_TRUE(f.handshake());
+    auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before);
+    auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before);
+
+    EXPECT_EQUAL(0u, server_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials);
+    EXPECT_EQUAL(0u, server_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(0u, client_stats.failed_tls_handshakes);
+    EXPECT_EQUAL(1u, server_stats.tls_connections);
+    EXPECT_EQUAL(1u, client_stats.tls_connections);
+}
+
 // TODO we can't test embedded nulls since the OpenSSL v3 extension APIs
 // take in null terminated strings as arguments... :I
 
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
index b771e35dab1..e291f39a834 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -8,6 +8,7 @@
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
 #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/transport_security_options_reading.h>
 #include <vespa/vespalib/net/tls/tls_crypto_engine.h>
@@ -243,8 +244,9 @@ CryptoEngine::get_default()
 }
 
 CryptoSocket::UP
-NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool)
+NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
 {
+    net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections();
     return std::make_unique<NullCryptoSocket>(std::move(socket));
 }
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
index 2f8c490b911..6dc48db68e4 100644
--- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
@@ -12,6 +12,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT
     peer_policies.cpp
     policy_checking_certificate_verifier.cpp
     protocol_snooping.cpp
+    statistics.cpp
     tls_context.cpp
     tls_crypto_engine.cpp
     tls_crypto_socket.cpp
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
index 9e38efabb2b..5f20280e0e2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "auto_reloading_tls_crypto_engine.h"
+#include "statistics.h"
 #include "tls_context.h"
 #include "tls_crypto_engine.h"
 #include "transport_security_options.h"
@@ -29,6 +30,7 @@ std::shared_ptr<TlsCryptoEngine> try_create_engine_from_tls_config(const vespali
     } catch (std::exception& e) {
         LOG(warning, "Failed to reload TLS config file (%s): '%s'. Old config remains in effect.",
             config_file_path.c_str(), e.what());
+        ConfigStatistics::get().inc_failed_config_reloads();
         return {};
     }
 }
@@ -78,6 +80,7 @@ void AutoReloadingTlsCryptoEngine::run_reload_loop() {
 void AutoReloadingTlsCryptoEngine::try_replace_current_engine() {
     auto new_engine = try_create_engine_from_tls_config(_config_file_path, _authorization_mode);
     if (new_engine) {
+        ConfigStatistics::get().inc_successful_config_reloads();
         std::lock_guard guard(_engine_mutex);
         _current_engine = std::move(new_engine);
     }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
index fee2b19f911..e8b1e32213d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
@@ -2,12 +2,16 @@
 #include "openssl_crypto_codec_impl.h"
 #include "openssl_tls_context_impl.h"
 #include "direct_buffer_bio.h"
+
 #include <vespa/vespalib/net/tls/crypto_codec.h>
 #include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/net/tls/statistics.h>
+
 #include <mutex>
 #include <vector>
 #include <memory>
 #include <stdexcept>
+
 #include <openssl/ssl.h>
 #include <openssl/crypto.h>
 #include <openssl/err.h>
@@ -250,9 +254,11 @@ HandshakeResult OpenSslCryptoCodecImpl::do_handshake_and_consume_peer_input_byte
             return handshake_failed();
         }
         LOG(debug, "SSL_do_handshake() is complete, using protocol %s", SSL_get_version(_ssl.get()));
+        ConnectionStatistics::get(_mode == Mode::Server).inc_tls_connections();
         return handshake_consumed_bytes_and_is_complete(static_cast<size_t>(consumed));
     } else {
         log_ssl_error("SSL_do_handshake()", ssl_result);
+        ConnectionStatistics::get(_mode == Mode::Server).inc_failed_tls_handshakes();
         return handshake_failed();
     }
 }
@@ -277,6 +283,7 @@ EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaint
         const int consumed = ::SSL_write(_ssl.get(), plaintext, to_consume);
         if (consumed < 0) {
             log_ssl_error("SSL_write()", ::SSL_get_error(_ssl.get(), consumed));
+            ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections();
             return encode_failed(); // TODO explicitly detect and log TLS renegotiation error (SSL_ERROR_WANT_READ)?
         } else if (consumed != to_consume) {
             LOG(error, "SSL_write() returned OK but did not consume all requested plaintext");
@@ -340,6 +347,7 @@ DecodeResult OpenSslCryptoCodecImpl::remap_ssl_read_failure_to_decode_result(int
         return decode_peer_has_closed();
     default:
         log_ssl_error("SSL_read()", ssl_error);
+        ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections();
         return decode_failed();
     }
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
index 53ed4d7bdff..b6e095f491d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
@@ -2,6 +2,7 @@
 #include "openssl_typedefs.h"
 #include "openssl_tls_context_impl.h"
 #include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <mutex>
@@ -243,6 +244,7 @@ void OpenSslTlsContextImpl::add_certificate_chain(vespalib::stringref chain_pem)
     if (!own_cert) {
         throw CryptoException("No X509 certificates could be found in provided chain");
     }
+    _expiry_time_point = extract_expiration_time(*own_cert);
     // Ownership of certificate is _not_ transferred, OpenSSL makes internal copy.
     // This is not well documented, but is mentioned by other impls.
     if (::SSL_CTX_use_certificate(_ctx.get(), own_cert.get()) != 1) {
@@ -314,6 +316,19 @@ void OpenSslTlsContextImpl::disable_renegotiation() {
 #endif
 }
 
+std::chrono::system_clock::time_point
+OpenSslTlsContextImpl::extract_expiration_time(::X509& cert) const {
+    ::ASN1_TIME* expiry = X509_get_notAfter(&cert); // Not owned by us.
+    int ndays = 0;
+    int nsecs = 0;
+    if (::ASN1_TIME_diff(&ndays, &nsecs, nullptr, expiry) != 1) {
+        throw CryptoException("ASN1_TIME_diff");
+    }
+    return (std::chrono::system_clock::now()
+            + std::chrono::hours(24 * ndays)
+            + std::chrono::seconds(nsecs));
+}
+
 namespace {
 
 // There's no good reason for entries to contain embedded nulls, aside from
@@ -407,52 +422,55 @@ int OpenSslTlsContextImpl::verify_cb_wrapper(int preverified_ok, ::X509_STORE_CT
     // since we trust the intermediates to have done their job.
     const bool is_peer_cert = (::X509_STORE_CTX_get_error_depth(store_ctx) == 0);
     if (!is_peer_cert) {
-        return 1; // OK for root/intermediate cert.
+        return 1; // OK for root/intermediate cert. Callback will be invoked again for other certs.
     }
     // Fetch the SSL instance associated with the X509_STORE_CTX
     const void* data = ::X509_STORE_CTX_get_ex_data(store_ctx, ::SSL_get_ex_data_X509_STORE_CTX_idx());
-    if (!data) {
-        return 0;
-    }
+    LOG_ASSERT(data != nullptr);
     const auto* ssl = static_cast<const ::SSL*>(data);
     const ::SSL_CTX* ssl_ctx = ::SSL_get_SSL_CTX(ssl);
-    if (!ssl_ctx) {
-        return 0;
-    }
+    LOG_ASSERT(ssl_ctx != nullptr);
     auto* self = static_cast<OpenSslTlsContextImpl*>(SSL_CTX_get_app_data(ssl_ctx));
-    if (!self) {
-        return 0;
+    LOG_ASSERT(self != nullptr);
+
+    if (self->verify_trusted_certificate(store_ctx)) {
+        return 1;
     }
-    const auto authz_mode = self->authorization_mode();
+    ConnectionStatistics::get(SSL_in_accept_init(ssl) != 0).inc_invalid_peer_credentials();
+    return 0;
+}
+
+bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_ctx) {
+    const auto authz_mode = authorization_mode();
     // TODO consider if we want to fill in peer credentials even if authorization is disabled
     if (authz_mode == AuthorizationMode::Disable) {
-        return 1;
+        return true;
     }
     ::X509* cert = ::X509_STORE_CTX_get_current_cert(store_ctx); // _not_ owned by us
     if (!cert) {
         LOG(error, "Got X509_STORE_CTX with preverified_ok == 1 but no current cert");
-        return 0;
+        return false;
     }
     PeerCredentials creds;
     if (!fill_certificate_common_name(cert, creds)) {
-        return 0;
+        return false;
     }
     if (!fill_certificate_subject_alternate_names(cert, creds)) {
-        return 0;
+        return false;
     }
     try {
-        const bool verified_by_cb = self->_cert_verify_callback->verify(creds);
+        const bool verified_by_cb = _cert_verify_callback->verify(creds);
         if (!verified_by_cb) {
             // TODO we should print the peer's remote address too, but that information is
             // not currently available to us here.
             LOG(warning, "Certificate verification failed for %s", to_string(creds).c_str());
-            return (authz_mode == AuthorizationMode::Enforce) ? 0 : 1;
+            return (authz_mode != AuthorizationMode::Enforce);
         }
     } catch (std::exception& e) {
         LOG(error, "Got exception during certificate verification callback: %s", e.what());
-        return 0;
+        return false;
     } // we don't expect any non-std::exception derived exceptions, so let them terminate the process.
-    return 1;
+    return true;
 }
 
 void OpenSslTlsContextImpl::enforce_peer_certificate_verification() {
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
index 757204325b0..92d8d895272 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
@@ -7,6 +7,8 @@
 #include <vespa/vespalib/net/tls/certificate_verification_callback.h>
 #include <vespa/vespalib/stllike/string.h>
 
+#include <chrono>
+
 namespace vespalib::net::tls::impl {
 
 class OpenSslTlsContextImpl : public TlsContext {
@@ -14,6 +16,7 @@ class OpenSslTlsContextImpl : public TlsContext {
     AuthorizationMode _authorization_mode;
     std::shared_ptr<CertificateVerificationCallback> _cert_verify_callback;
     TransportSecurityOptions _redacted_transport_options;
+    std::chrono::system_clock::time_point _expiry_time_point;
 public:
     OpenSslTlsContextImpl(const TransportSecurityOptions& ts_opts,
                           std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
@@ -21,13 +24,13 @@ public:
     ~OpenSslTlsContextImpl() override;
 
     ::SSL_CTX* native_context() const noexcept { return _ctx.get(); }
-    // Transport options this context was created with, but with the private key
-    // information scrubbed away.
-    const TransportSecurityOptions& transport_security_options() const noexcept {
+    const TransportSecurityOptions& transport_security_options() const noexcept override {
         return _redacted_transport_options;
     }
-    // AuthorizationMode this context was created with
-    AuthorizationMode authorization_mode() const noexcept { return _authorization_mode; }
+    AuthorizationMode authorization_mode() const noexcept override { return _authorization_mode; }
+    std::chrono::system_clock::time_point cert_expiration_time() const noexcept override {
+        return _expiry_time_point;
+    }
 private:
     // Note: single use per instance; does _not_ clear existing chain!
     void add_certificate_authorities(stringref ca_pem);
@@ -45,6 +48,10 @@ private:
     void enforce_peer_certificate_verification();
     void set_ssl_ctx_self_reference();
 
+    std::chrono::system_clock::time_point extract_expiration_time(::X509& cert) const;
+
+    bool verify_trusted_certificate(::X509_STORE_CTX* store_ctx);
+
     static int verify_cb_wrapper(int preverified_ok, ::X509_STORE_CTX* store_ctx);
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
index 4b9bc8a30b4..b6969250ae5 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "maybe_tls_crypto_socket.h"
+#include "statistics.h"
 #include "tls_crypto_socket.h"
 #include "protocol_snooping.h"
 #include <vespa/vespalib/data/smart_buffer.h>
@@ -54,6 +55,7 @@ public:
                 self = std::move(tls_socket);
                 return self->handshake();
             } else {
+                net::tls::ConnectionStatistics::get(true).inc_insecure_connections();
                 _factory.reset();
             }
         }
diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.cpp b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp
new file mode 100644
index 00000000000..d11aa60266e
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "statistics.h"
+
+namespace vespalib::net::tls {
+
+ConnectionStatistics ConnectionStatistics::client_stats = {};
+ConnectionStatistics ConnectionStatistics::server_stats = {};
+
+ConfigStatistics ConfigStatistics::instance = {};
+
+ConnectionStatistics::Snapshot ConnectionStatistics::snapshot() const noexcept {
+    Snapshot s;
+    s.insecure_connections       = insecure_connections.load(std::memory_order_relaxed);
+    s.tls_connections            = tls_connections.load(std::memory_order_relaxed);
+    s.failed_tls_handshakes      = failed_tls_handshakes.load(std::memory_order_relaxed);
+    s.invalid_peer_credentials   = invalid_peer_credentials.load(std::memory_order_relaxed);
+    s.broken_tls_connections     = broken_tls_connections.load(std::memory_order_relaxed);
+    return s;
+}
+
+ConnectionStatistics::Snapshot ConnectionStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept {
+    Snapshot s;
+    s.insecure_connections     = insecure_connections     - rhs.insecure_connections;
+    s.tls_connections          = tls_connections          - rhs.tls_connections;
+    s.failed_tls_handshakes    = failed_tls_handshakes    - rhs.failed_tls_handshakes;
+    s.invalid_peer_credentials = invalid_peer_credentials - rhs.invalid_peer_credentials;
+    s.broken_tls_connections   = broken_tls_connections   - rhs.broken_tls_connections;
+    return s;
+}
+
+ConfigStatistics::Snapshot ConfigStatistics::snapshot() const noexcept {
+    Snapshot s;
+    s.successful_config_reloads = successful_config_reloads.load(std::memory_order_relaxed);
+    s.failed_config_reloads     = failed_config_reloads.load(std::memory_order_relaxed);
+    return s;
+}
+
+ConfigStatistics::Snapshot ConfigStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept {
+    Snapshot s;
+    s.successful_config_reloads = successful_config_reloads - rhs.successful_config_reloads;
+    s.failed_config_reloads     = failed_config_reloads     - rhs.failed_config_reloads;
+    return s;
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.h b/vespalib/src/vespa/vespalib/net/tls/statistics.h
new file mode 100644
index 00000000000..1fce4f48b0c
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/statistics.h
@@ -0,0 +1,99 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <atomic>
+#include <stdint.h>
+
+namespace vespalib::net::tls {
+
+/**
+ * Low-level statistics set by connection and credential management code
+ * for TLS and insecure plaintext connections.
+ *
+ * A poor man's substitute for not currently having the ability to natively
+ * export metrics in vespalib. Should be removed in favor of proper metrics
+ * once this is possible.
+ *
+ * Fully thread safe.
+ */
+struct ConnectionStatistics {
+
+    // Number of insecure (legacy) plaintext connections established
+    std::atomic<uint64_t> insecure_connections     = 0;
+    // Number of TLS connections successfully established. Note that
+    // the handshake has to succeed for a connection to be counted here.
+    std::atomic<uint64_t> tls_connections          = 0;
+    // Number of connections that failed during the TLS handshake process.
+    // May be caused by bad certificates, invalid credentials, bad ciphers etc.
+    std::atomic<uint64_t> failed_tls_handshakes    = 0;
+    // Number of connections rejected because the certificate did not have
+    // credentials that matched the requirements given in the TLS config file.
+    std::atomic<uint64_t> invalid_peer_credentials = 0;
+    // Number of connections broken due to errors during TLS encoding or decoding
+    std::atomic<uint64_t> broken_tls_connections   = 0;
+
+    void inc_insecure_connections() noexcept {
+        insecure_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_tls_connections() noexcept {
+        tls_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_failed_tls_handshakes() noexcept {
+        failed_tls_handshakes.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_invalid_peer_credentials() noexcept {
+        invalid_peer_credentials.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_broken_tls_connections() noexcept {
+        broken_tls_connections.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    struct Snapshot {
+        uint64_t insecure_connections     = 0;
+        uint64_t tls_connections          = 0;
+        uint64_t failed_tls_handshakes    = 0;
+        uint64_t invalid_peer_credentials = 0;
+        uint64_t broken_tls_connections   = 0;
+
+        Snapshot subtract(const Snapshot& rhs) const noexcept;
+    };
+
+    // Acquires a snapshot of statistics that is expected to be reasonably up to date.
+    // Thread safe.
+    Snapshot snapshot() const noexcept;
+
+    static ConnectionStatistics client_stats;
+    static ConnectionStatistics server_stats;
+
+    static ConnectionStatistics& get(bool is_server) noexcept {
+        return (is_server ? server_stats : client_stats);
+    }
+};
+
+struct ConfigStatistics {
+    std::atomic<uint64_t> successful_config_reloads = 0;
+    std::atomic<uint64_t> failed_config_reloads     = 0;
+
+    void inc_successful_config_reloads() noexcept {
+        successful_config_reloads.fetch_add(1, std::memory_order_relaxed);
+    }
+    void inc_failed_config_reloads() noexcept {
+        failed_config_reloads.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    struct Snapshot {
+        uint64_t successful_config_reloads = 0;
+        uint64_t failed_config_reloads     = 0;
+
+        Snapshot subtract(const Snapshot& rhs) const noexcept;
+    };
+
+    // Acquires a snapshot of statistics that is expected to be reasonably up to date.
+    // Thread safe.
+    Snapshot snapshot() const noexcept;
+
+    static ConfigStatistics instance;
+    static ConfigStatistics& get() noexcept { return instance; }
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_context.h b/vespalib/src/vespa/vespalib/net/tls/tls_context.h
index 04538694ee7..26637a955c2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_context.h
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_context.h
@@ -2,6 +2,8 @@
 #pragma once
 
 #include "authorization_mode.h"
+
+#include <chrono>
 #include <memory>
 
 namespace vespalib::net::tls {
@@ -12,6 +14,14 @@ struct CertificateVerificationCallback;
 struct TlsContext {
     virtual ~TlsContext() = default;
 
+    // Transport options this context was created with, but with the private key
+    // information scrubbed away.
+    virtual const TransportSecurityOptions& transport_security_options() const noexcept = 0;
+    // AuthorizationMode this context was created with
+    virtual AuthorizationMode authorization_mode() const noexcept = 0;
+    // Expiration time point of the peer certificate context was created with
+    virtual std::chrono::system_clock::time_point cert_expiration_time() const noexcept = 0;
+
     // Create a TLS context which verifies certificates according to the provided options'
     // CA trust roots AND authorized peer policies
     static std::shared_ptr<TlsContext> create_default_context(const TransportSecurityOptions&,