diff options
18 files changed, 366 insertions, 40 deletions
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 2df3d3a9606..73873e78032 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(storage_storageserver storagemetricsset.cpp storagenode.cpp storagenodecontext.cpp + tls_statistics_metrics_wrapper.cpp INSTALL lib64 DEPENDS storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index 4ea9a9f9296..f0e64f0dfd1 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -12,8 +12,9 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) normalpri("normalpri", {{"memory"}}, "Message use from normal priority storage messages", this), highpri("highpri", {{"memory"}}, "Message use from high priority storage messages", this), veryhighpri("veryhighpri", {{"memory"}}, "Message use from very high priority storage messages", this) -{ } -MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {} +{} + +MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) : metrics::MetricSet("document_serialization", {{"docserialization"}}, @@ -42,8 +43,9 @@ DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSe "Number of times we reserialized a document because the " "compression it had in cache did not match what was configured", this) -{ } -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { } +{} + +DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, @@ -52,9 +54,11 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this) -{ } -StorageMetricSet::~StorageMetricSet() { } + documentSerialization(this), + tls_metrics(this) +{} + +StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { document::SerializableArray::Statistics stats( @@ -72,6 +76,12 @@ void StorageMetricSet::updateMetrics() { stats._serializedUncompressed); documentSerialization.inputWronglySerialized.set( stats._inputWronglySerialized); + + // Delta snapshotting is destructive, so if an explicit snapshot is triggered + // (instead of just regular periodic snapshots), some events will effectively + // be erased from history. This will no longer be a problem once we move to a + // metrics system built around absolute (rather than derived) values. + tls_metrics.update_metrics_with_snapshot_delta(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index 40b70821bcd..e9378010540 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -2,6 +2,8 @@ #pragma once +#include "tls_statistics_metrics_wrapper.h" + #include <vespa/metrics/metrics.h> namespace storage { @@ -39,6 +41,8 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse_visiting; DocumentSerializationMetricSet documentSerialization; + TlsStatisticsMetricsWrapper tls_metrics; + StorageMetricSet(); ~StorageMetricSet(); void updateMetrics(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index d159b6e5bdd..64c0970cf9d 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -7,6 +7,7 @@ #include "statereporter.h" #include "storagemetricsset.h" #include "storagenodecontext.h" +#include "tls_statistics_metrics_wrapper.h" #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> @@ -159,7 +160,7 @@ StorageNode::initialize() _context.getComponentRegister().setPriorityConfig(*_priorityConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - _metrics.reset(new StorageMetricSet); + _metrics = std::make_shared<StorageMetricSet>(); _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode")); _component->registerMetric(*_metrics); if (!_context.getComponentRegister().hasMetricManager()) { diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp new file mode 100644 index 00000000000..707e1c84036 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tls_statistics_metrics_wrapper.h" + +namespace storage { + +TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* owner) + : metrics::MetricSet("network", {}, "Network connection metrics", owner), + insecure_client_connections_established("insecure_client_connections_established", {}, + "Number of insecure (plaintext) client connections established", this), + insecure_server_connections_accepted("insecure_server_connections_accepted", {}, + "Number of insecure (plaintext) server connections accepted", this), + tls_client_connections_established("tls_client_connections_established", {}, + "Number of secure mTLS client connections established", this), + tls_server_connections_accepted("tls_server_connections_accepted", {}, + "Number of secure mTLS server connections accepted", this), + tls_handshakes_failed("tls_handshakes_failed", {}, "Number of client or " + "server connection attempts that failed during TLS handshaking", this), + peer_authorization_failures("peer_authorization_failures", {}, + "Number of TLS connection attempts failed due to bad or missing " + "peer certificate credentials", this), + tls_connections_broken("tls_connections_broken", {}, "Number of TLS " + "connections broken due to failures during frame encoding or decoding", this), + failed_tls_config_reloads("failed_tls_config_reloads", {}, "Number of times " + "background reloading of TLS config has failed", this), + last_client_stats_snapshot(), + last_server_stats_snapshot(), + last_config_stats_snapshot() +{} + +TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default; + +void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() { + auto server_current = vespalib::net::tls::ConnectionStatistics::get(true).snapshot(); + auto client_current = vespalib::net::tls::ConnectionStatistics::get(false).snapshot(); + auto server_delta = server_current.subtract(last_server_stats_snapshot); + auto client_delta = client_current.subtract(last_client_stats_snapshot); + + insecure_client_connections_established.set(client_delta.insecure_connections); + insecure_server_connections_accepted.set(server_delta.insecure_connections); + tls_client_connections_established.set(client_delta.tls_connections); + tls_server_connections_accepted.set(server_delta.tls_connections); + // We have underlying stats for both server and client here, but for the + // moment we just aggregate them up into combined metrics. Can be trivially + // split up into separate metrics later if deemed useful. + tls_handshakes_failed.set(client_delta.failed_tls_handshakes + + server_delta.failed_tls_handshakes); + peer_authorization_failures.set(client_delta.invalid_peer_credentials + + server_delta.invalid_peer_credentials); + tls_connections_broken.set(client_delta.broken_tls_connections + + server_delta.broken_tls_connections); + + auto config_current = vespalib::net::tls::ConfigStatistics::get().snapshot(); + auto config_delta = config_current.subtract(last_config_stats_snapshot); + + failed_tls_config_reloads.set(config_delta.failed_config_reloads); + + last_server_stats_snapshot = server_current; + last_client_stats_snapshot = client_current; + last_config_stats_snapshot = config_current; +} + +} diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h new file mode 100644 index 00000000000..9784aaddd1c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/metrics/metrics.h> +#include <vespa/vespalib/net/tls/statistics.h> + +#include <chrono> + +namespace storage { + +// Simple wrapper around low-level vespalib network statistics which +// converts the monotonically increasing counters to deltas during +// periodic metric snapshotting. +class TlsStatisticsMetricsWrapper : public metrics::MetricSet { + metrics::LongCountMetric insecure_client_connections_established; + metrics::LongCountMetric insecure_server_connections_accepted; + metrics::LongCountMetric tls_client_connections_established; + metrics::LongCountMetric tls_server_connections_accepted; + metrics::LongCountMetric tls_handshakes_failed; + metrics::LongCountMetric peer_authorization_failures; + metrics::LongCountMetric tls_connections_broken; + + metrics::LongCountMetric failed_tls_config_reloads; + + vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot; + vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot; + vespalib::net::tls::ConfigStatistics::Snapshot last_config_stats_snapshot; + +public: + explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner); + ~TlsStatisticsMetricsWrapper() override; + + void update_metrics_with_snapshot_delta(); +}; + +} diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp index 245368b6a7b..5dc85bc567f 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> @@ -106,17 +107,11 @@ struct Fixture { } vespalib::string current_cert_chain() const { - auto impl = engine->acquire_current_engine(); - // Leaks implementation details galore, but it's not very likely that we'll use - // anything but OpenSSL (or compatible APIs) in practice... - auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context()); - return ctx_impl.transport_security_options().cert_chain_pem(); + return engine->acquire_current_engine()->tls_context()->transport_security_options().cert_chain_pem(); } AuthorizationMode current_authorization_mode() const { - auto impl = engine->acquire_current_engine(); - auto& ctx_impl = dynamic_cast<const impl::OpenSslTlsContextImpl&>(*impl->tls_context()); - return ctx_impl.authorization_mode(); + return engine->acquire_current_engine()->tls_context()->authorization_mode(); } }; @@ -143,4 +138,15 @@ TEST_FF("Authorization mode is propagated to engine", Fixture(50ms, Authorizatio EXPECT_EQUAL(AuthorizationMode::LogOnly, f1.current_authorization_mode()); } +TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), TimeBomb(60)) { + auto before = ConfigStatistics::get().snapshot(); + + write_file("test_cert.pem.tmp", "Broken file oh no :("); + rename("test_cert.pem.tmp", "test_cert.pem", false, false); + + while (ConfigStatistics::get().snapshot().subtract(before).failed_config_reloads == 0) { + std::this_thread::sleep_for(10ms); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 69e0d44147e..f70c5670bc9 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/crypto_codec.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/tls_context.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h> @@ -619,6 +620,37 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF EXPECT_TRUE(f.handshake()); } +TEST_F("Failure statistics are incremented on authorization failures", CertFixture) { + reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce); + auto server_before = ConnectionStatistics::get(true).snapshot(); + auto client_before = ConnectionStatistics::get(false).snapshot(); + EXPECT_FALSE(f.handshake()); + auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before); + auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before); + + EXPECT_EQUAL(1u, server_stats.invalid_peer_credentials); + EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials); + EXPECT_EQUAL(1u, server_stats.failed_tls_handshakes); + EXPECT_EQUAL(0u, server_stats.tls_connections); + EXPECT_EQUAL(0u, client_stats.tls_connections); +} + +TEST_F("Success statistics are incremented on OK authorization", CertFixture) { + reset_peers_with_server_authz_mode(f, AuthorizationMode::Disable); + auto server_before = ConnectionStatistics::get(true).snapshot(); + auto client_before = ConnectionStatistics::get(false).snapshot(); + EXPECT_TRUE(f.handshake()); + auto server_stats = ConnectionStatistics::get(true).snapshot().subtract(server_before); + auto client_stats = ConnectionStatistics::get(false).snapshot().subtract(client_before); + + EXPECT_EQUAL(0u, server_stats.invalid_peer_credentials); + EXPECT_EQUAL(0u, client_stats.invalid_peer_credentials); + EXPECT_EQUAL(0u, server_stats.failed_tls_handshakes); + EXPECT_EQUAL(0u, client_stats.failed_tls_handshakes); + EXPECT_EQUAL(1u, server_stats.tls_connections); + EXPECT_EQUAL(1u, client_stats.tls_connections); +} + // TODO we can't test embedded nulls since the OpenSSL v3 extension APIs // take in null terminated strings as arguments... :I diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index b771e35dab1..e291f39a834 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> @@ -243,8 +244,9 @@ CryptoEngine::get_default() } CryptoSocket::UP -NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool) +NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) { + net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections(); return std::make_unique<NullCryptoSocket>(std::move(socket)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index 2f8c490b911..6dc48db68e4 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT peer_policies.cpp policy_checking_certificate_verifier.cpp protocol_snooping.cpp + statistics.cpp tls_context.cpp tls_crypto_engine.cpp tls_crypto_socket.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index 9e38efabb2b..5f20280e0e2 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "auto_reloading_tls_crypto_engine.h" +#include "statistics.h" #include "tls_context.h" #include "tls_crypto_engine.h" #include "transport_security_options.h" @@ -29,6 +30,7 @@ std::shared_ptr<TlsCryptoEngine> try_create_engine_from_tls_config(const vespali } catch (std::exception& e) { LOG(warning, "Failed to reload TLS config file (%s): '%s'. Old config remains in effect.", config_file_path.c_str(), e.what()); + ConfigStatistics::get().inc_failed_config_reloads(); return {}; } } @@ -78,6 +80,7 @@ void AutoReloadingTlsCryptoEngine::run_reload_loop() { void AutoReloadingTlsCryptoEngine::try_replace_current_engine() { auto new_engine = try_create_engine_from_tls_config(_config_file_path, _authorization_mode); if (new_engine) { + ConfigStatistics::get().inc_successful_config_reloads(); std::lock_guard guard(_engine_mutex); _current_engine = std::move(new_engine); } diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp index fee2b19f911..e8b1e32213d 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp @@ -2,12 +2,16 @@ #include "openssl_crypto_codec_impl.h" #include "openssl_tls_context_impl.h" #include "direct_buffer_bio.h" + #include <vespa/vespalib/net/tls/crypto_codec.h> #include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/net/tls/statistics.h> + #include <mutex> #include <vector> #include <memory> #include <stdexcept> + #include <openssl/ssl.h> #include <openssl/crypto.h> #include <openssl/err.h> @@ -250,9 +254,11 @@ HandshakeResult OpenSslCryptoCodecImpl::do_handshake_and_consume_peer_input_byte return handshake_failed(); } LOG(debug, "SSL_do_handshake() is complete, using protocol %s", SSL_get_version(_ssl.get())); + ConnectionStatistics::get(_mode == Mode::Server).inc_tls_connections(); return handshake_consumed_bytes_and_is_complete(static_cast<size_t>(consumed)); } else { log_ssl_error("SSL_do_handshake()", ssl_result); + ConnectionStatistics::get(_mode == Mode::Server).inc_failed_tls_handshakes(); return handshake_failed(); } } @@ -277,6 +283,7 @@ EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaint const int consumed = ::SSL_write(_ssl.get(), plaintext, to_consume); if (consumed < 0) { log_ssl_error("SSL_write()", ::SSL_get_error(_ssl.get(), consumed)); + ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections(); return encode_failed(); // TODO explicitly detect and log TLS renegotiation error (SSL_ERROR_WANT_READ)? } else if (consumed != to_consume) { LOG(error, "SSL_write() returned OK but did not consume all requested plaintext"); @@ -340,6 +347,7 @@ DecodeResult OpenSslCryptoCodecImpl::remap_ssl_read_failure_to_decode_result(int return decode_peer_has_closed(); default: log_ssl_error("SSL_read()", ssl_error); + ConnectionStatistics::get(_mode == Mode::Server).inc_broken_tls_connections(); return decode_failed(); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index 53ed4d7bdff..fb29af21ea8 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -2,6 +2,7 @@ #include "openssl_typedefs.h" #include "openssl_tls_context_impl.h" #include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/util/stringfmt.h> #include <mutex> @@ -407,52 +408,55 @@ int OpenSslTlsContextImpl::verify_cb_wrapper(int preverified_ok, ::X509_STORE_CT // since we trust the intermediates to have done their job. const bool is_peer_cert = (::X509_STORE_CTX_get_error_depth(store_ctx) == 0); if (!is_peer_cert) { - return 1; // OK for root/intermediate cert. + return 1; // OK for root/intermediate cert. Callback will be invoked again for other certs. } // Fetch the SSL instance associated with the X509_STORE_CTX const void* data = ::X509_STORE_CTX_get_ex_data(store_ctx, ::SSL_get_ex_data_X509_STORE_CTX_idx()); - if (!data) { - return 0; - } + LOG_ASSERT(data != nullptr); const auto* ssl = static_cast<const ::SSL*>(data); const ::SSL_CTX* ssl_ctx = ::SSL_get_SSL_CTX(ssl); - if (!ssl_ctx) { - return 0; - } + LOG_ASSERT(ssl_ctx != nullptr); auto* self = static_cast<OpenSslTlsContextImpl*>(SSL_CTX_get_app_data(ssl_ctx)); - if (!self) { - return 0; + LOG_ASSERT(self != nullptr); + + if (self->verify_trusted_certificate(store_ctx)) { + return 1; } - const auto authz_mode = self->authorization_mode(); + ConnectionStatistics::get(SSL_in_accept_init(ssl) != 0).inc_invalid_peer_credentials(); + return 0; +} + +bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_ctx) { + const auto authz_mode = authorization_mode(); // TODO consider if we want to fill in peer credentials even if authorization is disabled if (authz_mode == AuthorizationMode::Disable) { - return 1; + return true; } ::X509* cert = ::X509_STORE_CTX_get_current_cert(store_ctx); // _not_ owned by us if (!cert) { LOG(error, "Got X509_STORE_CTX with preverified_ok == 1 but no current cert"); - return 0; + return false; } PeerCredentials creds; if (!fill_certificate_common_name(cert, creds)) { - return 0; + return false; } if (!fill_certificate_subject_alternate_names(cert, creds)) { - return 0; + return false; } try { - const bool verified_by_cb = self->_cert_verify_callback->verify(creds); + const bool verified_by_cb = _cert_verify_callback->verify(creds); if (!verified_by_cb) { // TODO we should print the peer's remote address too, but that information is // not currently available to us here. LOG(warning, "Certificate verification failed for %s", to_string(creds).c_str()); - return (authz_mode == AuthorizationMode::Enforce) ? 0 : 1; + return (authz_mode != AuthorizationMode::Enforce); } } catch (std::exception& e) { LOG(error, "Got exception during certificate verification callback: %s", e.what()); - return 0; + return false; } // we don't expect any non-std::exception derived exceptions, so let them terminate the process. - return 1; + return true; } void OpenSslTlsContextImpl::enforce_peer_certificate_verification() { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h index 757204325b0..6b72dfe84ee 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h @@ -7,6 +7,8 @@ #include <vespa/vespalib/net/tls/certificate_verification_callback.h> #include <vespa/vespalib/stllike/string.h> +#include <chrono> + namespace vespalib::net::tls::impl { class OpenSslTlsContextImpl : public TlsContext { @@ -21,13 +23,10 @@ public: ~OpenSslTlsContextImpl() override; ::SSL_CTX* native_context() const noexcept { return _ctx.get(); } - // Transport options this context was created with, but with the private key - // information scrubbed away. - const TransportSecurityOptions& transport_security_options() const noexcept { + const TransportSecurityOptions& transport_security_options() const noexcept override { return _redacted_transport_options; } - // AuthorizationMode this context was created with - AuthorizationMode authorization_mode() const noexcept { return _authorization_mode; } + AuthorizationMode authorization_mode() const noexcept override { return _authorization_mode; } private: // Note: single use per instance; does _not_ clear existing chain! void add_certificate_authorities(stringref ca_pem); @@ -45,6 +44,8 @@ private: void enforce_peer_certificate_verification(); void set_ssl_ctx_self_reference(); + bool verify_trusted_certificate(::X509_STORE_CTX* store_ctx); + static int verify_cb_wrapper(int preverified_ok, ::X509_STORE_CTX* store_ctx); }; diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index 4b9bc8a30b4..b6969250ae5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "maybe_tls_crypto_socket.h" +#include "statistics.h" #include "tls_crypto_socket.h" #include "protocol_snooping.h" #include <vespa/vespalib/data/smart_buffer.h> @@ -54,6 +55,7 @@ public: self = std::move(tls_socket); return self->handshake(); } else { + net::tls::ConnectionStatistics::get(true).inc_insecure_connections(); _factory.reset(); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.cpp b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp new file mode 100644 index 00000000000..d11aa60266e --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "statistics.h" + +namespace vespalib::net::tls { + +ConnectionStatistics ConnectionStatistics::client_stats = {}; +ConnectionStatistics ConnectionStatistics::server_stats = {}; + +ConfigStatistics ConfigStatistics::instance = {}; + +ConnectionStatistics::Snapshot ConnectionStatistics::snapshot() const noexcept { + Snapshot s; + s.insecure_connections = insecure_connections.load(std::memory_order_relaxed); + s.tls_connections = tls_connections.load(std::memory_order_relaxed); + s.failed_tls_handshakes = failed_tls_handshakes.load(std::memory_order_relaxed); + s.invalid_peer_credentials = invalid_peer_credentials.load(std::memory_order_relaxed); + s.broken_tls_connections = broken_tls_connections.load(std::memory_order_relaxed); + return s; +} + +ConnectionStatistics::Snapshot ConnectionStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept { + Snapshot s; + s.insecure_connections = insecure_connections - rhs.insecure_connections; + s.tls_connections = tls_connections - rhs.tls_connections; + s.failed_tls_handshakes = failed_tls_handshakes - rhs.failed_tls_handshakes; + s.invalid_peer_credentials = invalid_peer_credentials - rhs.invalid_peer_credentials; + s.broken_tls_connections = broken_tls_connections - rhs.broken_tls_connections; + return s; +} + +ConfigStatistics::Snapshot ConfigStatistics::snapshot() const noexcept { + Snapshot s; + s.successful_config_reloads = successful_config_reloads.load(std::memory_order_relaxed); + s.failed_config_reloads = failed_config_reloads.load(std::memory_order_relaxed); + return s; +} + +ConfigStatistics::Snapshot ConfigStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept { + Snapshot s; + s.successful_config_reloads = successful_config_reloads - rhs.successful_config_reloads; + s.failed_config_reloads = failed_config_reloads - rhs.failed_config_reloads; + return s; +} + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.h b/vespalib/src/vespa/vespalib/net/tls/statistics.h new file mode 100644 index 00000000000..1fce4f48b0c --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/statistics.h @@ -0,0 +1,99 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <atomic> +#include <stdint.h> + +namespace vespalib::net::tls { + +/** + * Low-level statistics set by connection and credential management code + * for TLS and insecure plaintext connections. + * + * A poor man's substitute for not currently having the ability to natively + * export metrics in vespalib. Should be removed in favor of proper metrics + * once this is possible. + * + * Fully thread safe. + */ +struct ConnectionStatistics { + + // Number of insecure (legacy) plaintext connections established + std::atomic<uint64_t> insecure_connections = 0; + // Number of TLS connections successfully established. Note that + // the handshake has to succeed for a connection to be counted here. + std::atomic<uint64_t> tls_connections = 0; + // Number of connections that failed during the TLS handshake process. + // May be caused by bad certificates, invalid credentials, bad ciphers etc. + std::atomic<uint64_t> failed_tls_handshakes = 0; + // Number of connections rejected because the certificate did not have + // credentials that matched the requirements given in the TLS config file. + std::atomic<uint64_t> invalid_peer_credentials = 0; + // Number of connections broken due to errors during TLS encoding or decoding + std::atomic<uint64_t> broken_tls_connections = 0; + + void inc_insecure_connections() noexcept { + insecure_connections.fetch_add(1, std::memory_order_relaxed); + } + void inc_tls_connections() noexcept { + tls_connections.fetch_add(1, std::memory_order_relaxed); + } + void inc_failed_tls_handshakes() noexcept { + failed_tls_handshakes.fetch_add(1, std::memory_order_relaxed); + } + void inc_invalid_peer_credentials() noexcept { + invalid_peer_credentials.fetch_add(1, std::memory_order_relaxed); + } + void inc_broken_tls_connections() noexcept { + broken_tls_connections.fetch_add(1, std::memory_order_relaxed); + } + + struct Snapshot { + uint64_t insecure_connections = 0; + uint64_t tls_connections = 0; + uint64_t failed_tls_handshakes = 0; + uint64_t invalid_peer_credentials = 0; + uint64_t broken_tls_connections = 0; + + Snapshot subtract(const Snapshot& rhs) const noexcept; + }; + + // Acquires a snapshot of statistics that is expected to be reasonably up to date. + // Thread safe. + Snapshot snapshot() const noexcept; + + static ConnectionStatistics client_stats; + static ConnectionStatistics server_stats; + + static ConnectionStatistics& get(bool is_server) noexcept { + return (is_server ? server_stats : client_stats); + } +}; + +struct ConfigStatistics { + std::atomic<uint64_t> successful_config_reloads = 0; + std::atomic<uint64_t> failed_config_reloads = 0; + + void inc_successful_config_reloads() noexcept { + successful_config_reloads.fetch_add(1, std::memory_order_relaxed); + } + void inc_failed_config_reloads() noexcept { + failed_config_reloads.fetch_add(1, std::memory_order_relaxed); + } + + struct Snapshot { + uint64_t successful_config_reloads = 0; + uint64_t failed_config_reloads = 0; + + Snapshot subtract(const Snapshot& rhs) const noexcept; + }; + + // Acquires a snapshot of statistics that is expected to be reasonably up to date. + // Thread safe. + Snapshot snapshot() const noexcept; + + static ConfigStatistics instance; + static ConfigStatistics& get() noexcept { return instance; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_context.h b/vespalib/src/vespa/vespalib/net/tls/tls_context.h index 04538694ee7..ea2a652a701 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_context.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_context.h @@ -2,6 +2,7 @@ #pragma once #include "authorization_mode.h" + #include <memory> namespace vespalib::net::tls { @@ -12,6 +13,12 @@ struct CertificateVerificationCallback; struct TlsContext { virtual ~TlsContext() = default; + // Transport options this context was created with, but with the private key + // information scrubbed away. + virtual const TransportSecurityOptions& transport_security_options() const noexcept = 0; + // AuthorizationMode this context was created with + virtual AuthorizationMode authorization_mode() const noexcept = 0; + // Create a TLS context which verifies certificates according to the provided options' // CA trust roots AND authorized peer policies static std::shared_ptr<TlsContext> create_default_context(const TransportSecurityOptions&, |