Add functionality for detecting whether a client is using TLS or not

Inspects first 8 bytes of a client's initial data stream to determine
if it's (with very high confidence) a TLS ClientHello message.

7 files changed, 236 insertions, 4 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index b32c875cb26..3918c72e81a 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -59,6 +59,7 @@ vespa_define_module(
     src/tests/net/socket_spec
     src/tests/net/tls/direct_buffer_bio
     src/tests/net/tls/openssl_impl
+    src/tests/net/tls/protocol_snooping
     src/tests/net/tls/transport_options
     src/tests/objects/nbostream
     src/tests/optimized
diff --git a/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt
new file mode 100644
index 00000000000..1489859fe48
--- /dev/null
+++ b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_net_tls_protocol_snooping_test_app TEST
+    SOURCES
+    protocol_snooping_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_net_tls_protocol_snooping_test_app COMMAND vespalib_net_tls_protocol_snooping_test_app)
+
diff --git a/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp
new file mode 100644
index 00000000000..e4cf06f6631
--- /dev/null
+++ b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/net/tls/protocol_snooping.h>
+
+using namespace vespalib;
+using namespace vespalib::net::tls;
+
+TEST("min_header_bytes_to_observe() is 8") {
+    EXPECT_EQUAL(8u, min_header_bytes_to_observe());
+}
+
+TlsSnoopingResult do_snoop(const unsigned char* buf) {
+    return snoop_client_hello_header(reinterpret_cast<const char*>(buf));
+}
+
+TEST("Well-formed TLSv1.0 packet returns ProbablyTls") {
+    const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 0, 10 };
+    EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf));
+}
+
+TEST("Well-formed TLSv1.2 packet returns ProbablyTls") {
+    const unsigned char buf[] = { 22, 3, 3, 10, 255, 1, 0, 10 };
+    EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf));
+}
+
+TEST("Mismatching handshake header byte 1 returns HandshakeMismatch") {
+    const unsigned char buf[] = { 23, 3, 1, 10, 255, 1, 0, 10 };
+    EXPECT_EQUAL(TlsSnoopingResult::HandshakeMismatch, do_snoop(buf));
+}
+
+TEST("Mismatching handshake header byte 2 returns HandshakeMismatch") {
+    const unsigned char buf[] = { 22, 2, 1, 10, 255, 1, 0, 10 };
+    EXPECT_EQUAL(TlsSnoopingResult::HandshakeMismatch, do_snoop(buf));
+}
+
+TEST("Oversized record returns RecordSizeRfcViolation") {
+    const unsigned char buf1[] = { 22, 3, 1, 255, 255, 1, 0, 10 }; // 64k
+    //                                       ^^^^^^^^ big endian record length
+    EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf1));
+
+    const unsigned char buf2[] = { 22, 3, 1, 72, 1, 1, 0, 10 }; // 18K+1
+    EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf2));
+}
+
+TEST("Non-ClientHello handshake record returns RecordNotClientHello") {
+    const unsigned char buf[] = { 22, 3, 1, 10, 255, 2, 0, 10 };
+    //                                               ^ 1 == ClientHello
+    EXPECT_EQUAL(TlsSnoopingResult::RecordNotClientHello, do_snoop(buf));
+}
+
+TEST("Oversized or fragmented ClientHello record returns ClientHelloRecordTooBig") {
+    const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 1, 10 };
+    //                                                  ^ MSB of 24-bit record length
+    EXPECT_EQUAL(TlsSnoopingResult::ClientHelloRecordTooBig, do_snoop(buf));
+}
+
+TEST("Expected ClientHello record size mismatch returns ExpectedRecordSizeMismatch") {
+    const unsigned char buf[] = { 22, 3, 1, 10, 2, 1, 0, 10 };
+    //                                                   ^^ bits [8,16) of record length, should be 9
+    EXPECT_EQUAL(TlsSnoopingResult::ExpectedRecordSizeMismatch, do_snoop(buf));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
index 2d34a3e1c80..17abd82366d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT
     crypto_codec.cpp
     crypto_codec_adapter.cpp
     crypto_exception.cpp
+    protocol_snooping.cpp
     tls_context.cpp
     tls_crypto_engine.cpp
     transport_security_options.cpp
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
index 4c253fdf24c..53693dd7f40 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
@@ -11,10 +11,8 @@ namespace vespalib::net::tls { class TlsContext; }
 namespace vespalib::net::tls::impl {
 
 /*
- * Frame-level OpenSSL-backed TLSv1.2 crypto codec implementation.
- *
- * Currently has sub-optimal buffer management, and is mostly intended
- * as a starting point.
+ * Frame-level OpenSSL-backed TLSv1.2/TLSv1.3 (depending on OpenSSL version)
+ * crypto codec implementation.
  *
  * NOT thread safe per instance, but independent instances may be
  * used by different threads safely.
diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp
new file mode 100644
index 00000000000..04a0a29be64
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp
@@ -0,0 +1,128 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "protocol_snooping.h"
+#include <iostream>
+#include <cstdlib>
+#include <stdint.h>
+
+namespace vespalib::net::tls {
+
+namespace {
+
+// Precondition for all helper functions: buffer is at least `min_header_bytes_to_observe()` bytes long
+
+// From RFC 5246:
+// 0x16 - Handshake content type byte of TLSCiphertext record
+// 0x03 - First byte of 2-byte ProtocolVersion, always 3 on TLSv1.2 and v1.3
+inline bool is_tls_handshake_packet(const char* buf) {
+    return ((buf[0] == 0x16) && (buf[1] == 0x03));
+}
+
+// Next is the TLS minor version, either 1 or 3 depending on version (though the
+// RFCs say it _should_ be 1 for backwards compatibility reasons).
+// Yes, the TLS spec says that you should technically ignore the protocol version
+// field here, but we want all the signals we can get.
+inline bool is_expected_tls_protocol_version(const char* buf) {
+    return ((buf[2] == 0x01) || (buf[2] == 0x03));
+}
+
+// Length is big endian u16 in bytes 3, 4
+inline uint16_t tls_record_length(const char* buf) {
+    return (uint16_t(static_cast<unsigned char>(buf[3]) << 8)
+            + static_cast<unsigned char>(buf[4]));
+}
+
+// First byte of Handshake record in byte 5, which shall be ClientHello (0x01)
+inline bool is_client_hello_handshake_record(const char* buf) {
+    return (buf[5] == 0x01);
+}
+
+// Last 2 bytes are the 2 first big-endian bytes of a 3-byte Handshake
+// record length field. No support for records that are large enough that
+// the MSB should ever be non-zero.
+inline bool client_hello_record_size_within_expected_bounds(const char* buf) {
+    return (buf[6] == 0x00);
+}
+
+// The byte after the MSB of the 24-bit handshake record size should be equal
+// to the most significant byte of the record length value, minus the Handshake
+// record header size.
+// Again, we make the assumption that ClientHello messages are not fragmented,
+// so their max size must be <= 16KiB. This also just happens to be a lower
+// number than the minimum FS4/FRT packet type byte at the same location.
+// Oooh yeah, leaky abstractions to the rescue!
+inline bool handshake_record_size_matches_length(const char* buf, uint16_t length) {
+    return (static_cast<unsigned char>(buf[7]) == ((length - 4) >> 8));
+}
+
+} // anon ns
+
+TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept {
+    if (!is_tls_handshake_packet(buf)) {
+        return TlsSnoopingResult::HandshakeMismatch;
+    }
+    if (!is_expected_tls_protocol_version(buf)) {
+        return TlsSnoopingResult::ProtocolVersionMismatch;
+    }
+    // Length of TLS record follows. Must be <= 16KiB + 2048 (16KiB + 256 on v1.3).
+    // We expect that the first record contains _only_ a ClientHello with no coalescing
+    // and no fragmentation. This is technically a violation of the TLS spec, but this
+    // particular detection logic is only intended to be used against other Vespa nodes
+    // where we control frame sizes and where such fragmentation should not take place.
+    // We also do not support TLSv1.3 0-RTT which may trigger early data.
+    uint16_t length = tls_record_length(buf);
+    if (length > (16384 + 2048)) {
+        return TlsSnoopingResult::RecordSizeRfcViolation;
+    }
+    if (!is_client_hello_handshake_record(buf)) {
+        return TlsSnoopingResult::RecordNotClientHello;
+    }
+    if (!client_hello_record_size_within_expected_bounds(buf)) {
+        return TlsSnoopingResult::ClientHelloRecordTooBig;
+    }
+    if (!handshake_record_size_matches_length(buf, length)) {
+        return TlsSnoopingResult::ExpectedRecordSizeMismatch;
+    }
+    // Hooray! It very probably most likely is a TLS connection! :D
+    return TlsSnoopingResult::ProbablyTls;
+}
+
+const char* to_string(TlsSnoopingResult result) noexcept {
+    switch (result) {
+    case TlsSnoopingResult::ProbablyTls:                return "ProbablyTls";
+    case TlsSnoopingResult::HandshakeMismatch:          return "HandshakeMismatch";
+    case TlsSnoopingResult::ProtocolVersionMismatch:    return "ProtocolVersionMismatch";
+    case TlsSnoopingResult::RecordSizeRfcViolation:     return "RecordSizeRfcViolation";
+    case TlsSnoopingResult::RecordNotClientHello:       return "RecordNotClientHello";
+    case TlsSnoopingResult::ClientHelloRecordTooBig:    return "ClientHelloRecordTooBig";
+    case TlsSnoopingResult::ExpectedRecordSizeMismatch: return "ExpectedRecordSizeMismatch";
+    default: return "Unknown TlsSnoopingResult";
+    }
+}
+
+std::ostream& operator<<(std::ostream& os, TlsSnoopingResult result) {
+    os << to_string(result);
+    return os;
+}
+
+const char* describe_result(TlsSnoopingResult result) noexcept {
+    switch (result) {
+    case TlsSnoopingResult::ProbablyTls:
+        return "client data matches TLS heuristics, very likely a TLS connection";
+    case TlsSnoopingResult::HandshakeMismatch:
+        return "not a TLS handshake packet";
+    case TlsSnoopingResult::ProtocolVersionMismatch:
+        return "ProtocolVersion mismatch";
+    case TlsSnoopingResult::RecordSizeRfcViolation:
+        return "ClientHello record size is greater than RFC allows";
+    case TlsSnoopingResult::RecordNotClientHello:
+        return "record is not ClientHello";
+    case TlsSnoopingResult::ClientHelloRecordTooBig:
+        return "ClientHello record is too big (fragmented?)";
+    case TlsSnoopingResult::ExpectedRecordSizeMismatch:
+        return "ClientHello vs Handshake header record size mismatch";
+    default:
+        return "Unknown TlsSnoopingResult";
+    }
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h
new file mode 100644
index 00000000000..59f6ecf6e9d
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <iosfwd>
+#include <stddef.h>
+
+namespace vespalib::net::tls {
+
+constexpr inline size_t min_header_bytes_to_observe() { return 8; }
+
+enum class TlsSnoopingResult {
+    ProbablyTls, // Very safe to assume TLSv1.x client
+    HandshakeMismatch, // Almost guaranteed to trigger for plaintext RPC
+    ProtocolVersionMismatch,
+    RecordSizeRfcViolation,
+    RecordNotClientHello,
+    ClientHelloRecordTooBig,
+    ExpectedRecordSizeMismatch
+};
+
+const char* to_string(TlsSnoopingResult) noexcept;
+std::ostream& operator<<(std::ostream& os, TlsSnoopingResult);
+
+// Precondition: buf is at least `min_header_bytes_to_observe()` bytes long. This is the minimum amount
+// of bytes always sent for a packet in our existing plaintext production protocols and
+// therefore the maximum we can expect to always be present.
+// Yes, this is a pragmatic and delightfully leaky abstraction.
+TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept;
+
+const char* describe_result(TlsSnoopingResult result) noexcept;
+
+}