Introduce custom OpenSSL BIOs for providing direct buffer read/write

BIOs offer a dynamic view into source or sink (const/mutable) buffers
and avoids overhead of copying from/to memory BIOs. Also strictly
enforces buffer sizes to ensure there are no hidden reallocs.

Additionally make code OpenSSL 1.1+ and TLSv1.3 compatible.

9 files changed, 745 insertions, 147 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index fb3b08b325f..b32c875cb26 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -57,6 +57,7 @@ vespa_define_module(
     src/tests/net/send_fd
     src/tests/net/socket
     src/tests/net/socket_spec
+    src/tests/net/tls/direct_buffer_bio
     src/tests/net/tls/openssl_impl
     src/tests/net/tls/transport_options
     src/tests/objects/nbostream
diff --git a/vespalib/src/tests/net/tls/direct_buffer_bio/CMakeLists.txt b/vespalib/src/tests/net/tls/direct_buffer_bio/CMakeLists.txt
new file mode 100644
index 00000000000..cbce4d3651f
--- /dev/null
+++ b/vespalib/src/tests/net/tls/direct_buffer_bio/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_net_tls_direct_buffer_bio_test_app TEST
+    SOURCES
+    direct_buffer_bio_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_net_tls_direct_buffer_bio_test_app
+               COMMAND vespalib_net_tls_direct_buffer_bio_test_app)
+
diff --git a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp
new file mode 100644
index 00000000000..4c51e91fcf8
--- /dev/null
+++ b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp
@@ -0,0 +1,138 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/net/tls/impl/direct_buffer_bio.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <cassert>
+
+using namespace vespalib;
+using namespace vespalib::net::tls::impl;
+
+struct Fixture {
+    BioPtr mutable_bio;
+    BioPtr const_bio;
+    vespalib::string tmp_buf;
+
+    Fixture()
+        : mutable_bio(new_mutable_direct_buffer_bio()),
+          const_bio(new_const_direct_buffer_bio()),
+          tmp_buf('X', 64)
+    {
+        assert(mutable_bio && const_bio);
+    }
+};
+
+TEST_F("BIOs without associated buffers return zero pending", Fixture) {
+    EXPECT_EQUAL(0, BIO_pending(f.mutable_bio.get()));
+    EXPECT_EQUAL(0, BIO_pending(f.const_bio.get()));
+}
+
+TEST_F("Const BIO has initial pending equal to size of associated buffer", Fixture) {
+    vespalib::string to_read = "I sure love me some data";
+    ConstBufferViewGuard g(*f.const_bio, &to_read[0], to_read.size());
+    EXPECT_EQUAL(static_cast<int>(to_read.size()), BIO_pending(f.const_bio.get()));
+}
+
+TEST_F("Mutable BIO has initial pending of 0 with associated buffer (pending == written bytes)", Fixture) {
+    MutableBufferViewGuard g(*f.mutable_bio, &f.tmp_buf[0], f.tmp_buf.size());
+    EXPECT_EQUAL(0, BIO_pending(f.mutable_bio.get()));
+}
+
+TEST_F("Mutable BIO_write writes to associated buffer", Fixture) {
+    MutableBufferViewGuard g(*f.mutable_bio, &f.tmp_buf[0], f.tmp_buf.size());
+    vespalib::string to_write = "hello world!";
+    int ret = ::BIO_write(f.mutable_bio.get(), to_write.data(), static_cast<int>(to_write.size()));
+    EXPECT_EQUAL(static_cast<int>(to_write.size()), ret);
+    EXPECT_EQUAL(to_write, vespalib::stringref(f.tmp_buf.data(), to_write.size()));
+    EXPECT_EQUAL(static_cast<int>(to_write.size()), BIO_pending(f.mutable_bio.get()));
+}
+
+TEST_F("Mutable BIO_write moves write cursor per invocation", Fixture) {
+    MutableBufferViewGuard g(*f.mutable_bio, &f.tmp_buf[0], f.tmp_buf.size());
+    vespalib::string to_write = "hello world!";
+
+    int ret = ::BIO_write(f.mutable_bio.get(), to_write.data(), 3); // 'hel'
+    ASSERT_EQUAL(3, ret);
+    EXPECT_EQUAL(3, BIO_pending(f.mutable_bio.get()));
+    ret = ::BIO_write(f.mutable_bio.get(), to_write.data() + 3, 5); // 'lo wo'
+    ASSERT_EQUAL(5, ret);
+    EXPECT_EQUAL(8, BIO_pending(f.mutable_bio.get()));
+    ret = ::BIO_write(f.mutable_bio.get(), to_write.data() + 8, 4); // 'rld!'
+    ASSERT_EQUAL(4, ret);
+    EXPECT_EQUAL(12, BIO_pending(f.mutable_bio.get()));
+
+    EXPECT_EQUAL(to_write, vespalib::stringref(f.tmp_buf.data(), to_write.size()));
+}
+
+TEST_F("Const BIO_read reads from associated buffer", Fixture) {
+    vespalib::string to_read = "look at this fancy data!";
+    ConstBufferViewGuard g(*f.const_bio, &to_read[0], to_read.size());
+
+    int ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[0], static_cast<int>(f.tmp_buf.size()));
+    EXPECT_EQUAL(static_cast<int>(to_read.size()), ret);
+    EXPECT_EQUAL(ret, static_cast<int>(to_read.size()));
+
+    EXPECT_EQUAL(to_read, vespalib::stringref(f.tmp_buf.data(), to_read.size()));
+}
+
+TEST_F("Const BIO_read moves read cursor per invocation", Fixture) {
+    vespalib::string to_read = "look at this fancy data!";
+    ConstBufferViewGuard g(*f.const_bio, &to_read[0], to_read.size());
+
+    EXPECT_EQUAL(24, BIO_pending(f.const_bio.get()));
+    int ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[0], 8); // 'look at '
+    ASSERT_EQUAL(8, ret);
+    EXPECT_EQUAL(16, BIO_pending(f.const_bio.get()));
+    ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[8], 10); // 'this fancy'
+    ASSERT_EQUAL(10, ret);
+    EXPECT_EQUAL(6, BIO_pending(f.const_bio.get()));
+    ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[18], 20); // ' data!' (with extra destination space available)
+    ASSERT_EQUAL(6, ret);
+    EXPECT_EQUAL(0, BIO_pending(f.const_bio.get()));
+
+    EXPECT_EQUAL(to_read, vespalib::stringref(f.tmp_buf.data(), to_read.size()));
+}
+
+TEST_F("Const BIO read EOF returns -1 by default and sets BIO retry flag", Fixture) {
+    ConstBufferViewGuard g(*f.const_bio, "", 0);
+    int ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[0], static_cast<int>(f.tmp_buf.size()));
+    EXPECT_EQUAL(-1, ret);
+    EXPECT_NOT_EQUAL(0, BIO_should_retry(f.const_bio.get()));
+}
+
+TEST_F("Can invoke BIO_(set|get)_close", Fixture) {
+    (void)BIO_set_close(f.mutable_bio.get(), 0);
+    EXPECT_EQUAL(0, BIO_get_close(f.mutable_bio.get()));
+    (void)BIO_set_close(f.mutable_bio.get(), 1);
+    EXPECT_EQUAL(1, BIO_get_close(f.mutable_bio.get()));
+}
+
+TEST_F("BIO_write on const BIO returns failure", Fixture) {
+    vespalib::string data = "safe and cozy data :3";
+    vespalib::string to_read = data;
+    ConstBufferViewGuard g(*f.const_bio, &to_read[0], to_read.size());
+
+    int ret = ::BIO_write(f.const_bio.get(), "unsafe", 6);
+    EXPECT_EQUAL(-1, ret);
+    EXPECT_EQUAL(0, BIO_should_retry(f.mutable_bio.get()));
+    EXPECT_EQUAL(data, to_read);
+}
+
+TEST_F("BIO_read on mutable BIO returns failure", Fixture) {
+    MutableBufferViewGuard g(*f.mutable_bio, &f.tmp_buf[0], f.tmp_buf.size());
+
+    vespalib::string dummy_buf;
+    dummy_buf.reserve(128);
+    int ret = ::BIO_read(f.mutable_bio.get(), &dummy_buf[0], static_cast<int>(dummy_buf.size()));
+    EXPECT_EQUAL(-1, ret);
+    EXPECT_EQUAL(0, BIO_should_retry(f.mutable_bio.get()));
+}
+
+TEST_F("Can do read on zero-length nullptr const buffer", Fixture) {
+    ConstBufferViewGuard g(*f.const_bio, nullptr, 0);
+    int ret = ::BIO_read(f.const_bio.get(), &f.tmp_buf[0], static_cast<int>(f.tmp_buf.size()));
+    EXPECT_EQUAL(-1, ret);
+    EXPECT_NOT_EQUAL(0, BIO_should_retry(f.const_bio.get()));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
+
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
index 254a9b213ba..3dd800dadf0 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -12,6 +12,9 @@
 #include <vespa/vespalib/data/smart_buffer.h>
 #include <assert.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP(".vespalib.net.crypto_engine");
+
 namespace vespalib {
 
 namespace {
@@ -172,6 +175,8 @@ CryptoEngine::SP create_default_crypto_engine() {
     if (cfg_file.empty()) {
         return std::make_shared<NullCryptoEngine>();
     }
+
+    LOG(debug, "Using TLS crypto engine with config file '%s'", cfg_file.c_str());
     auto tls_opts = net::tls::read_options_from_json_file(cfg_file);
     return std::make_shared<TlsCryptoEngine>(*tls_opts);
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/impl/CMakeLists.txt
index a5a8e8d3eb9..6129fde3c6c 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(vespalib_vespalib_net_tls_impl OBJECT
     SOURCES
+    direct_buffer_bio.cpp
     openssl_tls_context_impl.cpp
     openssl_crypto_codec_impl.cpp
     DEPENDS
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp
new file mode 100644
index 00000000000..8fe59e55bbd
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp
@@ -0,0 +1,418 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "direct_buffer_bio.h"
+#include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <utility>
+#include <cassert>
+
+#include <vespa/vespalib/util/backtrace.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio");
+
+/*
+ * The official OpenSSL docs are basically devoid of information on how to write
+ * your own BIOs, so most of the information used to implement our custom BIOs
+ * is gleaned from other implementations and by reading the OpenSSL source code.
+ *
+ * Primary references used for implementation:
+ *  - https://github.com/openssl/openssl/blob/master/crypto/bio/bss_mem.c
+ *  - https://github.com/indutny/uv_ssl_t/blob/master/src/bio.c
+ */
+
+namespace vespalib::net::tls::impl {
+
+namespace {
+
+int buffer_bio_init(::BIO* bio);
+int buffer_bio_destroy(::BIO* bio);
+int mutable_buffer_bio_write(::BIO* bio, const char* src_buf, int len);
+int const_buffer_bio_write(::BIO* bio, const char* src_buf, int len);
+int mutable_buffer_bio_read(::BIO* bio, char* dest_buf, int len);
+int const_buffer_bio_read(::BIO* bio, char* dest_buf, int len);
+long mutable_buffer_bio_ctrl(::BIO* bio, int cmd, long num, void* ptr);
+long const_buffer_bio_ctrl(::BIO* bio, int cmd, long num, void* ptr);
+
+// How to wrangle BIOs and their methods is completely changed after OpenSSL 1.1
+// For older versions, we must directly create a struct with callback fields set
+// and can access the BIO fields directly. In 1.1 and beyond everything is hidden
+// by indirection functions (these are _not_ available in prior versions).
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+
+#if !defined(BIO_TYPE_START)
+#  define BIO_TYPE_START 128 // Constant hoisted from OpenSSL >= 1.1.0
+#endif
+
+const ::BIO_METHOD mutable_buf_method_instance = {
+    (BIO_TYPE_START + 1) | BIO_TYPE_SOURCE_SINK, // BIO_TYPE_SOURCE_SINK sets high bits, not low bits, so no clobbering
+    "mutable direct buffer access BIO",
+    mutable_buffer_bio_write, // write func
+    mutable_buffer_bio_read, // read func
+    nullptr, // puts func
+    nullptr, // gets func
+    mutable_buffer_bio_ctrl, // ctrl func
+    buffer_bio_init, // init func
+    buffer_bio_destroy, // destroy func
+    nullptr, // callback ctrl func
+};
+
+const ::BIO_METHOD const_buf_method_instance = {
+    (BIO_TYPE_START + 2) | BIO_TYPE_SOURCE_SINK,
+    "const direct buffer access BIO",
+    const_buffer_bio_write, // write func
+    const_buffer_bio_read, // read func
+    nullptr, // puts func
+    nullptr, // gets func
+    const_buffer_bio_ctrl, // ctrl func
+    buffer_bio_init, // init func
+    buffer_bio_destroy, // destroy func
+    nullptr, // callback ctrl func
+};
+
+struct BioMethodWrapper {
+    const ::BIO_METHOD* method; // Global instance
+    int type_index;
+};
+
+BioMethodWrapper mutable_buf_method() {
+    return {&mutable_buf_method_instance, mutable_buf_method_instance.type};
+}
+
+BioMethodWrapper const_buf_method() {
+    return {&const_buf_method_instance, const_buf_method_instance.type};
+}
+
+void set_bio_data(::BIO& bio, void* ptr) {
+    bio.ptr = ptr;
+}
+
+void* get_bio_data(::BIO& bio) {
+    return bio.ptr;
+}
+
+void set_bio_shutdown(::BIO& bio, int shutdown) {
+    bio.shutdown = shutdown;
+}
+
+int get_bio_shutdown(::BIO& bio) {
+    return bio.shutdown;
+}
+
+void set_bio_init(::BIO& bio, int init) {
+    bio.init = init;
+}
+
+#else // OpenSSL 1.1 and beyond
+
+struct BioMethodDeleter {
+    void operator()(::BIO_METHOD* meth) const noexcept {
+        ::BIO_meth_free(meth);
+    }
+};
+using BioMethodPtr = std::unique_ptr<::BIO_METHOD, BioMethodDeleter>;
+
+struct BioMethodWrapper {
+    BioMethodPtr method;
+    int type_index;
+};
+
+struct BioMethodParams {
+    const char* bio_name;
+    int (*bio_write)(::BIO*, const char*, int);
+    int (*bio_read)(::BIO*, char*, int);
+    long (*bio_ctrl)(::BIO*, int, long, void*);
+};
+
+BioMethodWrapper create_bio_method(const BioMethodParams& params) {
+    int type_index = ::BIO_get_new_index() | BIO_TYPE_SOURCE_SINK;
+    if (type_index == -1) {
+        throw CryptoException("BIO_get_new_index");
+    }
+    BioMethodPtr bm(::BIO_meth_new(type_index, params.bio_name));
+    if (!::BIO_meth_set_create(bm.get(), buffer_bio_init) ||
+        !::BIO_meth_set_destroy(bm.get(), buffer_bio_destroy) ||
+        !::BIO_meth_set_write(bm.get(), params.bio_write) ||
+        !::BIO_meth_set_read(bm.get(), params.bio_read) ||
+        !::BIO_meth_set_ctrl(bm.get(), params.bio_ctrl)) {
+        throw CryptoException("Failed to set BIO_METHOD callback");
+    }
+    return {std::move(bm), type_index};
+}
+
+BioMethodWrapper create_mutable_bio_method() {
+    return create_bio_method({"mutable direct buffer access BIO", mutable_buffer_bio_write,
+                              mutable_buffer_bio_read, mutable_buffer_bio_ctrl});
+}
+
+BioMethodWrapper create_const_bio_method() {
+    return create_bio_method({"const direct buffer access BIO", const_buffer_bio_write,
+                              const_buffer_bio_read, const_buffer_bio_ctrl});
+}
+
+const BioMethodWrapper& mutable_buf_method() {
+    static BioMethodWrapper wrapper = create_mutable_bio_method();
+    return wrapper;
+}
+
+const BioMethodWrapper& const_buf_method() {
+    static BioMethodWrapper wrapper = create_const_bio_method();
+    return wrapper;
+}
+
+void set_bio_data(::BIO& bio, void* ptr) {
+    ::BIO_set_data(&bio, ptr);
+}
+
+void set_bio_shutdown(::BIO& bio, int shutdown) {
+    ::BIO_set_shutdown(&bio, shutdown);
+}
+
+int get_bio_shutdown(::BIO& bio) {
+    return ::BIO_get_shutdown(&bio);
+}
+
+void set_bio_init(::BIO& bio, int init) {
+    ::BIO_set_init(&bio, init);
+}
+
+void* get_bio_data(::BIO& bio) {
+    return ::BIO_get_data(&bio);
+}
+
+#endif
+
+BioPtr new_direct_buffer_bio(const ::BIO_METHOD& method) {
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+    auto* bio = ::BIO_new(const_cast<::BIO_METHOD*>(&method)); // ugh, older OpenSSL const-ness is a disaster.
+#else
+    auto* bio = ::BIO_new(&method);
+#endif
+    if (!bio) {
+        return BioPtr();
+    }
+    set_bio_data(*bio, nullptr); // Just to make sure this isn't set yet.
+    return BioPtr(bio);
+}
+
+} // anon ns
+
+BioPtr new_mutable_direct_buffer_bio() {
+    return new_direct_buffer_bio(*mutable_buf_method().method);
+}
+
+BioPtr new_const_direct_buffer_bio() {
+    return new_direct_buffer_bio(*const_buf_method().method);
+}
+
+namespace {
+
+int buffer_bio_init(::BIO* bio) {
+    // "shutdown" here means "should BIO close underlying resource?". Since
+    // our BIOs don't ever allocate anything we just use this value as something
+    // that can be set by BIO_set_close() and read by BIO_get_close().
+    set_bio_shutdown(*bio, 1);
+    set_bio_init(*bio, 1);
+    return 1;
+}
+
+int buffer_bio_destroy(::BIO* bio) {
+    set_bio_data(*bio, nullptr); // We don't own anything.
+    return 1;
+}
+
+int mutable_buffer_bio_write(::BIO* bio, const char* src_buf, int len) {
+    LOG_ASSERT(len >= 0);
+
+    BIO_clear_retry_flags(bio);
+    if (!get_bio_data(*bio)) {
+        // TODO replace with assertion once we _know_ it should never happen in practice..!
+        LOG(error, "Got buffer write of length %d to a non-bound mutable BIO!", len);
+        LOG(error, "%s", getStackTrace(0).c_str());
+        return -1;
+    }
+
+    const auto sz_len = static_cast<size_t>(len);
+    if (sz_len == 0) {
+        return 0;
+    }
+    auto* dest_buf = static_cast<MutableBufferView*>(get_bio_data(*bio));
+    // sz_len is <= INT32_MAX while pos/size are size_t, so no overflow on 64-bit
+    // since the caller enforces that buffer sizes are < INT32_MAX.
+    if (dest_buf->pos + sz_len > dest_buf->size) {
+        return -1;
+    }
+    // Source and destination buffers should never overlap.
+    memcpy(dest_buf->buffer + dest_buf->pos, src_buf, sz_len);
+    dest_buf->pos += sz_len;
+
+    return len;
+}
+
+int const_buffer_bio_write(::BIO* bio, const char* src_buf, int len) {
+    (void) bio;
+    (void) src_buf;
+    // Const buffers are read only!
+    LOG(error, "BIO_write() of length %d called on read-only BIO", len);
+    return -1;
+}
+
+int mutable_buffer_bio_read(::BIO* bio, char* dest_buf, int len) {
+    (void) bio;
+    (void) dest_buf;
+    // Mutable buffers are write only!
+    LOG(error, "BIO_read() of length %d called on write-only BIO", len);
+    return -1;
+}
+
+int const_buffer_bio_read(::BIO* bio, char* dest_buf, int len) {
+    LOG_ASSERT(len >= 0);
+
+    BIO_clear_retry_flags(bio);
+    if (!get_bio_data(*bio)) {
+        // TODO replace with assertion once we _know_ it should never happen in practice..!
+        LOG(error, "Got buffer read of length %d to a non-bound const BIO!", len);
+        LOG(error, "%s", getStackTrace(0).c_str());
+        return -1;
+    }
+
+    const auto sz_len = static_cast<size_t>(len);
+    auto* src_buf = static_cast<ConstBufferView*>(get_bio_data(*bio));
+    const auto readable = std::min(sz_len, src_buf->size - src_buf->pos);
+    if (readable != 0) {
+        // Source and destination buffers should never overlap.
+        memcpy(dest_buf, src_buf->buffer + src_buf->pos, readable);
+        src_buf->pos += readable;
+        return static_cast<int>(readable);
+    }
+    // Since a BIO might point to different buffers between SSL_* invocations,
+    // we want OpenSSL to retry later. _Not_ setting this or not returning -1 will
+    // cause OpenSSL to return SSL_ERROR_SYSCALL. Ask me how I know.
+    BIO_set_retry_read(bio);
+    return -1;
+}
+
+template <typename BufferType>
+long do_buffer_bio_ctrl(::BIO* bio, int cmd, long num, void* ptr) {
+    const auto* buf_view = static_cast<const BufferType*>(get_bio_data(*bio));
+    long ret = 1;
+
+    switch (cmd) {
+    case BIO_CTRL_EOF: // Is the buffer exhausted?
+        if (buf_view != nullptr) {
+            ret = static_cast<int>(buf_view->pos == buf_view->size);
+        }
+        break;
+    case BIO_CTRL_INFO: // How much data remains in buffer?
+        ret = (buf_view != nullptr) ? buf_view->pending() : 0;
+        if (ptr) {
+            *static_cast<void**>(ptr) = nullptr; // Semantics: who knows? But everyone's doing it!
+        }
+        break;
+    case BIO_CTRL_GET_CLOSE: // Is the BIO in auto close mode?
+        ret = get_bio_shutdown(*bio);
+        break;
+    case BIO_CTRL_SET_CLOSE: // Should the BIO be in auto close mode? Spoiler alert: we don't really care.
+        set_bio_shutdown(*bio, static_cast<int>(num));
+        break;
+    case BIO_CTRL_WPENDING:
+        ret = 0;
+        break;
+    case BIO_CTRL_PENDING:
+        ret = (buf_view != nullptr) ? buf_view->pending() : 0;
+        break;
+    case BIO_CTRL_DUP:
+    case BIO_CTRL_FLUSH:
+        ret = 1; // Same as memory OpenSSL BIO ctrl func.
+        break;
+    case BIO_CTRL_RESET:
+    case BIO_C_SET_BUF_MEM:
+    case BIO_C_GET_BUF_MEM_PTR:
+    case BIO_C_SET_BUF_MEM_EOF_RETURN:
+        LOG_ASSERT(!"Unsupported BIO control function called");
+    case BIO_CTRL_PUSH:
+    case BIO_CTRL_POP:
+    default:
+        ret = 0; // Not supported (but be gentle, since it's actually invoked)
+        break;
+    }
+    return ret;
+}
+
+long mutable_buffer_bio_ctrl(::BIO* bio, int cmd, long num, void* ptr) {
+    return do_buffer_bio_ctrl<MutableBufferView>(bio, cmd, num, ptr);
+}
+
+long const_buffer_bio_ctrl(::BIO* bio, int cmd, long num, void* ptr) {
+    return do_buffer_bio_ctrl<ConstBufferView>(bio, cmd, num, ptr);
+}
+
+MutableBufferView mutable_buffer_view_of(char* buffer, size_t sz) {
+    return {buffer, sz, 0, 0};
+}
+
+ConstBufferView const_buffer_view_of(const char* buffer, size_t sz) {
+    return {buffer, sz, 0};
+}
+
+[[maybe_unused]] bool is_const_bio(::BIO& bio) noexcept {
+    return (::BIO_method_type(&bio) == const_buf_method().type_index);
+}
+
+[[maybe_unused]] bool is_mutable_bio(::BIO& bio) noexcept {
+    return (::BIO_method_type(&bio) == mutable_buf_method().type_index);
+}
+
+// There is a cute little bug in BIO_meth_new() present in v1.1.0h which
+// causes the provided BIO method type to not be actually written into the
+// target BIO_METHOD instance. This means that any assertions that check the
+// BIO's method type on this version is doomed to fail.
+// See https://github.com/openssl/openssl/pull/5812
+#if ((OPENSSL_VERSION_NUMBER & 0xfffffff0L) != 0x10100080L)
+#  define WHEN_NO_OPENSSL_BIO_TYPE_BUG(expr) expr
+#else
+#  define WHEN_NO_OPENSSL_BIO_TYPE_BUG(expr)
+#endif
+
+void set_bio_mutable_buffer_view(::BIO& bio, MutableBufferView* view) {
+    WHEN_NO_OPENSSL_BIO_TYPE_BUG(LOG_ASSERT(is_mutable_bio(bio)));
+    set_bio_data(bio, view);
+}
+
+void set_bio_const_buffer_view(::BIO& bio, ConstBufferView* view) {
+    WHEN_NO_OPENSSL_BIO_TYPE_BUG(LOG_ASSERT(is_const_bio(bio)));
+    set_bio_data(bio, view);
+}
+
+// Precondition: bio must have been created by a call to either
+// new_mutable_direct_buffer_bio() or new_const_direct_buffer_bio()
+void unset_bio_buffer_view(::BIO& bio) {
+    WHEN_NO_OPENSSL_BIO_TYPE_BUG(LOG_ASSERT(is_mutable_bio(bio) || is_const_bio(bio)));
+    set_bio_data(bio, nullptr);
+}
+
+} // anon ns
+
+ConstBufferViewGuard::ConstBufferViewGuard(::BIO& bio, const char* buffer, size_t sz) noexcept
+    : _bio(bio),
+      _view(const_buffer_view_of(buffer, sz))
+{
+    WHEN_NO_OPENSSL_BIO_TYPE_BUG(LOG_ASSERT(is_const_bio(bio)));
+    set_bio_const_buffer_view(bio, &_view);
+}
+
+ConstBufferViewGuard::~ConstBufferViewGuard() {
+    unset_bio_buffer_view(_bio);
+}
+
+MutableBufferViewGuard::MutableBufferViewGuard(::BIO& bio, char* buffer, size_t sz) noexcept
+    : _bio(bio),
+      _view(mutable_buffer_view_of(buffer, sz))
+{
+    WHEN_NO_OPENSSL_BIO_TYPE_BUG(LOG_ASSERT(is_mutable_bio(bio)));
+    set_bio_mutable_buffer_view(bio, &_view);
+}
+
+MutableBufferViewGuard::~MutableBufferViewGuard() {
+    unset_bio_buffer_view(_bio);
+}
+
+}
\ No newline at end of file
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h
new file mode 100644
index 00000000000..581d43d6f29
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h
@@ -0,0 +1,90 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "openssl_typedefs.h"
+#include <openssl/bio.h>
+
+/*
+ * Custom BIO implementations which offer direct write/read only buffer
+ * access to underlying memory buffers. This removes the need to allocate
+ * separate memory BIOs into/from which data is redundantly copied.
+ *
+ * These BIOs are merely views into buffers that the user must set appropriately
+ * before invoking OpenSSL functions that invoke them. The ability to set buffers
+ * is only available via scoped guards that cannot be copied or moved.
+ *
+ * Since no buffer allocation is ever done by these BIOs, it is the responsibility
+ * of the caller to provide sufficiently large buffers that OpenSSL operations can
+ * make progress.
+ *
+ * The BIOs ensure that OpenSSL cannot write to read-only buffers and vice versa.
+ */
+
+namespace vespalib::net::tls::impl {
+
+BioPtr new_mutable_direct_buffer_bio();
+BioPtr new_const_direct_buffer_bio();
+
+struct MutableBufferView {
+    // Could use a pointer pair instead (or just modify the ptr), but being explicit is good for readability.
+    char* buffer;
+    size_t size;
+    size_t pos;
+    size_t rpos;
+
+    // Pending means "how much is written"
+    size_t pending() const noexcept {
+        return pos;
+    }
+};
+
+struct ConstBufferView {
+    const char* buffer;
+    size_t size;
+    size_t pos;
+
+    // Pending means "how much is left to read"
+    size_t pending() const noexcept {
+        return size - pos;
+    }
+};
+
+class ConstBufferViewGuard {
+    ::BIO& _bio;
+    ConstBufferView _view;
+public:
+    // Important: buffer view pointer and the buffer it points to MUST be
+    // valid until unset_bio_buffer_view is called! Exception to the latter is
+    // if the data buffer length is 0 AND the data buffer pointer is nullptr.
+    // Precondition: bio must have been created by a call to new_const_direct_buffer_bio()
+    ConstBufferViewGuard(::BIO& bio, const char* buffer, size_t sz) noexcept;
+    ~ConstBufferViewGuard();
+
+    // The current active buffer view has a reference into our own struct, so
+    // we cannot allow that pointer to be invalidated by copies or moves.
+    ConstBufferViewGuard(const ConstBufferViewGuard&) = delete;
+    ConstBufferViewGuard& operator=(const ConstBufferViewGuard&) = delete;
+    ConstBufferViewGuard(ConstBufferViewGuard&&) = delete;
+    ConstBufferViewGuard& operator=(ConstBufferViewGuard&&) = delete;
+};
+
+class MutableBufferViewGuard {
+    ::BIO& _bio;
+    MutableBufferView _view;
+public:
+    // Important: buffer view pointer and the buffer it points to MUST be
+    // valid until unset_bio_buffer_view is called! Exception to the latter is
+    // if the data buffer length is 0 AND the data buffer pointer is nullptr.
+    // Precondition: bio must have been created by a call to new_mutable_direct_buffer_bio()
+    MutableBufferViewGuard(::BIO& bio, char* buffer, size_t sz) noexcept;
+    ~MutableBufferViewGuard();
+
+    // The current active buffer view has a reference into our own struct, so
+    // we cannot allow that pointer to be invalidated by copies or moves.
+    MutableBufferViewGuard(const MutableBufferViewGuard&) = delete;
+    MutableBufferViewGuard& operator=(const MutableBufferViewGuard&) = delete;
+    MutableBufferViewGuard(MutableBufferViewGuard&&) = delete;
+    MutableBufferViewGuard& operator=(MutableBufferViewGuard&&) = delete;
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
index a563a43baac..d1cddcfaf8c 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "openssl_crypto_codec_impl.h"
 #include "openssl_tls_context_impl.h"
+#include "direct_buffer_bio.h"
 #include <vespa/vespalib/net/tls/crypto_codec.h>
 #include <vespa/vespalib/net/tls/crypto_exception.h>
 #include <mutex>
@@ -80,10 +81,6 @@ HandshakeResult handshake_consumed_bytes_and_needs_more_peer_data(size_t consume
     return {consumed, 0, HandshakeResult::State::NeedsMorePeerData};
 }
 
-HandshakeResult handshake_produced_bytes_and_needs_more_peer_data(size_t produced) noexcept {
-    return {0, produced, HandshakeResult::State::NeedsMorePeerData};
-}
-
 HandshakeResult handshake_consumed_bytes_and_is_complete(size_t consumed) noexcept {
     return {consumed, 0, HandshakeResult::State::Done};
 }
@@ -124,12 +121,19 @@ DecodeResult decoded_bytes(size_t consumed, size_t produced, DecodeResult::State
     return {consumed, produced, state};
 }
 
-BioPtr new_tls_frame_memory_bio() {
-    BioPtr bio(::BIO_new(BIO_s_mem()));
+BioPtr new_tls_frame_mutable_memory_bio() {
+    BioPtr bio(new_mutable_direct_buffer_bio());
     if (!bio) {
-        throw CryptoException("IO_new(BIO_s_mem()) failed; out of memory?");
+        throw CryptoException("new_mutable_direct_buffer_bio() failed; out of memory?");
+    }
+    return bio;
+}
+
+BioPtr new_tls_frame_const_memory_bio() {
+    BioPtr bio(new_const_direct_buffer_bio());
+    if (!bio) {
+        throw CryptoException("new_const_direct_buffer_bio() failed; out of memory?");
     }
-    BIO_set_write_buf_size(bio.get(), 0); // 0 ==> default max frame size
     return bio;
 }
 
@@ -148,10 +152,6 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(::SSL_CTX& ctx, Mode mode)
      * a bit more straight forward to implement a full duplex API with two
      * separate BIOs, but there is little available documentation as to the
      * 'hows' and 'whys' around this.
-     * There are claims from core OpenSSL devs[0] that BIO pairs are more efficient,
-     * so we may reconsider the current approach (or just use the "OpenSSL controls
-     * the file descriptor" yolo approach for simplicity, assuming they do optimal
-     * stuff internally).
      *
      * Our BIOs are used as follows:
      *
@@ -167,14 +167,16 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(::SSL_CTX& ctx, Mode mode)
      *  encode() : SSL_write(plaintext) --(_output_bio ciphertext)--> BIO_read  --> [peer]
      *  decode() : SSL_read(plaintext) <--(_input_bio ciphertext)--   BIO_write <-- [peer]
      *
-     * To avoid blowing the sizes of BIOs out of the water, we do our best to encode and decode
-     * on a per-TLS frame granularity (16K) maximum.
      */
-    BioPtr tmp_input_bio  = new_tls_frame_memory_bio();
-    BioPtr tmp_output_bio = new_tls_frame_memory_bio();
-    // Connect BIOs used internally by OpenSSL. This transfers ownership. No return value to check.
-    // TODO replace with explicit SSL_set0_rbio/SSL_set0_wbio on OpenSSL >= v1.1
+    BioPtr tmp_input_bio  = new_tls_frame_const_memory_bio();
+    BioPtr tmp_output_bio = new_tls_frame_mutable_memory_bio();
+    // Connect BIOs used internally by OpenSSL. This transfers ownership. No return values to check.
+#if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+    ::SSL_set0_rbio(_ssl.get(), tmp_input_bio.get());
+    ::SSL_set0_wbio(_ssl.get(), tmp_output_bio.get());
+#else
     ::SSL_set_bio(_ssl.get(), tmp_input_bio.get(), tmp_output_bio.get());
+#endif
     _input_bio  = tmp_input_bio.release();
     _output_bio = tmp_output_bio.release();
     if (_mode == Mode::Client) {
@@ -186,22 +188,6 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(::SSL_CTX& ctx, Mode mode)
 
 // TODO remove spammy logging once code is stable
 
-// Produces bytes previously written to _output_bio by SSL_do_handshake or SSL_write
-int OpenSslCryptoCodecImpl::drain_outgoing_network_bytes_if_any(
-        char *to_peer, size_t to_peer_buf_size) noexcept {
-    int out_pending = BIO_pending(_output_bio);
-    if (out_pending > 0) {
-        int copied = ::BIO_read(_output_bio, to_peer, static_cast<int>(to_peer_buf_size));
-        // TODO BIO_should_retry here? Semantics are unclear, especially for memory BIOs.
-        LOG(spam, "BIO_read copied out %d bytes of ciphertext from _output_bio", copied);
-        if (copied < 0) {
-            LOG(error, "Memory BIO_read() failed with BIO_pending() > 0");
-        }
-        return copied;
-    }
-    return out_pending;
-}
-
 HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t from_peer_buf_size,
                                                   char* to_peer, size_t to_peer_buf_size) noexcept {
     LOG_ASSERT(verify_buf(from_peer, from_peer_buf_size) && verify_buf(to_peer, to_peer_buf_size));
@@ -209,75 +195,53 @@ HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t
     if (SSL_is_init_finished(_ssl.get())) {
         return handshake_completed();
     }
-    // Still ciphertext data left? If so, get rid of it before we start a new operation
-    // that wants to fill the output BIO.
-    int produced = drain_outgoing_network_bytes_if_any(to_peer, to_peer_buf_size);
-    if (produced > 0) {
-        // Handshake isn't complete yet and we've got stuff to send. Need to continue handshake
-        // once more data is available from the peer.
-        return handshake_produced_bytes_and_needs_more_peer_data(static_cast<size_t>(produced));
-    } else if (produced < 0) {
-        return handshake_failed();
-    }
-    const auto consume_res = do_handshake_and_consume_peer_input_bytes(from_peer, from_peer_buf_size);
+    ConstBufferViewGuard const_view_guard(*_input_bio, from_peer, from_peer_buf_size);
+    MutableBufferViewGuard mut_view_guard(*_output_bio, to_peer, to_peer_buf_size);
+
+    const auto consume_res = do_handshake_and_consume_peer_input_bytes();
     LOG_ASSERT(consume_res.bytes_produced == 0);
     if (consume_res.failed()) {
         return consume_res;
     }
     // SSL_do_handshake() might have produced more data to send. Note: handshake may
     // be complete at this point.
-    produced = drain_outgoing_network_bytes_if_any(to_peer, to_peer_buf_size);
-    if (produced < 0) {
-        return handshake_failed();
-    }
+    int produced = BIO_pending(_output_bio);
     return handshaked_bytes(consume_res.bytes_consumed, static_cast<size_t>(produced), consume_res.state);
 }
 
-HandshakeResult OpenSslCryptoCodecImpl::do_handshake_and_consume_peer_input_bytes(
-        const char *from_peer, size_t from_peer_buf_size) noexcept {
-    // Feed the SSL session input in frame-sized chunks between each call to SSL_do_handshake().
-    // This is primarily to ensure we don't shove unbounded amounts of data into the BIO
-    // in the case that someone naughty is sending us tons of garbage over the socket.
-    size_t consumed_total = 0;
-    while (true) {
-        // Assumption: SSL_do_handshake will place all required outgoing handshake
-        // data in the output memory BIO without requiring WANT_WRITE. Freestanding
-        // memory BIOs are _supposedly_ auto-resizing, so this should work transparently.
-        // At the very least, if this is not the case we'll auto-fail the connection
-        // and quickly find out..!
-        // TODO test multi-frame sized handshake
-        // TODO should we invoke ::ERR_clear_error() prior?
-        int ssl_result = ::SSL_do_handshake(_ssl.get());
-        ssl_result = ::SSL_get_error(_ssl.get(), ssl_result);
-
-        if (ssl_result == SSL_ERROR_WANT_READ) {
-            LOG(spam, "SSL_do_handshake() returned SSL_ERROR_WANT_READ");
-            if (from_peer_buf_size - consumed_total > 0) {
-                int consumed = ::BIO_write(_input_bio, from_peer + consumed_total,
-                                           static_cast<int>(std::min(MaximumTlsFrameSize, from_peer_buf_size - consumed_total)));
-                LOG(spam, "BIO_write copied in %d bytes of ciphertext to _input_bio", consumed);
-                if (consumed < 0) {
-                    LOG(error, "Memory BIO_write() returned %d", consumed); // TODO BIO_need_retry?
-                    return handshake_failed();
-                }
-                consumed_total += consumed; // TODO protect against consumed == 0?
-                continue;
-            } else {
-                return handshake_consumed_bytes_and_needs_more_peer_data(consumed_total);
-            }
-        } else if (ssl_result == SSL_ERROR_NONE) {
-            // At this point SSL_do_handshake has stated it does not need any more peer data, i.e.
-            // the handshake is complete.
-            if (!SSL_is_init_finished(_ssl.get())) {
-                LOG(error, "SSL handshake is not completed even though no more peer data is requested");
-                return handshake_failed();
-            }
-            return handshake_consumed_bytes_and_is_complete(consumed_total);
-        } else {
-            LOG(error, "SSL_do_handshake() returned unexpected error: %s", ssl_error_to_str(ssl_result));
+HandshakeResult OpenSslCryptoCodecImpl::do_handshake_and_consume_peer_input_bytes() noexcept {
+    // Assumption: SSL_do_handshake will place all required outgoing handshake
+    // data in the output memory BIO without requiring WANT_WRITE.
+    // TODO test multi-frame sized handshake
+    const long pending_read_before = BIO_pending(_input_bio);
+
+    ::ERR_clear_error();
+    int ssl_result = ::SSL_do_handshake(_ssl.get());
+    ssl_result = ::SSL_get_error(_ssl.get(), ssl_result);
+
+    const long consumed = pending_read_before - BIO_pending(_input_bio);
+    LOG_ASSERT(consumed >= 0);
+
+    if (ssl_result == SSL_ERROR_WANT_READ) {
+        LOG(spam, "SSL_do_handshake() returned SSL_ERROR_WANT_READ");
+
+        return handshake_consumed_bytes_and_needs_more_peer_data(static_cast<size_t>(consumed));
+    } else if (ssl_result == SSL_ERROR_NONE) {
+        // At this point SSL_do_handshake has stated it does not need any more peer data, i.e.
+        // the handshake is complete.
+        if (!SSL_is_init_finished(_ssl.get())) {
+            LOG(error, "SSL handshake is not completed even though no more peer data is requested");
             return handshake_failed();
         }
-    };
+        LOG(debug, "SSL_do_handshake() is complete, using protocol %s", SSL_get_version(_ssl.get()));
+        return handshake_consumed_bytes_and_is_complete(static_cast<size_t>(consumed));
+    } else {
+        LOG(error, "SSL_do_handshake() returned unexpected error: %s", ssl_error_to_str(ssl_result));
+        char buf[256];
+        ERR_error_string_n(ERR_get_error(), buf, sizeof(buf));
+        LOG(error, "%s", buf);
+        return handshake_failed();
+    }
 }
 
 EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaintext_size,
@@ -288,13 +252,16 @@ EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaint
         LOG(error, "OpenSslCryptoCodecImpl::encode() called before handshake completed");
         return encode_failed();
     }
+
+    MutableBufferViewGuard mut_view_guard(*_output_bio, ciphertext, ciphertext_size);
+    // _input_bio not read from here.
+
     size_t bytes_consumed = 0;
     if (plaintext_size != 0) {
         int to_consume = static_cast<int>(std::min(plaintext_size, MaximumFramePlaintextSize));
         // SSL_write encodes plaintext to ciphertext and writes to _output_bio
         int consumed = ::SSL_write(_ssl.get(), plaintext, to_consume);
-        LOG(spam, "After SSL_write() -> %d, _input_bio pending=%d, _output_bio pending=%d",
-            consumed, BIO_pending(_input_bio), BIO_pending(_output_bio));
+        LOG(spam, "After SSL_write() -> %d _output_bio pending=%d", consumed, BIO_pending(_output_bio));
         if (consumed < 0) {
             int ssl_error = ::SSL_get_error(_ssl.get(), consumed);
             LOG(error, "SSL_write() failed to write frame, got error %s", ssl_error_to_str(ssl_error));
@@ -306,15 +273,7 @@ EncodeResult OpenSslCryptoCodecImpl::encode(const char* plaintext, size_t plaint
         }
         bytes_consumed = static_cast<size_t>(consumed);
     }
-
-    int produced = drain_outgoing_network_bytes_if_any(ciphertext, ciphertext_size);
-    if (produced < 0) {
-        return encode_failed();
-    }
-    if (BIO_pending(_output_bio) != 0) {
-        LOG(error, "Residual data left in output BIO on encode(); provided buffer is too small");
-        return encode_failed();
-    }
+    int produced = BIO_pending(_output_bio);
     return encoded_bytes(bytes_consumed, static_cast<size_t>(produced));
 }
 DecodeResult OpenSslCryptoCodecImpl::decode(const char* ciphertext, size_t ciphertext_size,
@@ -325,15 +284,18 @@ DecodeResult OpenSslCryptoCodecImpl::decode(const char* ciphertext, size_t ciphe
         LOG(error, "OpenSslCryptoCodecImpl::decode() called before handshake completed");
         return decode_failed();
     }
+    ConstBufferViewGuard const_view_guard(*_input_bio, ciphertext, ciphertext_size);
+    // _output_bio not written to here
+
+    const int input_pending_before = BIO_pending(_input_bio);
     auto produce_res = drain_and_produce_plaintext_from_ssl(plaintext, static_cast<int>(plaintext_size));
-    if ((produce_res.bytes_produced > 0) || produce_res.failed()) {
-        return produce_res; // TODO gRPC [1] handles this differently... allows fallthrough
-    }
-    int consumed = consume_peer_input_bytes(ciphertext, ciphertext_size);
-    if (consumed < 0) {
-        return decode_failed();
-    }
-    produce_res = drain_and_produce_plaintext_from_ssl(plaintext, static_cast<int>(plaintext_size));
+    const int input_pending_after = BIO_pending(_input_bio);
+
+    LOG_ASSERT(input_pending_before >= input_pending_after);
+    const int consumed = input_pending_before - input_pending_after;
+    LOG(spam, "decode: consumed %d bytes (ciphertext buffer %d -> %d bytes), produced %zu bytes. Need read: %s",
+        consumed, input_pending_before, input_pending_after, produce_res.bytes_produced,
+        (produce_res.state == DecodeResult::State::NeedsMorePeerData) ? "yes" : "no");
     return decoded_bytes(static_cast<size_t>(consumed), produce_res.bytes_produced, produce_res.state);
 }
 
@@ -345,37 +307,24 @@ DecodeResult OpenSslCryptoCodecImpl::drain_and_produce_plaintext_from_ssl(
     // depending on how much TLS frame data is available and its size relative
     // to the receiving plaintext buffer.
     int produced = ::SSL_read(_ssl.get(), plaintext, static_cast<int>(plaintext_size));
-    LOG(spam, "After SSL_read() -> %d, _input_bio pending=%d, _output_bio pending=%d",
-        produced, BIO_pending(_input_bio), BIO_pending(_output_bio));
     if (produced > 0) {
         // At least 1 frame decoded successfully.
         return decoded_frames_with_plaintext_bytes(static_cast<size_t>(produced));
     } else {
         int ssl_error = ::SSL_get_error(_ssl.get(), produced);
         switch (ssl_error) {
-            case SSL_ERROR_WANT_READ:
-                // SSL_read() was not able to decode a full frame with the ciphertext that
-                // we've fed it thus far; caller must feed it some and then try again.
-                LOG(spam, "SSL_read() returned SSL_ERROR_WANT_READ, must get more ciphertext");
-                return decode_needs_more_peer_data();
-            default:
-                LOG(error, "SSL_read() returned unexpected error: %s", ssl_error_to_str(ssl_error));
-                return decode_failed();
+        case SSL_ERROR_WANT_READ:
+            // SSL_read() was not able to decode a full frame with the ciphertext that
+            // we've fed it thus far; caller must feed it some and then try again.
+            LOG(spam, "SSL_read() returned SSL_ERROR_WANT_READ, must get more ciphertext");
+            return decode_needs_more_peer_data();
+        default:
+            LOG(error, "SSL_read() returned unexpected error: %s", ssl_error_to_str(ssl_error));
+            return decode_failed();
         }
     }
 }
 
-int OpenSslCryptoCodecImpl::consume_peer_input_bytes(
-        const char* ciphertext, size_t ciphertext_size) noexcept {
-    // TODO BIO_need_retry on failure? Can this even happen for memory BIOs?
-    int consumed = ::BIO_write(_input_bio, ciphertext, static_cast<int>(std::min(MaximumTlsFrameSize, ciphertext_size)));
-    LOG(spam, "BIO_write copied in %d bytes of ciphertext to _input_bio", consumed);
-    if (consumed < 0) {
-        LOG(error, "Memory BIO_write() returned %d", consumed);
-    }
-    return consumed;
-}
-
 }
 
 // External references:
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
index 44ca8859596..4c253fdf24c 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
@@ -55,21 +55,7 @@ public:
     DecodeResult decode(const char* ciphertext, size_t ciphertext_size,
                         char* plaintext, size_t plaintext_size) noexcept override;
 private:
-    /*
-     * Returns
-     *   n > 0 if n bytes written to `to_peer`. Always <= to_peer_buf_size
-     *   n == 0 if no bytes pending in output BIO
-     *   n < 0 on error
-     */
-    int drain_outgoing_network_bytes_if_any(char *to_peer, size_t to_peer_buf_size) noexcept;
-    /*
-     * Returns
-     *   n > 0 if n bytes written to `ciphertext`. Always <= ciphertext_size
-     *   n == 0 if no bytes pending in input BIO
-     *   n < 0 on error
-     */
-    int consume_peer_input_bytes(const char* ciphertext, size_t ciphertext_size) noexcept;
-    HandshakeResult do_handshake_and_consume_peer_input_bytes(const char *from_peer, size_t from_peer_buf_size) noexcept;
+    HandshakeResult do_handshake_and_consume_peer_input_bytes() noexcept;
     DecodeResult drain_and_produce_plaintext_from_ssl(char* plaintext, size_t plaintext_size) noexcept;
 };