first version of high-level c++ socket crypto APIs

includes fall-back implementation for non-encrypted communications and
a very simple xor encryption implementation for testing and example
purposes.

5 files changed, 314 insertions, 0 deletions

diff --git a/vespalib/src/vespa/vespalib/net/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/CMakeLists.txt
index f652be53560..2e0d15382d0 100644
--- a/vespalib/src/vespa/vespalib/net/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/CMakeLists.txt
@@ -2,6 +2,8 @@
 vespa_add_library(vespalib_vespalib_net OBJECT
     SOURCES
     async_resolver.cpp
+    crypto_engine.cpp
+    crypto_socket.cpp
     selector.cpp
     server_socket.cpp
     socket.cpp
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
new file mode 100644
index 00000000000..78499d6d18e
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -0,0 +1,174 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "crypto_engine.h"
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <vespa/vespalib/xxhash/xxhash.h>
+#include <assert.h>
+
+namespace vespalib {
+
+namespace {
+
+struct HashState {
+    using clock = std::chrono::high_resolution_clock;
+    const void       *self;
+    clock::time_point now;
+    HashState() : self(this), now(clock::now()) {}
+};
+
+char gen_key() {
+    HashState hash_state;
+    std::this_thread::sleep_for(std::chrono::microseconds(42));
+    return XXH64(&hash_state, sizeof(hash_state), 0);
+}
+
+class NullCryptoSocket : public CryptoSocket
+{
+private:
+    SocketHandle _socket;
+public:
+    NullCryptoSocket(SocketHandle socket) : _socket(std::move(socket)) {}
+    int get_fd() const override { return _socket.get(); }
+    HandshakeResult handshake() override { return HandshakeResult::DONE; }
+    size_t min_read_buffer_size() const override { return 1; }
+    ssize_t read(char *buf, size_t len) override { return _socket.read(buf, len); }
+    ssize_t drain(char *, size_t) override { return 0; }
+    ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); }
+    ssize_t flush() override { return 0; }
+};
+
+class XorCryptoSocket : public CryptoSocket
+{
+private:
+    static constexpr size_t CHUNK_SIZE = 4096;
+    enum class OP { READ_KEY, WRITE_KEY };
+    std::vector<OP> _op_stack;
+    char              _my_key;
+    char              _peer_key;
+    std::vector<char> _readbuf;
+    std::vector<char> _writebuf;    
+    SocketHandle      _socket;
+
+    bool is_blocked(ssize_t res, int error) const {
+        return ((res < 0) && ((error == EWOULDBLOCK) || (error == EAGAIN)));
+    }
+
+    HandshakeResult try_read_key() {
+        ssize_t res = _socket.read(&_peer_key, 1);
+        if (is_blocked(res, errno)) {
+            return HandshakeResult::NEED_READ;
+        }
+        return (res == 1)
+            ? HandshakeResult::DONE
+            : HandshakeResult::FAIL;
+    }
+
+    HandshakeResult try_write_key() {
+        ssize_t res = _socket.write(&_my_key, 1);
+        if (is_blocked(res, errno)) {
+            return HandshakeResult::NEED_WRITE;
+        }
+        return (res == 1)
+            ? HandshakeResult::DONE
+            : HandshakeResult::FAIL;
+    }
+
+    HandshakeResult perform_hs_op(OP op) {
+        if (op == OP::READ_KEY) {
+            return try_read_key();
+        } else {
+            assert(op == OP::WRITE_KEY);
+            return try_write_key();
+        }
+    }
+
+public:
+    XorCryptoSocket(SocketHandle socket, bool is_server)
+        : _op_stack(is_server
+                    ? std::vector<OP>({OP::WRITE_KEY, OP::READ_KEY})
+                    : std::vector<OP>({OP::READ_KEY, OP::WRITE_KEY})),
+          _my_key(gen_key()),
+          _peer_key(0),
+          _readbuf(),
+          _writebuf(),
+          _socket(std::move(socket)) {}
+    int get_fd() const override { return _socket.get(); }
+    HandshakeResult handshake() override {
+        while (!_op_stack.empty()) {
+            HandshakeResult partial_result = perform_hs_op(_op_stack.back());
+            if (partial_result != HandshakeResult::DONE) {
+                return partial_result;
+            }
+            _op_stack.pop_back();
+        }
+        return HandshakeResult::DONE;
+    }
+    size_t min_read_buffer_size() const override { return 1; }
+    ssize_t read(char *buf, size_t len) override {
+        if (_readbuf.empty()) {
+            _readbuf.resize(CHUNK_SIZE);
+            ssize_t res = _socket.read(&_readbuf[0], _readbuf.size());
+            if (res > 0) {
+                _readbuf.resize(res);
+            } else {
+                _readbuf.clear();
+                return res;
+            }
+        }
+        return drain(buf, len);
+    }
+    ssize_t drain(char *buf, size_t len) override {
+        size_t frame = std::min(len, _readbuf.size());
+        for (size_t i = 0; i < frame; ++i) {
+            buf[i] = (_readbuf[i] ^ _my_key);
+        }
+        _readbuf.erase(_readbuf.begin(), _readbuf.begin() + frame);
+        return frame;
+    }
+    ssize_t write(const char *buf, size_t len) override {
+        ssize_t res = flush();
+        while (res > 0) {
+            res = flush();
+        }
+        if (res < 0) {
+            return res;
+        }
+        size_t frame = std::min(len, CHUNK_SIZE);
+        for (size_t i = 0; i < frame; ++i) {
+            _writebuf.push_back(buf[i] ^ _peer_key);
+        }
+        return frame;
+    }
+    ssize_t flush() override {
+        if (!_writebuf.empty()) {
+            ssize_t res = _socket.write(&_writebuf[0], _writebuf.size());
+            if (res > 0) {
+                _writebuf.erase(_writebuf.begin(), _writebuf.begin() + res);                
+            } else {
+                assert(res < 0);
+            }
+            return res;
+        }
+        return 0;
+    }
+};
+
+}
+
+CryptoEngine::~CryptoEngine() = default;
+
+CryptoSocket::UP
+NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool)
+{
+    return std::make_unique<NullCryptoSocket>(std::move(socket));
+}
+
+CryptoSocket::UP
+XorCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
+{
+    return std::make_unique<XorCryptoSocket>(std::move(socket), is_server);
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h
new file mode 100644
index 00000000000..0f491838a6a
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h
@@ -0,0 +1,39 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "socket_handle.h"
+#include "crypto_socket.h"
+#include <memory>
+
+namespace vespalib {
+
+/**
+ * Component responsible for wrapping low-level sockets into
+ * appropriate CryptoSocket instances. This is the top-level interface
+ * used by code wanting to perform network io with appropriate
+ * encryption.
+ **/
+struct CryptoEngine {
+    using SP = std::shared_ptr<CryptoEngine>;
+    virtual CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) = 0;
+    virtual ~CryptoEngine();
+};
+
+/**
+ * Crypto engine without encryption.
+ **/
+struct NullCryptoEngine : public CryptoEngine {
+    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
+};
+
+/**
+ * Very simple crypto engine that requires connection handshaking and
+ * data transformation. Used to test encryption integration separate
+ * from TLS.
+ **/
+struct XorCryptoEngine : public CryptoEngine {
+    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
+};
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/crypto_socket.cpp
new file mode 100644
index 00000000000..416e21d483a
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/crypto_socket.cpp
@@ -0,0 +1,9 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "crypto_socket.h"
+
+namespace vespalib {
+
+CryptoSocket::~CryptoSocket() = default;
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h
new file mode 100644
index 00000000000..7fe7871960f
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h
@@ -0,0 +1,90 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+
+namespace vespalib {
+
+/**
+ * Abstraction of a low-level async network socket which can produce
+ * io events and allows encrypting written data and decrypting read
+ * data. The interface is complexified to handle the use of internal
+ * buffers that may mask io events and pending work. The interface is
+ * simplified by assuming there will be no mid-stream re-negotiation
+ * (no read/write cross-dependencies). Handshaking is explicit and
+ * up-front. This interface is initially designed for persistent
+ * transport connections where closing the connection has no
+ * application-level semantics.
+ **/
+struct CryptoSocket {
+    using UP = std::unique_ptr<CryptoSocket>;
+
+    /**
+     * Get the underlying file descriptor used to detect io events.
+     **/
+    virtual int get_fd() const = 0;
+
+    enum class HandshakeResult { FAIL, DONE, NEED_READ, NEED_WRITE };
+
+    /**
+     * Try to progress the initial connection handshake. Handshaking
+     * will be done once, before any normal reads or writes are
+     * performed. Re-negotiation at a later stage will not be
+     * permitted. This function will be called multiple times until
+     * the status is either DONE or FAIL. When NEED_READ or NEED_WRITE
+     * is returned, the handshake function will be called again when
+     * the appropriate io event has triggered.
+     **/
+    virtual HandshakeResult handshake() = 0;
+
+    /**
+     * This function should be called after handshaking has completed
+     * before calling the read function. It dictates the minimum size
+     * of the application read buffer presented to the read
+     * function. This is needed to support frame-based stateless
+     * decryption of incoming data.
+     **/
+    virtual size_t min_read_buffer_size() const = 0;
+
+    /**
+     * Called when the underlying socket has available data. Read
+     * through the entire input pipeline. The semantics are the same
+     * as with a normal socket read (errno, EOF, etc.).
+     **/
+    virtual ssize_t read(char *buf, size_t len) = 0;
+
+    /**
+     * Similar to read, but this function is not allowed to read from
+     * the underlying socket. This is to enable the application to
+     * make sure that there is no more input data in the read pipeline
+     * that is independent of data not yet read from the actual
+     * socket. Draining data from the input pipeline is done to
+     * prevent masking read events. NOTE: This function should return
+     * 0 when all data has been drained, and the application MUST NOT
+     * interpret that as EOF.
+     **/
+    virtual ssize_t drain(char *buf, size_t len) = 0;
+
+    /**
+     * Called when the application has data it wants to write. Write
+     * through the entire output pipeline. The semantics are the same
+     * as with a normal socket write (errno, etc.).
+     **/
+    virtual ssize_t write(const char *buf, size_t len) = 0;
+
+    /**
+     * Try to flush data in the write pipeline that is not depenedent
+     * on data not yet written by the application into the underlying
+     * socket. This is to enable the application to identify pending
+     * work that may not be completed until the underlying socket is
+     * ready for writing more data. The semantics are the same as with
+     * a normal socket write (errno, etc.) with the exception that 0
+     * will be returned when there is no more data to flush.
+     **/
+    virtual ssize_t flush() = 0;
+
+    virtual ~CryptoSocket();
+};
+
+} // namespace vespalib