half_close for crypto sockets

9 files changed, 149 insertions, 4 deletions

diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
index 91babc51476..67077f86f1e 100644
--- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
+++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
@@ -76,6 +76,11 @@ void flush(CryptoSocket &socket) {
     ASSERT_TRUE((res == 0) || is_blocked(res));
 }
 
+void half_close(CryptoSocket &socket) {
+    auto res = socket.half_close();
+    ASSERT_TRUE((res == 0) || is_blocked(res));
+}
+
 //-----------------------------------------------------------------------------
 
 vespalib::string read_bytes(CryptoSocket &socket, SmartBuffer &read_buffer, size_t wanted_bytes) {
@@ -86,7 +91,25 @@ vespalib::string read_bytes(CryptoSocket &socket, SmartBuffer &read_buffer, size
         drain(socket, read_buffer);
     }
     auto data = read_buffer.obtain();
-    return vespalib::string(data.data, wanted_bytes);
+    vespalib::string message(data.data, wanted_bytes);
+    read_buffer.evict(message.size());
+    return message;
+}
+
+//-----------------------------------------------------------------------------
+
+void read_EOF(CryptoSocket &socket, SmartBuffer &read_buffer) {
+    ASSERT_EQUAL(read_buffer.obtain().size, 0u);
+    SingleFdSelector selector(socket.get_fd());
+    ASSERT_TRUE(selector.wait_readable());
+    size_t chunk_size = std::max(size_t(4096), socket.min_read_buffer_size());
+    auto chunk = read_buffer.reserve(chunk_size);
+    auto res = socket.read(chunk.data, chunk.size);
+    while (is_blocked(res)) {
+        ASSERT_TRUE(selector.wait_readable());
+        res = socket.read(chunk.data, chunk.size);
+    }
+    ASSERT_EQUAL(res, 0);
 }
 
 //-----------------------------------------------------------------------------
@@ -106,6 +129,35 @@ void write_bytes(CryptoSocket &socket, const vespalib::string &message) {
 
 //-----------------------------------------------------------------------------
 
+void write_EOF(CryptoSocket &socket) {
+    SingleFdSelector selector(socket.get_fd());
+    ASSERT_TRUE(selector.wait_writable());
+    auto res = socket.half_close();
+    while (is_blocked(res)) {
+        ASSERT_TRUE(selector.wait_writable());
+        res = socket.half_close();
+    }
+    ASSERT_EQUAL(res, 0);
+}
+
+//-----------------------------------------------------------------------------
+
+void verify_graceful_shutdown(CryptoSocket &socket, SmartBuffer &read_buffer, bool is_server) {
+    if(is_server) {
+        TEST_DO(write_EOF(socket));
+        TEST_DO(read_EOF(socket, read_buffer));
+        TEST_DO(read_EOF(socket, read_buffer));
+        TEST_DO(read_EOF(socket, read_buffer));
+    } else {
+        TEST_DO(read_EOF(socket, read_buffer));
+        TEST_DO(read_EOF(socket, read_buffer));
+        TEST_DO(read_EOF(socket, read_buffer));
+        TEST_DO(write_EOF(socket));
+    }
+}
+
+//-----------------------------------------------------------------------------
+
 void verify_socket_io(CryptoSocket &socket, SmartBuffer &read_buffer, bool is_server) {
     vespalib::string client_message = "please pick up, I need to talk to you";
     vespalib::string server_message = "hello, this is the server speaking";
@@ -153,6 +205,7 @@ void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_ser
     TEST_DO(verify_handshake(*my_socket));
     drain(*my_socket, read_buffer);
     TEST_DO(verify_socket_io(*my_socket, read_buffer, is_server));
+    TEST_DO(verify_graceful_shutdown(*my_socket, read_buffer, is_server));
 }
 
 //-----------------------------------------------------------------------------
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
index d45b0cea1c5..cdce2d41bf6 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -46,6 +46,7 @@ public:
     ssize_t drain(char *, size_t) override { return 0; }
     ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); }
     ssize_t flush() override { return 0; }
+    ssize_t half_close() override { return _socket.half_close(); }
 };
 
 class XorCryptoSocket : public CryptoSocket
@@ -168,6 +169,16 @@ public:
         }
         return 0; // done
     }
+    ssize_t half_close() override {
+        auto flush_res = flush();
+        while (flush_res > 0) {
+            flush_res = flush();
+        }
+        if (flush_res < 0) {
+            return flush_res;
+        }
+        return _socket.half_close();
+    }
 };
 
 CryptoEngine::SP create_default_crypto_engine() {
diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h
index fa76acbc2cd..3a95cf33add 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_socket.h
+++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h
@@ -88,6 +88,43 @@ struct CryptoSocket {
      **/
     virtual ssize_t flush() = 0;
 
+    /**
+     * Signal the end of outgoing data. Note that this might require
+     * writing data to the underlying socket to notify the client that
+     * no more data will be sent. This function should be treated as a
+     * combination of write and flush and should be re-tried after the
+     * socket becomes writable if EWOULDBLOCK is returned. Neither
+     * write nor flush should be called after this function is
+     * called. When this function indicates success (returns 0) all
+     * pending data has been written to the underlying socket and the
+     * write aspect of the socket has been shut down. Performing
+     * half_close on one end of a connection will eventually lead to
+     * the other end receiving EOF after all application data has been
+     * read. Note that closing the socket immediately after performing
+     * half_close might still result in data loss since there is no
+     * way of knowing when the data has actually been sent on the
+     * network.
+     *
+     * Ideal graceful shutdown is initiated by one end performing
+     * half_close on the connection. When the other end receives EOF
+     * it performs half_close on its end of the connection. When both
+     * ends have received EOF the sockets can be closed. The ideal
+     * scenario is broken by two things: (1) the two generals paradox,
+     * which proves that both endpoints coming to an agreement about
+     * the connection being gracefully shut down is not possible. (2)
+     * clients tend to do random things with the connection, leaving
+     * it up to the server to be the more responsible party.
+     *
+     * Real-life graceful-ish shutdown (server-side) should be
+     * performed by doing half_close on the server end of the
+     * connection. Any incoming data should be read in the hope of
+     * getting EOF. The socket should be closed when either EOF is
+     * reached, a read error occurred (typically connection reset by
+     * peer or similar) or a timeout is reached (application
+     * equivalent of the linger option on the socket).
+     **/
+    virtual ssize_t half_close() = 0;
+
     virtual ~CryptoSocket();
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/socket_handle.cpp b/vespalib/src/vespa/vespalib/net/socket_handle.cpp
index 36eae46031f..aaaad1624bb 100644
--- a/vespalib/src/vespa/vespalib/net/socket_handle.cpp
+++ b/vespalib/src/vespa/vespalib/net/socket_handle.cpp
@@ -47,6 +47,12 @@ SocketHandle::shutdown()
 }
 
 int
+SocketHandle::half_close()
+{
+    return ::shutdown(_fd, SHUT_WR);
+}
+
+int
 SocketHandle::get_so_error() const
 {
     if (!valid()) {
diff --git a/vespalib/src/vespa/vespalib/net/socket_handle.h b/vespalib/src/vespa/vespalib/net/socket_handle.h
index 1d942d8530f..0bf3f6b43c7 100644
--- a/vespalib/src/vespa/vespalib/net/socket_handle.h
+++ b/vespalib/src/vespa/vespalib/net/socket_handle.h
@@ -59,6 +59,7 @@ public:
     ssize_t write(const char *buf, size_t len);
     SocketHandle accept();
     void shutdown();
+    int half_close();
     int get_so_error() const;
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp
index 28aede69d22..569cce6db7b 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp
@@ -96,15 +96,19 @@ ssize_t
 CryptoCodecAdapter::read(char *buf, size_t len)
 {
     auto drain_res = drain(buf, len);
-    if (drain_res != 0) {
+    if ((drain_res != 0) || _got_tls_close) {
         return drain_res;
     }
     auto fill_res = fill_input();
     if (fill_res <= 0) {
+        if (fill_res == 0) {
+            fill_res = -1;
+            errno = EIO;
+        }
         return fill_res;
     }
     drain_res = drain(buf, len);
-    if (drain_res != 0) {
+    if ((drain_res != 0) || _got_tls_close) {
         return drain_res;
     }
     errno = EWOULDBLOCK;
@@ -120,6 +124,9 @@ CryptoCodecAdapter::drain(char *buf, size_t len)
         errno = EIO;
         return -1;        
     }
+    if (res.closed()) {
+        _got_tls_close = true;
+    }
     _input.evict(res.bytes_consumed);
     return res.bytes_produced;
 }
@@ -163,4 +170,28 @@ CryptoCodecAdapter::flush()
     return 0; // done
 }
 
+ssize_t
+CryptoCodecAdapter::half_close()
+{
+    auto flush_res = flush_all();
+    if ((flush_res < 0)) {
+        return flush_res;
+    }
+    if (!_encoded_tls_close) {
+        auto dst = _output.reserve(_codec->min_encode_buffer_size());
+        auto res = _codec->half_close(dst.data, dst.size);
+        if (res.failed) {
+            errno = EIO;
+            return -1;
+        }
+        _output.commit(res.bytes_produced);
+        _encoded_tls_close = true;
+    }
+    flush_res = flush_all();
+    if ((flush_res < 0)) {
+        return flush_res;
+    }
+    return _socket.half_close();
+}
+
 } // namespace vespalib::net::tls
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h
index af2db3525d3..6af04d67d41 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h
@@ -20,6 +20,8 @@ private:
     SmartBuffer                  _output;
     SocketHandle                 _socket;
     std::unique_ptr<CryptoCodec> _codec;
+    bool                         _got_tls_close;
+    bool                         _encoded_tls_close;
 
     bool is_blocked(ssize_t res, int error) const {
         return ((res < 0) && ((error == EWOULDBLOCK) || (error == EAGAIN)));
@@ -30,7 +32,8 @@ private:
     ssize_t flush_all();  // -1/0 -> error/ok
 public:
     CryptoCodecAdapter(SocketHandle socket, std::unique_ptr<CryptoCodec> codec)
-        : _input(64 * 1024), _output(64 * 1024), _socket(std::move(socket)), _codec(std::move(codec)) {}
+        : _input(64 * 1024), _output(64 * 1024), _socket(std::move(socket)), _codec(std::move(codec)),
+          _got_tls_close(false), _encoded_tls_close(false) {}
     void inject_read_data(const char *buf, size_t len) override;
     int get_fd() const override { return _socket.get(); }
     HandshakeResult handshake() override;
@@ -39,6 +42,7 @@ public:
     ssize_t drain(char *, size_t) override;
     ssize_t write(const char *buf, size_t len) override;
     ssize_t flush() override;
+    ssize_t half_close() override;
 };
 
 } // namespace vespalib::net::tls
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
index 1e8fbce13eb..cd5c4114844 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
@@ -78,6 +78,7 @@ public:
     }
     ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); }
     ssize_t flush() override { return 0; }
+    ssize_t half_close() override { return _socket.half_close(); }
 };
 
 } // namespace vespalib::<unnamed>
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h
index 3c7963cc948..9d3a190829a 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h
@@ -30,6 +30,7 @@ public:
     ssize_t drain(char *buf, size_t len) override { return _socket->drain(buf, len); }
     ssize_t write(const char *buf, size_t len) override { return _socket->write(buf, len); }
     ssize_t flush() override { return _socket->flush(); }
+    ssize_t half_close() override { return _socket->half_close(); }
 };
 
 } // namespace vespalib