diff options
author | Tor Brede Vekterli <vekterli@oath.com> | 2018-10-12 15:36:48 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-12 15:36:48 +0200 |
commit | c8e17ae332cae0ddeaf50a60eaf1ba5a7dc589b3 (patch) | |
tree | e18fff66d40bd06b77f5d79f45b31d6b2d9620e8 | |
parent | 53f1c3df28cb50cc203f334e65dd649523acc4f3 (diff) | |
parent | c3365fadfb772fe69f2edc16a7ad3cfe8e5840b3 (diff) |
Merge pull request #7288 from vespa-engine/havardpe/fbench-tls-support
Havardpe/fbench tls support
-rw-r--r-- | fbench/src/fbench/client.cpp | 4 | ||||
-rw-r--r-- | fbench/src/fbench/client.h | 3 | ||||
-rw-r--r-- | fbench/src/fbench/fbench.cpp | 94 | ||||
-rw-r--r-- | fbench/src/fbench/fbench.h | 5 | ||||
-rw-r--r-- | fbench/src/geturl/geturl.cpp | 5 | ||||
-rw-r--r-- | fbench/src/httpclient/httpclient.cpp | 52 | ||||
-rw-r--r-- | fbench/src/httpclient/httpclient.h | 24 | ||||
-rw-r--r-- | fbench/src/test/httpclient.cpp | 7 | ||||
-rw-r--r-- | fbench/src/test/httpclient_splitstring.cpp | 6 |
9 files changed, 162 insertions, 38 deletions
diff --git a/fbench/src/fbench/client.cpp b/fbench/src/fbench/client.cpp index 0200ea2d067..754fc809511 100644 --- a/fbench/src/fbench/client.cpp +++ b/fbench/src/fbench/client.cpp @@ -8,13 +8,13 @@ #include <cassert> #include <cstring> -Client::Client(ClientArguments *args) +Client::Client(vespalib::CryptoEngine::SP engine, ClientArguments *args) : _args(args), _status(new ClientStatus()), _reqTimer(new Timer()), _cycleTimer(new Timer()), _masterTimer(new Timer()), - _http(new HTTPClient(_args->_hostname, _args->_port, + _http(new HTTPClient(std::move(engine), _args->_hostname, _args->_port, _args->_keepAlive, _args->_headerBenchmarkdataCoverage, _args->_extraHeaders, _args->_authority)), _reader(new FileReader()), diff --git a/fbench/src/fbench/client.h b/fbench/src/fbench/client.h index 1e0b750dbb2..ce7c13fc982 100644 --- a/fbench/src/fbench/client.h +++ b/fbench/src/fbench/client.h @@ -4,6 +4,7 @@ #include <fstream> #include <atomic> #include <thread> +#include <vespa/vespalib/net/crypto_engine.h> #define FBENCH_DELIMITER "\n[--xxyyzz--FBENCH_MAGIC_DELIMITER--zzyyxx--]\n" @@ -188,7 +189,7 @@ public: * The client arguments given to this method becomes the * responsibility of the client. **/ - Client(ClientArguments *args); + Client(vespalib::CryptoEngine::SP engine, ClientArguments *args); /** * Delete objects owned by this client, including the client arguments. diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index c98c3ead4fe..36c60e08ef7 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -3,16 +3,40 @@ #include <httpclient/httpclient.h> #include <util/filereader.h> #include <util/clientstatus.h> +#include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/net/tls/transport_security_options.h> +#include <vespa/vespalib/net/tls/tls_crypto_engine.h> +#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/io/mapped_file_input.h> #include "client.h" #include "fbench.h" #include <cstring> #include <cmath> #include <csignal> +namespace { + +std::string maybe_load(const std::string &file_name, bool &failed) { + std::string content; + if (!file_name.empty()) { + vespalib::MappedFileInput file(file_name); + if (file.valid()) { + content = std::string(file.get().data, file.get().size); + } else { + fprintf(stderr, "could not load file: '%s'\n", file_name.c_str()); + failed = true; + } + } + return content; +} + +} + sig_atomic_t exitSignal = 0; FBench::FBench() - : _clients(), + : _crypto_engine(), + _clients(), _ignoreCount(0), _cycle(0), _filenamePattern(NULL), @@ -35,6 +59,44 @@ FBench::~FBench() free(_outputPattern); } +bool +FBench::init_crypto_engine(const std::string &ca_certs_file_name, + const std::string &cert_chain_file_name, + const std::string &private_key_file_name) +{ + if (ca_certs_file_name.empty() && + cert_chain_file_name.empty() && + private_key_file_name.empty()) + { + _crypto_engine = std::make_shared<vespalib::NullCryptoEngine>(); + return true; + } + if (ca_certs_file_name.empty()) { + fprintf(stderr, "CA certificate required; specify with -T\n"); + return false; + } + if (cert_chain_file_name.empty() != private_key_file_name.empty()) { + fprintf(stderr, "both client certificate AND client private key required; specify with -C and -K\n"); + return false; + } + bool load_failed = false; + vespalib::net::tls::TransportSecurityOptions + tls_opts(maybe_load(ca_certs_file_name, load_failed), + maybe_load(cert_chain_file_name, load_failed), + maybe_load(private_key_file_name, load_failed)); + if (load_failed) { + fprintf(stderr, "failed to load transport security options\n"); + return false; + } + try { + _crypto_engine = std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); + } catch (vespalib::net::tls::CryptoException &e) { + fprintf(stderr, "%s\n", e.what()); + return false; + } + return true; +} + void FBench::InitBenchmark(int numClients, int ignoreCount, int cycle, const char *filenamePattern, const char *outputPattern, @@ -78,7 +140,7 @@ FBench::CreateClients() off_beg = _queryfileOffset[i]; off_end = _queryfileOffset[i+1]; } - client = std::make_unique<Client>( + client = std::make_unique<Client>(_crypto_engine, new ClientArguments(i, _clients.size(), _filenamePattern, _outputPattern, _hostnames[i % _hostnames.size()].c_str(), _ports[i % _ports.size()], _cycle, @@ -226,12 +288,15 @@ FBench::Usage() printf(" -o <str> : save query results to output files with the given pattern\n"); printf(" (default is not saving.)\n"); printf(" -r <num> : number of times to re-use each query file. -1 means no limit [-1]\n"); - printf(" -m <num> : max line size in input query files [8192].\n"); + printf(" -m <num> : max line size in input query files [131072].\n"); printf(" Can not be less than the minimum [1024].\n"); printf(" -p <num> : print summary every <num> seconds.\n"); printf(" -k : disable HTTP keep-alive.\n"); - printf(" -y : write data on coverage to output file (must used with -x).\n"); - printf(" -z : use single query file to be distributed between clients.\n\n"); + printf(" -y : write data on coverage to output file.\n"); + printf(" -z : use single query file to be distributed between clients.\n"); + printf(" -T <str> : CA certificate file to verify peer against.\n"); + printf(" -C <str> : client certificate file name.\n"); + printf(" -K <str> : client private key file name.\n\n"); printf(" <hostname> : the host you want to benchmark.\n"); printf(" <port> : the port to use when contacting the host.\n\n"); printf("Several hostnames and ports can be listed\n"); @@ -263,6 +328,9 @@ FBench::Main(int argc, char *argv[]) const char *outputFilePattern = NULL; std::string queryStringToAppend; std::string extraHeaders; + std::string ca_certs_file_name; // -T + std::string cert_chain_file_name; // -C + std::string private_key_file_name; // -K int restartLimit = -1; bool keepAlive = true; @@ -282,7 +350,7 @@ FBench::Main(int argc, char *argv[]) idx = 1; optError = false; - while((opt = GetOpt(argc, argv, "H:A:a:n:c:l:i:s:q:o:r:m:p:kxyzP", arg, idx)) != -1) { + while((opt = GetOpt(argc, argv, "H:A:T:C:K:a:n:c:l:i:s:q:o:r:m:p:kxyzP", arg, idx)) != -1) { switch(opt) { case 'A': authority = arg; @@ -294,6 +362,15 @@ FBench::Main(int argc, char *argv[]) return -1; } break; + case 'T': + ca_certs_file_name = std::string(arg); + break; + case 'C': + cert_chain_file_name = std::string(arg); + break; + case 'K': + private_key_file_name = std::string(arg); + break; case 'a': queryStringToAppend = std::string(arg); break; @@ -365,6 +442,11 @@ FBench::Main(int argc, char *argv[]) return -1; } + if (!init_crypto_engine(ca_certs_file_name, cert_chain_file_name, private_key_file_name)) { + fprintf(stderr, "failed to initialize crypto engine\n"); + return -1; + } + short hosts = args / 2; for (int i=0; i<hosts; ++i) diff --git a/fbench/src/fbench/fbench.h b/fbench/src/fbench/fbench.h index 4c5885d8988..8cbab2e6d6c 100644 --- a/fbench/src/fbench/fbench.h +++ b/fbench/src/fbench/fbench.h @@ -10,6 +10,7 @@ class FBench { private: + vespalib::CryptoEngine::SP _crypto_engine; std::vector<Client::UP> _clients; int _numClients; int _ignoreCount; @@ -32,6 +33,10 @@ private: std::string _extraHeaders; std::string _authority; + bool init_crypto_engine(const std::string &ca_certs_file_name, + const std::string &cert_chain_file_name, + const std::string &private_key_file_name); + void InitBenchmark(int numClients, int ignoreCount, int cycle, const char *filenamePattern, const char *outputPattern, int byteLimit, int restartLimit, int maxLineSize, diff --git a/fbench/src/geturl/geturl.cpp b/fbench/src/geturl/geturl.cpp index f279b0b55c8..868d33f30a9 100644 --- a/fbench/src/geturl/geturl.cpp +++ b/fbench/src/geturl/geturl.cpp @@ -1,4 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <iostream> @@ -10,7 +12,8 @@ main(int argc, char** argv) return -1; } - HTTPClient client(argv[1], atoi(argv[2]), false, false); + auto engine = std::make_shared<vespalib::NullCryptoEngine>(); + HTTPClient client(engine, argv[1], atoi(argv[2]), false, false); if (!client.Fetch(argv[3], &std::cout).Ok()) { fprintf(stderr, "geturl: could not fetch 'http://%s:%d%s'\n", argv[1], atoi(argv[2]), argv[3]); diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index c49ef5da12c..002d2770dcd 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -17,10 +17,12 @@ HTTPClient::ChunkedReader HTTPClient::ChunkedReader::_instance; -HTTPClient::HTTPClient(const char *hostname, int port, +HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, int port, bool keepAlive, bool headerBenchmarkdataCoverage, const std::string & extraHeaders, const std::string &authority) - : _socket(new FastOS_Socket()), + : _engine(std::move(engine)), + _address(vespalib::SocketAddress::select_remote(port, hostname)), + _socket(), _hostname(hostname), _port(port), _keepAlive(keepAlive), @@ -48,7 +50,6 @@ HTTPClient::HTTPClient(const char *hostname, int port, _dataDone(false), _reader(NULL) { - _socket->SetAddressByHostName(port, hostname); if (_authority == "") { char tmp[1024]; snprintf(tmp, 1024, "%s:%d", hostname, port); @@ -56,17 +57,31 @@ HTTPClient::HTTPClient(const char *hostname, int port, } } +bool +HTTPClient::connect_socket() +{ + _socket.reset(); + auto handle = _address.connect([](auto &h) + { + return (h.set_nodelay(true) && + h.set_linger(false, 0)); + }); + if (!handle.valid()) { + return false; + } + _socket = vespalib::SyncCryptoSocket::create(*_engine, std::move(handle), false); + return bool(_socket); +} + ssize_t HTTPClient::FillBuffer() { - _bufused = _socket->Read(_buf, _bufsize); // may be -1 + _bufused = _socket->read(_buf, _bufsize); // may be -1 _bufpos = 0; return _bufused; } HTTPClient::~HTTPClient() { - if (_socket) - _socket->Close(); delete [] _buf; } @@ -148,9 +163,9 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen // try to reuse connection if keep-alive is enabled if (_keepAlive - && _socket->IsOpened() - && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) - && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen) + && _socket + && _socket->write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen) && FillBuffer() > 0) { // DEBUG @@ -161,17 +176,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen } return true; } else { - _socket->Close(); + _socket.reset(); ResetBuffer(); } // try to open new connection to server - if (_socket->SetSoBlocking(true) - && _socket->Connect() - && _socket->SetNoDelay(true) - && _socket->SetSoLinger(false, 0) - && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) - && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen)) + if (connect_socket() + && _socket->write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen)) { // DEBUG @@ -181,7 +193,7 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen } return true; } else { - _socket->Close(); + _socket.reset(); } // DEBUG @@ -395,7 +407,7 @@ HTTPClient::ConnCloseReader::Read(HTTPClient &client, res = fromBuffer; } if ((len - fromBuffer) > (len >> 1)) { - readRes = client._socket->Read(static_cast<char *>(buf) + readRes = client._socket->read(static_cast<char *>(buf) + fromBuffer, len - fromBuffer); if (readRes < 0) { client.Close(); @@ -434,7 +446,7 @@ HTTPClient::ContentLengthReader::Read(HTTPClient &client, readLen = (len - fromBuffer < client._contentLength - client._dataRead) ? len - fromBuffer : client._contentLength - client._dataRead; - readRes = client._socket->Read(static_cast<char *>(buf) + readRes = client._socket->read(static_cast<char *>(buf) + fromBuffer, readLen); if (readRes < 0) { client.Close(); @@ -510,7 +522,7 @@ HTTPClient::Close() || _connectionCloseGiven || !_dataDone || (_httpVersion == 0 && !_keepAliveGiven)) ? - _socket->Close() : true; + (_socket.reset(), true) : true; } HTTPClient::FetchStatus diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h index 783545744d5..9c3ccd437d1 100644 --- a/fbench/src/httpclient/httpclient.h +++ b/fbench/src/httpclient/httpclient.h @@ -3,7 +3,9 @@ #include <ostream> #include <memory> -#include <vespa/fastos/socket.h> +#include <vespa/vespalib/net/sync_crypto_socket.h> +#include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/net/socket_address.h> /** * This class implements a HTTP client that may be used to fetch @@ -88,7 +90,10 @@ protected: }; friend class HTTPClient::ChunkedReader; - std::unique_ptr<FastOS_Socket> _socket; + vespalib::CryptoEngine::SP _engine; + vespalib::SocketAddress _address; + vespalib::SyncCryptoSocket::UP _socket; + std::string _hostname; int _port; bool _keepAlive; @@ -131,6 +136,17 @@ protected: _bufused = 0; } + /** + * (re)connects the socket to the host/port specified in the + * constructor. The hostname is not resolved again; the resolve + * result is cached by the constructor. Also sets tcp nodelay flag + * and disables lingering. Note to servers: This is a no-nonsense + * socket that will be closed in your face in very ungraceful + * ways. Do not expect half-close niceties or tls session + * termination packets. + **/ + bool connect_socket(); + /** * Fill the internal buffer with data from the url we are connected * to. @@ -215,8 +231,8 @@ public: * @param port the TCP port to use when contacting the host. * @param keepAlive flag indicating if keep-alive should be enabled. **/ - HTTPClient(const char *hostname, int port, bool keepAlive, - bool headerBenchmarkdataCoverage, const std::string & extraHeaders="", const std::string &authority = ""); + HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, int port, bool keepAlive, + bool headerBenchmarkdataCoverage, const std::string & extraHeaders="", const std::string &authority = ""); /** * Disconnect from server and free memory. diff --git a/fbench/src/test/httpclient.cpp b/fbench/src/test/httpclient.cpp index 4201da68b97..0d350c393c1 100644 --- a/fbench/src/test/httpclient.cpp +++ b/fbench/src/test/httpclient.cpp @@ -1,4 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <iostream> #include <thread> @@ -11,13 +13,14 @@ main(int argc, char **argv) return 1; } + auto engine = std::make_shared<vespalib::NullCryptoEngine>(); HTTPClient *client; ssize_t len; if(argc == 4) { - client = new HTTPClient(argv[1], atoi(argv[2]), false, true); + client = new HTTPClient(engine, argv[1], atoi(argv[2]), false, true); } else { - client = new HTTPClient(argv[1], atoi(argv[2]), true, true); + client = new HTTPClient(engine, argv[1], atoi(argv[2]), true, true); } std::ostream * output = & std::cout; diff --git a/fbench/src/test/httpclient_splitstring.cpp b/fbench/src/test/httpclient_splitstring.cpp index d766b0f8f4b..4f3e0027db0 100644 --- a/fbench/src/test/httpclient_splitstring.cpp +++ b/fbench/src/test/httpclient_splitstring.cpp @@ -1,13 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <cstring> class DebugHTTPClient : public HTTPClient { public: - DebugHTTPClient(const char* server, int port, bool keepAlive) - : HTTPClient(server, port, keepAlive, true) {} + DebugHTTPClient() + : HTTPClient(std::make_shared<vespalib::NullCryptoEngine>(), + "localhost", 80, true, true) {} static void SplitLineTest(const char *input); static void DebugSplitLine(); |