diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
| commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
| tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vbench | |
Publish
Diffstat (limited to 'vbench')
266 files changed, 7037 insertions, 0 deletions
diff --git a/vbench/.gitignore b/vbench/.gitignore new file mode 100644 index 00000000000..a9b20e8992d --- /dev/null +++ b/vbench/.gitignore @@ -0,0 +1,2 @@ +Makefile +Testing diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt new file mode 100644 index 00000000000..05ba9815ddc --- /dev/null +++ b/vbench/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalib + staging_vespalib + + LIBS + src/vbench + src/vbench/core + src/vbench/http + src/vbench/test + src/vbench/vbench + + APPS + src/apps/dumpurl + src/apps/vbench + + TESTS + src/tests/app_dumpurl + src/tests/app_vbench + src/tests/benchmark_headers + src/tests/buffered_output + src/tests/byte_input + src/tests/dispatcher + src/tests/dropped_tagger + src/tests/handler_thread + src/tests/hex_number + src/tests/http_client + src/tests/http_connection + src/tests/http_connection_pool + src/tests/input_file_reader + src/tests/latency_analyzer + src/tests/line_reader + src/tests/mapped_file_input + src/tests/memory + src/tests/qps_analyzer + src/tests/qps_tagger + src/tests/request_dumper + src/tests/request_generator + src/tests/request_sink + src/tests/server_socket + src/tests/server_spec + src/tests/server_tagger + src/tests/simple_buffer + src/tests/socket + src/tests/taint + src/tests/time_queue + src/tests/timer + src/tests/writable_memory +) diff --git a/vbench/OWNERS b/vbench/OWNERS new file mode 100644 index 00000000000..12b533ec610 --- /dev/null +++ b/vbench/OWNERS @@ -0,0 +1 @@ +havardpe diff --git a/vbench/README b/vbench/README new file mode 100644 index 00000000000..b3c3fe2b603 --- /dev/null +++ b/vbench/README @@ -0,0 +1,20 @@ +vbench is a flexible framework used to benchmark HTTP server +performance. + +TODO + +* support multiple request dimensions with separate analyze chains. + +* log information for different dimensions to different files. + +* save failed requests to disk. + +* enable/disable keep-alive per request. + +* save selected responses to disk. + +* support 'real-time' query log replay (need format). + +* tag all components in model with unique names. + +* support external get/set of selected values. diff --git a/vbench/src/.gitignore b/vbench/src/.gitignore new file mode 100644 index 00000000000..7466c45681b --- /dev/null +++ b/vbench/src/.gitignore @@ -0,0 +1,4 @@ +/Makefile.ini +/project.dsw +/config_command.sh +/vbench.mak diff --git a/vbench/src/apps/dumpurl/.gitignore b/vbench/src/apps/dumpurl/.gitignore new file mode 100644 index 00000000000..27d8a9f5901 --- /dev/null +++ b/vbench/src/apps/dumpurl/.gitignore @@ -0,0 +1,5 @@ +/.depend +/Makefile +/dumpurl +/out.* +vbench_dumpurl_app diff --git a/vbench/src/apps/dumpurl/CMakeLists.txt b/vbench/src/apps/dumpurl/CMakeLists.txt new file mode 100644 index 00000000000..00f00da47e8 --- /dev/null +++ b/vbench/src/apps/dumpurl/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_dumpurl_app + SOURCES + dumpurl.cpp + INSTALL bin + DEPENDS + vbench +) diff --git a/vbench/src/apps/dumpurl/dumpurl.cpp b/vbench/src/apps/dumpurl/dumpurl.cpp new file mode 100644 index 00000000000..a3bbc9be714 --- /dev/null +++ b/vbench/src/apps/dumpurl/dumpurl.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vbench/http/http_result_handler.h> +#include <vbench/http/server_spec.h> +#include <vbench/http/http_client.h> + +using namespace vbench; + +class App : public FastOS_Application +{ +public: + int Main(); +}; + +struct MyHttpHandler : public HttpResultHandler { + virtual void handleHeader(const string &name, const string &value) { + fprintf(stderr, "got header: '%s': '%s'\n", name.c_str(), value.c_str()); + } + virtual void handleContent(const Memory &data) { + fprintf(stderr, "got data: %zu bytes\n", data.size); + fwrite(data.data, 1, data.size, stdout); + } + virtual void handleFailure(const string &reason) { + fprintf(stderr, "got FAILURE: '%s'\n", reason.c_str()); + } +}; + +int +App::Main() +{ + if (_argc != 4) { + printf("usage: dumpurl <host> <port> <url>\n"); + return -1; + } + MyHttpHandler myHandler; + bool ok = HttpClient::fetch(ServerSpec(_argv[1], atoi(_argv[2])), _argv[3], myHandler); + return ok ? 0 : 1; +} + +int main(int argc, char **argv) { + App app; + return app.Entry(argc, argv); +} diff --git a/vbench/src/apps/vbench/.gitignore b/vbench/src/apps/vbench/.gitignore new file mode 100644 index 00000000000..6df0c28e125 --- /dev/null +++ b/vbench/src/apps/vbench/.gitignore @@ -0,0 +1,4 @@ +/.depend +/Makefile +/vbench +vbench_app diff --git a/vbench/src/apps/vbench/CMakeLists.txt b/vbench/src/apps/vbench/CMakeLists.txt new file mode 100644 index 00000000000..5c9e423db3c --- /dev/null +++ b/vbench/src/apps/vbench/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_app + SOURCES + vbench.cpp + INSTALL bin + DEPENDS + vbench +) diff --git a/vbench/src/apps/vbench/input.txt b/vbench/src/apps/vbench/input.txt new file mode 100644 index 00000000000..17b2919d56f --- /dev/null +++ b/vbench/src/apps/vbench/input.txt @@ -0,0 +1,30 @@ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ +/ diff --git a/vbench/src/apps/vbench/run.sh b/vbench/src/apps/vbench/run.sh new file mode 100755 index 00000000000..0cd7abc3902 --- /dev/null +++ b/vbench/src/apps/vbench/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +env $(make ldl) ./vbench --input input.txt --qps 2 --host www.host.com --port 80 diff --git a/vbench/src/apps/vbench/vbench.cfg b/vbench/src/apps/vbench/vbench.cfg new file mode 100644 index 00000000000..88ceffd35b8 --- /dev/null +++ b/vbench/src/apps/vbench/vbench.cfg @@ -0,0 +1,16 @@ +{ + http_threads: 1000, + inputs: [ + { + source: { type: 'RequestGenerator', file: 'input.txt' }, + prepare: [ + { type: 'ServerTagger', host: 'www.host.com', port:80 }, + { type: 'QpsTagger', qps: 2 } + ] + } + ], + analyze: [ + { type: 'QpsAnalyzer' }, + { type: 'LatencyAnalyzer' } + ] +} diff --git a/vbench/src/apps/vbench/vbench.cpp b/vbench/src/apps/vbench/vbench.cpp new file mode 100644 index 00000000000..614856cd18c --- /dev/null +++ b/vbench/src/apps/vbench/vbench.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/util/runnable_pair.h> +#include <vbench/vbench/vbench.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <string> +#include <iostream> + +using namespace vbench; + +typedef vespalib::SignalHandler SIG; + +struct NotifyDone : public vespalib::Runnable { + vespalib::Gate &done; + NotifyDone(vespalib::Gate &d) : done(d) {} + virtual void run() { + done.countDown(); + } +}; + +void setupSignals() { + SIG::PIPE.ignore(); + SIG::INT.hook(); + SIG::TERM.hook(); +} + +int run(const std::string &cfg_name) { + MappedFileInput cfg_file(cfg_name); + if (cfg_file.tainted()) { + fprintf(stderr, "could not load config file: %s\n", + cfg_file.tainted().reason().c_str()); + return 1; + } + vespalib::Slime cfg; + vespalib::slime::Memory mapped_cfg(cfg_file.get().data, + cfg_file.get().size); + if (!vespalib::slime::JsonFormat::decode(mapped_cfg, cfg)) { + fprintf(stderr, "unable to parse config file: %s\n", + cfg.toString().c_str()); + return 1; + } + setupSignals(); + vespalib::Gate done; + VBench vbench(cfg); + NotifyDone notify(done); + vespalib::RunnablePair runBoth(vbench, notify); + vespalib::Thread thread(runBoth); + thread.start(); + while (!SIG::INT.check() && !SIG::TERM.check() && !done.await(1000)) {} + if (!done.await(0)) { + vbench.abort(); + done.await(); + } + if (vbench.tainted()) { + fprintf(stderr, "vbench failed: %s\n", + vbench.tainted().reason().c_str()); + return 1; + } + return 0; +} + +int usage(const char *prog) { + fprintf(stderr, "vbench -- vespa benchmarking tool\n\n"); + fprintf(stderr, "usage: %s run <config-file>\n", prog); + fprintf(stderr, " run benchmarking as described in the config file.\n\n"); + return 1; +} + +int main(int argc, char **argv) { + if (argc > 1) { + std::string mode = argv[1]; + if (mode == "run" && argc == 3) { + return run(argv[2]); + } + } + return usage(argv[0]); +} diff --git a/vbench/src/testlist.txt b/vbench/src/testlist.txt new file mode 100644 index 00000000000..2991159d87e --- /dev/null +++ b/vbench/src/testlist.txt @@ -0,0 +1,31 @@ +tests/app_dumpurl +tests/app_vbench +tests/benchmark_headers +tests/buffered_output +tests/byte_input +tests/dispatcher +tests/dropped_tagger +tests/handler_thread +tests/hex_number +tests/http_client +tests/http_connection +tests/http_connection_pool +tests/input_file_reader +tests/latency_analyzer +tests/line_reader +tests/mapped_file_input +tests/memory +tests/qps_analyzer +tests/qps_tagger +tests/request_dumper +tests/request_generator +tests/request_sink +tests/server_socket +tests/server_spec +tests/server_tagger +tests/simple_buffer +tests/socket +tests/taint +tests/time_queue +tests/timer +tests/writable_memory diff --git a/vbench/src/tests/.gitignore b/vbench/src/tests/.gitignore new file mode 100644 index 00000000000..a3e9c375723 --- /dev/null +++ b/vbench/src/tests/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +*_test diff --git a/vbench/src/tests/app_dumpurl/.gitignore b/vbench/src/tests/app_dumpurl/.gitignore new file mode 100644 index 00000000000..6cf8101a380 --- /dev/null +++ b/vbench/src/tests/app_dumpurl/.gitignore @@ -0,0 +1 @@ +vbench_app_dumpurl_test_app diff --git a/vbench/src/tests/app_dumpurl/CMakeLists.txt b/vbench/src/tests/app_dumpurl/CMakeLists.txt new file mode 100644 index 00000000000..2ff66e4edcd --- /dev/null +++ b/vbench/src/tests/app_dumpurl/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_app_dumpurl_test_app + SOURCES + app_dumpurl_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_app_dumpurl_test_app NO_VALGRIND COMMAND vbench_app_dumpurl_test_app) diff --git a/vbench/src/tests/app_dumpurl/FILES b/vbench/src/tests/app_dumpurl/FILES new file mode 100644 index 00000000000..54382f23234 --- /dev/null +++ b/vbench/src/tests/app_dumpurl/FILES @@ -0,0 +1 @@ +app_dumpurl_test.cpp diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp new file mode 100644 index 00000000000..48a8ce156f0 --- /dev/null +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> +#include <vespa/vespalib/util/slaveproc.h> + +using namespace vbench; +using vespalib::SlaveProc; + +bool endsWith(const Memory &mem, const string &str) { + return (mem.size < str.size()) ? false + : (strncmp(mem.data + mem.size - str.size(), str.data(), str.size()) == 0); +} + +void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { + ByteInput in(input, 256); + while (!endsWith(buffer.get(), end)) { + int c = in.get(); + if (c < 0) { + return; + } + buffer.reserve(1).data[0] = c; + buffer.commit(1, 0); + } +} + +TEST("dumpurl usage") { + std::string out; + EXPECT_FALSE(SlaveProc::run("../../apps/dumpurl/vbench_dumpurl_app", out)); + fprintf(stderr, "%s\n", out.c_str()); +} + +TEST_MT_F("run dumpurl", 2, ServerSocket()) { + if (thread_id == 0) { + Stream::UP stream = f1.accept(); + SimpleBuffer ignore; + readUntil(*stream, ignore, "\r\n\r\n"); + BufferedOutput out(*stream, 256); + out.append("HTTP/1.1 200\r\n"); + out.append("content-length: 4\r\n"); + out.append("\r\n"); + out.append("data"); + } else { + std::string out; + EXPECT_TRUE(SlaveProc::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", + f1.port()).c_str(), out)); + fprintf(stderr, "%s\n", out.c_str()); + } +} + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/app_vbench/.gitignore b/vbench/src/tests/app_vbench/.gitignore new file mode 100644 index 00000000000..0f290bb76ac --- /dev/null +++ b/vbench/src/tests/app_vbench/.gitignore @@ -0,0 +1,3 @@ +/vbench.cfg +/vbench.out +vbench_app_vbench_test_app diff --git a/vbench/src/tests/app_vbench/CMakeLists.txt b/vbench/src/tests/app_vbench/CMakeLists.txt new file mode 100644 index 00000000000..6295571cc09 --- /dev/null +++ b/vbench/src/tests/app_vbench/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_app_vbench_test_app + SOURCES + app_vbench_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_app_vbench_test_app NO_VALGRIND COMMAND vbench_app_vbench_test_app) diff --git a/vbench/src/tests/app_vbench/FILES b/vbench/src/tests/app_vbench/FILES new file mode 100644 index 00000000000..8a53b5dfde7 --- /dev/null +++ b/vbench/src/tests/app_vbench/FILES @@ -0,0 +1,3 @@ +app_vbench_test.cpp +vbench.cfg +vbench.out diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp new file mode 100644 index 00000000000..87461702075 --- /dev/null +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> +#include <vespa/vespalib/util/slaveproc.h> + +using namespace vbench; +using vespalib::SlaveProc; + +bool endsWith(const Memory &mem, const string &str) { + return (mem.size < str.size()) ? false + : (strncmp(mem.data + mem.size - str.size(), str.data(), str.size()) == 0); +} + +void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { + ByteInput in(input, 256); + while (!endsWith(buffer.get(), end)) { + int c = in.get(); + if (c < 0) { + return; + } + buffer.reserve(1).data[0] = c; + buffer.commit(1, 0); + } +} + +TEST("vbench usage") { + std::string out; + EXPECT_FALSE(SlaveProc::run("../../apps/vbench/vbench_app", out)); + fprintf(stderr, "%s\n", out.c_str()); +} + +TEST_MT_F("run vbench", 2, ServerSocket()) { + if (thread_id == 0) { + for (;;) { + Stream::UP stream = f1.accept(); + if (stream.get() == 0) { + break; + } + SimpleBuffer ignore; + readUntil(*stream, ignore, "\r\n\r\n"); + BufferedOutput out(*stream, 256); + out.append("HTTP/1.1 200\r\n"); + out.append("content-length: 4\r\n"); + out.append("X-Yahoo-Vespa-NumHits 10\r\n"); + out.append("X-Yahoo-Vespa-TotalHitCount 100\r\n"); + out.append("\r\n"); + out.append("data"); + } + } else { + std::string out; + EXPECT_TRUE(SlaveProc::run("cp vbench.cfg.template vbench.cfg")); + EXPECT_TRUE(SlaveProc::run(strfmt("sed -i 's/_LOCAL_PORT_/%d/' vbench.cfg", f1.port()).c_str())); + EXPECT_TRUE(SlaveProc::run("../../apps/vbench/vbench_app run vbench.cfg 2> vbench.out", out)); + fprintf(stderr, "%s\n", out.c_str()); + f1.close(); + } +} + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/app_vbench/input.txt b/vbench/src/tests/app_vbench/input.txt new file mode 100644 index 00000000000..c9a57e7a555 --- /dev/null +++ b/vbench/src/tests/app_vbench/input.txt @@ -0,0 +1,72 @@ +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo +/foo diff --git a/vbench/src/tests/app_vbench/vbench.cfg.template b/vbench/src/tests/app_vbench/vbench.cfg.template new file mode 100644 index 00000000000..de8e30a7fbe --- /dev/null +++ b/vbench/src/tests/app_vbench/vbench.cfg.template @@ -0,0 +1,25 @@ +{ + http_threads: 32, + inputs: [ + { + source: { type: 'RequestGenerator', file: 'input.txt' }, + prepare: [ + { type: 'ServerTagger', host: 'localhost', port:_LOCAL_PORT_ }, + { type: 'QpsTagger', qps: 10 } + ] + }, + { + source: { type: 'RequestGenerator', file: 'input.txt' }, + prepare: [ + { type: 'ServerTagger', host: 'localhost', port:_LOCAL_PORT_ }, + { type: 'QpsTagger', qps: 10 } + ] + } + ], + analyze: [ + { type: 'IgnoreBefore', time: 1.0 }, + { type: 'QpsAnalyzer' }, + { type: 'LatencyAnalyzer' }, + { type: 'RequestDumper' } + ] +} diff --git a/vbench/src/tests/benchmark_headers/.gitignore b/vbench/src/tests/benchmark_headers/.gitignore new file mode 100644 index 00000000000..ca919fcf3e5 --- /dev/null +++ b/vbench/src/tests/benchmark_headers/.gitignore @@ -0,0 +1 @@ +vbench_benchmark_headers_test_app diff --git a/vbench/src/tests/benchmark_headers/CMakeLists.txt b/vbench/src/tests/benchmark_headers/CMakeLists.txt new file mode 100644 index 00000000000..ab8f0bce02e --- /dev/null +++ b/vbench/src/tests/benchmark_headers/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_benchmark_headers_test_app + SOURCES + benchmark_headers_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_benchmark_headers_test_app COMMAND vbench_benchmark_headers_test_app) diff --git a/vbench/src/tests/benchmark_headers/FILES b/vbench/src/tests/benchmark_headers/FILES new file mode 100644 index 00000000000..d2beb7f953f --- /dev/null +++ b/vbench/src/tests/benchmark_headers/FILES @@ -0,0 +1 @@ +benchmark_headers_test.cpp diff --git a/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp new file mode 100644 index 00000000000..9da9a100613 --- /dev/null +++ b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("require that benchmark headers can be set") { + vbench::BenchmarkHeaders headers; + EXPECT_FALSE(headers.num_hits.is_set); + EXPECT_FALSE(headers.num_fasthits.is_set); + EXPECT_FALSE(headers.num_grouphits.is_set); + EXPECT_FALSE(headers.num_errors.is_set); + EXPECT_FALSE(headers.total_hit_count.is_set); + EXPECT_FALSE(headers.num_docsums.is_set); + EXPECT_FALSE(headers.query_hits.is_set); + EXPECT_FALSE(headers.query_offset.is_set); + EXPECT_FALSE(headers.search_time.is_set); + EXPECT_FALSE(headers.attr_time.is_set); + EXPECT_FALSE(headers.fill_time.is_set); + EXPECT_FALSE(headers.docs_searched.is_set); + EXPECT_FALSE(headers.nodes_searched.is_set); + EXPECT_FALSE(headers.full_coverage.is_set); + + headers.handleHeader("X-Yahoo-Vespa-NumHits", "1"); + headers.handleHeader("X-Yahoo-Vespa-NumFastHits", "2"); + headers.handleHeader("X-Yahoo-Vespa-NumGroupHits", "3"); + headers.handleHeader("X-Yahoo-Vespa-NumErrors", "4"); + headers.handleHeader("X-Yahoo-Vespa-TotalHitCount", "5"); + headers.handleHeader("X-Yahoo-Vespa-NumDocsums", "6"); + headers.handleHeader("X-Yahoo-Vespa-QueryHits", "7"); + headers.handleHeader("X-Yahoo-Vespa-QueryOffset", "8"); + headers.handleHeader("X-Yahoo-Vespa-SearchTime", "9"); + headers.handleHeader("X-Yahoo-Vespa-AttributeFetchTime" , "10"); + headers.handleHeader("X-Yahoo-Vespa-FillTime", "11"); + headers.handleHeader("X-Yahoo-Vespa-DocsSearched", "12"); + headers.handleHeader("X-Yahoo-Vespa-NodesSearched", "13"); + headers.handleHeader("X-Yahoo-Vespa-FullCoverage", "14"); + + EXPECT_TRUE(headers.num_hits.is_set); + EXPECT_TRUE(headers.num_fasthits.is_set); + EXPECT_TRUE(headers.num_grouphits.is_set); + EXPECT_TRUE(headers.num_errors.is_set); + EXPECT_TRUE(headers.total_hit_count.is_set); + EXPECT_TRUE(headers.num_docsums.is_set); + EXPECT_TRUE(headers.query_hits.is_set); + EXPECT_TRUE(headers.query_offset.is_set); + EXPECT_TRUE(headers.search_time.is_set); + EXPECT_TRUE(headers.attr_time.is_set); + EXPECT_TRUE(headers.fill_time.is_set); + EXPECT_TRUE(headers.docs_searched.is_set); + EXPECT_TRUE(headers.nodes_searched.is_set); + EXPECT_TRUE(headers.full_coverage.is_set); + + EXPECT_EQUAL(headers.num_hits.value, 1.0); + EXPECT_EQUAL(headers.num_fasthits.value, 2.0); + EXPECT_EQUAL(headers.num_grouphits.value, 3.0); + EXPECT_EQUAL(headers.num_errors.value, 4.0); + EXPECT_EQUAL(headers.total_hit_count.value, 5.0); + EXPECT_EQUAL(headers.num_docsums.value, 6.0); + EXPECT_EQUAL(headers.query_hits.value, 7.0); + EXPECT_EQUAL(headers.query_offset.value, 8.0); + EXPECT_EQUAL(headers.search_time.value, 9.0); + EXPECT_EQUAL(headers.attr_time.value, 10.0); + EXPECT_EQUAL(headers.fill_time.value, 11.0); + EXPECT_EQUAL(headers.docs_searched.value, 12.0); + EXPECT_EQUAL(headers.nodes_searched.value, 13.0); + EXPECT_EQUAL(headers.full_coverage.value, 14.0); +} + +TEST("require that benchmark headers can be converted to string") { + vbench::BenchmarkHeaders headers; + headers.handleHeader("X-Yahoo-Vespa-NumErrors", "4"); + headers.handleHeader("X-Yahoo-Vespa-TotalHitCount", "5"); + headers.handleHeader("X-Yahoo-Vespa-NumDocsums", "6"); + vbench::string result = headers.toString(); + fprintf(stderr, "%s", result.c_str()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/buffered_output/.gitignore b/vbench/src/tests/buffered_output/.gitignore new file mode 100644 index 00000000000..ba59218c6c5 --- /dev/null +++ b/vbench/src/tests/buffered_output/.gitignore @@ -0,0 +1 @@ +vbench_buffered_output_test_app diff --git a/vbench/src/tests/buffered_output/CMakeLists.txt b/vbench/src/tests/buffered_output/CMakeLists.txt new file mode 100644 index 00000000000..7dfc6c326a7 --- /dev/null +++ b/vbench/src/tests/buffered_output/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_buffered_output_test_app + SOURCES + buffered_output_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_buffered_output_test_app COMMAND vbench_buffered_output_test_app) diff --git a/vbench/src/tests/buffered_output/FILES b/vbench/src/tests/buffered_output/FILES new file mode 100644 index 00000000000..86ca0ac84c1 --- /dev/null +++ b/vbench/src/tests/buffered_output/FILES @@ -0,0 +1 @@ +buffered_output_test.cpp diff --git a/vbench/src/tests/buffered_output/buffered_output_test.cpp b/vbench/src/tests/buffered_output/buffered_output_test.cpp new file mode 100644 index 00000000000..79bd490c71c --- /dev/null +++ b/vbench/src/tests/buffered_output/buffered_output_test.cpp @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("buffered output") { + SimpleBuffer buffer; + { // produce data + BufferedOutput dst(buffer, 3); + dst.append('a').append('b').append('c').append('\n'); + dst.append("foo bar").append('\n'); + dst.append(string("str")).append('\n'); + dst.printf("%d + %d = %d\n", 2, 2, 4); + } + { // verify data + LineReader src(buffer, 3); + string str; + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("abc", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("foo bar", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("str", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("2 + 2 = 4", str); + EXPECT_TRUE(!src.readLine(str)); + EXPECT_EQUAL("", str); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/byte_input/.gitignore b/vbench/src/tests/byte_input/.gitignore new file mode 100644 index 00000000000..350435ffda4 --- /dev/null +++ b/vbench/src/tests/byte_input/.gitignore @@ -0,0 +1 @@ +vbench_byte_input_test_app diff --git a/vbench/src/tests/byte_input/CMakeLists.txt b/vbench/src/tests/byte_input/CMakeLists.txt new file mode 100644 index 00000000000..5838ed58d76 --- /dev/null +++ b/vbench/src/tests/byte_input/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_byte_input_test_app + SOURCES + byte_input_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_byte_input_test_app COMMAND vbench_byte_input_test_app) diff --git a/vbench/src/tests/byte_input/FILES b/vbench/src/tests/byte_input/FILES new file mode 100644 index 00000000000..c625277f587 --- /dev/null +++ b/vbench/src/tests/byte_input/FILES @@ -0,0 +1 @@ +byte_input_test.cpp diff --git a/vbench/src/tests/byte_input/byte_input_test.cpp b/vbench/src/tests/byte_input/byte_input_test.cpp new file mode 100644 index 00000000000..251b16449be --- /dev/null +++ b/vbench/src/tests/byte_input/byte_input_test.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("byte input") { + SimpleBuffer buffer; + { + BufferedOutput out(buffer, 10); + out.append("abcdefgh"); + } + EXPECT_EQUAL(8u, buffer.get().size); + { + ByteInput in(buffer, 3); + EXPECT_EQUAL('a', in.get()); + EXPECT_EQUAL('b', in.get()); + EXPECT_EQUAL('c', in.get()); + EXPECT_EQUAL('d', in.get()); + } + EXPECT_EQUAL(4u, buffer.get().size); + { + ByteInput in(buffer, 3); + EXPECT_EQUAL('e', in.get()); + EXPECT_EQUAL('f', in.get()); + EXPECT_EQUAL('g', in.get()); + EXPECT_EQUAL('h', in.get()); + EXPECT_EQUAL(-1, in.get()); + EXPECT_EQUAL(-1, in.get()); + } + EXPECT_EQUAL(0u, buffer.get().size); + { + ByteInput in(buffer, 3); + EXPECT_EQUAL(-1, in.get()); + } + EXPECT_EQUAL(0u, buffer.get().size); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/create-test.sh b/vbench/src/tests/create-test.sh new file mode 100755 index 00000000000..793bfd9ef48 --- /dev/null +++ b/vbench/src/tests/create-test.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +gen_project_file() { + echo "generating '$1' ..." + echo "APPLICATION ${test}_test" > $1 + echo "OBJS ${test}_test" >> $1 + echo "LIBS vbench/test/test" >> $1 + echo "LIBS vbench/vbench" >> $1 + echo "EXTERNALLIBS staging_vespalib" >> $1 + echo "" >> $1 + echo "CUSTOMMAKE" >> $1 + echo "test: ${test}_test" >> $1 + echo -e "\t\$(LDL) \$(VALGRIND) ./${test}_test" >> $1 +} + +gen_source() { + echo "generating '$1' ..." + echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1 + echo "#include <vbench/test/all.h>" >> $1 + echo "" >> $1 + echo "using namespace vbench;" >> $1 + echo "" >> $1 + echo "TEST(\"foo\") {" >> $1 + echo " EXPECT_TRUE(true);" >> $1 + echo "}" >> $1 + echo "" >> $1 + echo "TEST_MAIN() { TEST_RUN_ALL(); }" >> $1 +} + +gen_file_list() { + echo "generating '$1' ..." + echo "${test}_test.cpp" > $1 +} + +if [ $# -ne 1 ]; then + echo "usage: $0 <name>" + echo " name: name of the test to create" + exit 1 +fi + +test=$1 +if [ -e $test ]; then + echo "$test already present, don't want to mess it up..." + exit 1 +fi + +echo "creating directory '$test' ..." +mkdir -p $test || exit 1 +cd $test || exit 1 +test=`basename $test` + +gen_project_file fastos.project +gen_source ${test}_test.cpp +gen_file_list FILES diff --git a/vbench/src/tests/dispatcher/.gitignore b/vbench/src/tests/dispatcher/.gitignore new file mode 100644 index 00000000000..6b038801688 --- /dev/null +++ b/vbench/src/tests/dispatcher/.gitignore @@ -0,0 +1 @@ +vbench_dispatcher_test_app diff --git a/vbench/src/tests/dispatcher/CMakeLists.txt b/vbench/src/tests/dispatcher/CMakeLists.txt new file mode 100644 index 00000000000..0cdc9ffb2d3 --- /dev/null +++ b/vbench/src/tests/dispatcher/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_dispatcher_test_app + SOURCES + dispatcher_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_dispatcher_test_app COMMAND vbench_dispatcher_test_app) diff --git a/vbench/src/tests/dispatcher/FILES b/vbench/src/tests/dispatcher/FILES new file mode 100644 index 00000000000..8e526a62463 --- /dev/null +++ b/vbench/src/tests/dispatcher/FILES @@ -0,0 +1 @@ +dispatcher_test.cpp diff --git a/vbench/src/tests/dispatcher/dispatcher_test.cpp b/vbench/src/tests/dispatcher/dispatcher_test.cpp new file mode 100644 index 00000000000..b4afafdaed5 --- /dev/null +++ b/vbench/src/tests/dispatcher/dispatcher_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +struct MyHandler : public Handler<int> { + int value; + MyHandler() : value(-1) {} + virtual void handle(std::unique_ptr<int> v) { value = (v.get() != 0) ? *v : 0; } +}; + +struct Fetcher : public vespalib::Runnable { + Provider<int> &provider; + Handler<int> &handler; + Fetcher(Provider<int> &p, Handler<int> &h) : provider(p), handler(h) {} + virtual void run() { handler.handle(provider.provide()); } +}; + +TEST("dispatcher") { + MyHandler dropped; + MyHandler handler1; + MyHandler handler2; + Dispatcher<int> dispatcher(dropped); + Fetcher fetcher1(dispatcher, handler1); + Fetcher fetcher2(dispatcher, handler2); + vespalib::Thread thread1(fetcher1); + vespalib::Thread thread2(fetcher2); + thread1.start(); + EXPECT_TRUE(dispatcher.waitForThreads(1, 512)); + thread2.start(); + EXPECT_TRUE(dispatcher.waitForThreads(2, 512)); + EXPECT_EQUAL(-1, dropped.value); + EXPECT_EQUAL(-1, handler1.value); + EXPECT_EQUAL(-1, handler2.value); + dispatcher.handle(std::unique_ptr<int>(new int(1))); + dispatcher.handle(std::unique_ptr<int>(new int(2))); + dispatcher.handle(std::unique_ptr<int>(new int(3))); + thread1.join(); + thread2.join(); + EXPECT_EQUAL(3, dropped.value); + EXPECT_EQUAL(2, handler1.value); + EXPECT_EQUAL(1, handler2.value); + dispatcher.close(); + { + dispatcher.handle(std::unique_ptr<int>(new int(4))); + EXPECT_EQUAL(3, dropped.value); + MyHandler handler3; + Fetcher fetcher3(dispatcher, handler3); + EXPECT_EQUAL(-1, handler3.value); + fetcher3.run(); + EXPECT_EQUAL(0, handler3.value); + } +} + +TEST_FF("dispatcher poll timeout", MyHandler(), Dispatcher<int>(f1)) { + EXPECT_FALSE(f2.waitForThreads(1, 2)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/dropped_tagger/.gitignore b/vbench/src/tests/dropped_tagger/.gitignore new file mode 100644 index 00000000000..71ef5c44a9d --- /dev/null +++ b/vbench/src/tests/dropped_tagger/.gitignore @@ -0,0 +1 @@ +vbench_dropped_tagger_test_app diff --git a/vbench/src/tests/dropped_tagger/CMakeLists.txt b/vbench/src/tests/dropped_tagger/CMakeLists.txt new file mode 100644 index 00000000000..2fc41839d5d --- /dev/null +++ b/vbench/src/tests/dropped_tagger/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_dropped_tagger_test_app + SOURCES + dropped_tagger_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_dropped_tagger_test_app COMMAND vbench_dropped_tagger_test_app) diff --git a/vbench/src/tests/dropped_tagger/FILES b/vbench/src/tests/dropped_tagger/FILES new file mode 100644 index 00000000000..535abf7ee58 --- /dev/null +++ b/vbench/src/tests/dropped_tagger/FILES @@ -0,0 +1 @@ +dropped_tagger_test.cpp diff --git a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp new file mode 100644 index 00000000000..60cbf6d03a7 --- /dev/null +++ b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_FF("dropped tagger", RequestReceptor(), DroppedTagger(f1)) { + Request::UP req(new Request()); + EXPECT_EQUAL(Request::STATUS_OK, req->status()); + f2.handle(std::move(req)); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_EQUAL(Request::STATUS_DROPPED, f1.request->status()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/handler_thread/.gitignore b/vbench/src/tests/handler_thread/.gitignore new file mode 100644 index 00000000000..41890930291 --- /dev/null +++ b/vbench/src/tests/handler_thread/.gitignore @@ -0,0 +1 @@ +vbench_handler_thread_test_app diff --git a/vbench/src/tests/handler_thread/CMakeLists.txt b/vbench/src/tests/handler_thread/CMakeLists.txt new file mode 100644 index 00000000000..ee69cc594de --- /dev/null +++ b/vbench/src/tests/handler_thread/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_handler_thread_test_app + SOURCES + handler_thread_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_handler_thread_test_app COMMAND vbench_handler_thread_test_app) diff --git a/vbench/src/tests/handler_thread/FILES b/vbench/src/tests/handler_thread/FILES new file mode 100644 index 00000000000..978aa147772 --- /dev/null +++ b/vbench/src/tests/handler_thread/FILES @@ -0,0 +1 @@ +handler_thread_test.cpp diff --git a/vbench/src/tests/handler_thread/handler_thread_test.cpp b/vbench/src/tests/handler_thread/handler_thread_test.cpp new file mode 100644 index 00000000000..43f0c4d4a1b --- /dev/null +++ b/vbench/src/tests/handler_thread/handler_thread_test.cpp @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +struct MyHandler : Handler<int> { + std::vector<int> values; + virtual void handle(std::unique_ptr<int> value) { + values.push_back(*value); + vespalib::Thread::sleep(10); // for improved coverage + } +}; + +TEST("handler thread") { + MyHandler handler; + HandlerThread<int> th(handler); + th.handle(std::unique_ptr<int>(new int(1))); + th.handle(std::unique_ptr<int>(new int(2))); + th.handle(std::unique_ptr<int>(new int(3))); + th.join(); + th.handle(std::unique_ptr<int>(new int(4))); + th.handle(std::unique_ptr<int>(new int(5))); + ASSERT_EQUAL(3u, handler.values.size()); + EXPECT_EQUAL(1, handler.values[0]); + EXPECT_EQUAL(2, handler.values[1]); + EXPECT_EQUAL(3, handler.values[2]); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/hex_number/.gitignore b/vbench/src/tests/hex_number/.gitignore new file mode 100644 index 00000000000..4fef44ee162 --- /dev/null +++ b/vbench/src/tests/hex_number/.gitignore @@ -0,0 +1 @@ +vbench_hex_number_test_app diff --git a/vbench/src/tests/hex_number/CMakeLists.txt b/vbench/src/tests/hex_number/CMakeLists.txt new file mode 100644 index 00000000000..37875434e5e --- /dev/null +++ b/vbench/src/tests/hex_number/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_hex_number_test_app + SOURCES + hex_number_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_hex_number_test_app COMMAND vbench_hex_number_test_app) diff --git a/vbench/src/tests/hex_number/FILES b/vbench/src/tests/hex_number/FILES new file mode 100644 index 00000000000..488979a3169 --- /dev/null +++ b/vbench/src/tests/hex_number/FILES @@ -0,0 +1 @@ +hex_number_test.cpp diff --git a/vbench/src/tests/hex_number/hex_number_test.cpp b/vbench/src/tests/hex_number/hex_number_test.cpp new file mode 100644 index 00000000000..12833814979 --- /dev/null +++ b/vbench/src/tests/hex_number/hex_number_test.cpp @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("hex number") { + EXPECT_EQUAL(0u, HexNumber("").value()); + EXPECT_EQUAL(0u, HexNumber("").length()); + EXPECT_EQUAL(0u, HexNumber("000").value()); + EXPECT_EQUAL(3u, HexNumber("000").length()); + EXPECT_EQUAL(0x1ee7u, HexNumber("1ee7").value()); + EXPECT_EQUAL(0x0123456789abcdefU, HexNumber("0123456789abcdef").value()); + EXPECT_EQUAL(0xfedcba9876543210U, HexNumber("FEDCBA9876543210").value()); + EXPECT_EQUAL(16u, HexNumber("0123456789ABCDEF").length()); + EXPECT_EQUAL(0xdefu, HexNumber("defghijk").value()); + EXPECT_EQUAL(3u, HexNumber("defghijk").length()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/http_client/.gitignore b/vbench/src/tests/http_client/.gitignore new file mode 100644 index 00000000000..7e86463762c --- /dev/null +++ b/vbench/src/tests/http_client/.gitignore @@ -0,0 +1 @@ +vbench_http_client_test_app diff --git a/vbench/src/tests/http_client/CMakeLists.txt b/vbench/src/tests/http_client/CMakeLists.txt new file mode 100644 index 00000000000..3a01f9712a3 --- /dev/null +++ b/vbench/src/tests/http_client/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_http_client_test_app + SOURCES + http_client_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_http_client_test_app COMMAND vbench_http_client_test_app) diff --git a/vbench/src/tests/http_client/FILES b/vbench/src/tests/http_client/FILES new file mode 100644 index 00000000000..a89979fbcba --- /dev/null +++ b/vbench/src/tests/http_client/FILES @@ -0,0 +1 @@ +http_client_test.cpp diff --git a/vbench/src/tests/http_client/http_client_test.cpp b/vbench/src/tests/http_client/http_client_test.cpp new file mode 100644 index 00000000000..ce763e09b7d --- /dev/null +++ b/vbench/src/tests/http_client/http_client_test.cpp @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +void checkMemory(const string &expect, const Memory &mem) { + EXPECT_EQUAL(expect, string(mem.data, mem.size)); +} + +bool endsWith(const Memory &mem, const string &str) { + return (mem.size < str.size()) ? false + : (strncmp(mem.data + mem.size - str.size(), str.data(), str.size()) == 0); +} + +void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { + ByteInput in(input, 256); + while (!endsWith(buffer.get(), end)) { + int c = in.get(); + if (c < 0) { + return; + } + buffer.reserve(1).data[0] = c; + buffer.commit(1, 0); + } +} + +TEST_MT_F("verify request", 2, ServerSocket()) { + if (thread_id == 0) { + SimpleBuffer expect; + { + BufferedOutput dst(expect, 256); + dst.append("GET /this/is/the/url HTTP/1.1\r\n"); + dst.append("Host: localhost\r\n"); + dst.append("User-Agent: vbench\r\n"); + dst.append("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); + dst.append("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); + dst.append("\r\n"); + } + SimpleBuffer actual; + Stream::UP stream = f1.accept(); + ASSERT_TRUE(stream.get() != 0); + readUntil(*stream, actual, "\r\n\r\n"); + EXPECT_TRUE(expect == actual); + } else { + SimpleHttpResultHandler handler; + HttpClient::fetch(ServerSpec("localhost", f1.port()), + "/this/is/the/url", handler); + } +} + +TEST_MT_F("verify connection close", 2, ServerSocket()) { + if (thread_id == 0) { + Stream::UP stream = f1.accept(); + SimpleBuffer ignore; + readUntil(*stream, ignore, "\r\n\r\n"); + BufferedOutput out(*stream, 256); + out.append("HTTP/1.0 200\r\n"); + out.append("\r\n"); + out.append("data"); + } else { + SimpleHttpResultHandler handler; + HttpClient::fetch(ServerSpec("localhost", f1.port()), + "/foo", handler); + EXPECT_EQUAL(0u, handler.failures().size()); + EXPECT_EQUAL(0u, handler.headers().size()); + TEST_DO(checkMemory("data", handler.content())); + } +} + +TEST_MT_F("verify content length", 2, ServerSocket()) { + if (thread_id == 0) { + Stream::UP stream = f1.accept(); + SimpleBuffer ignore; + readUntil(*stream, ignore, "\r\n\r\n"); + BufferedOutput out(*stream, 256); + out.append("HTTP/1.1 200\r\n"); + out.append("content-length: 4\r\n"); + out.append("\r\n"); + out.append("data"); + } else { + SimpleHttpResultHandler handler; + HttpClient::fetch(ServerSpec("localhost", f1.port()), + "/foo", handler); + EXPECT_EQUAL(0u, handler.failures().size()); + EXPECT_EQUAL(1u, handler.headers().size()); + TEST_DO(checkMemory("data", handler.content())); + } +} + +TEST_MT_F("verify chunked encoding", 2, ServerSocket()) { + if (thread_id == 0) { + Stream::UP stream = f1.accept(); + SimpleBuffer ignore; + readUntil(*stream, ignore, "\r\n\r\n"); + BufferedOutput out(*stream, 256); + out.append("HTTP/1.1 200\r\n"); + out.append("transfer-encoding: chunked\r\n"); + out.append("\r\n"); + out.append("2\r\n"); + out.append("da\r\n"); + out.append("2\r\n"); + out.append("ta\r\n"); + out.append("0\r\n"); + out.append("\r\n"); + } else { + SimpleHttpResultHandler handler; + HttpClient::fetch(ServerSpec("localhost", f1.port()), + "/foo", handler); + if (handler.failures().size() > 0) { + fprintf(stderr, "failure: %s\n", handler.failures()[0].c_str()); + } + EXPECT_EQUAL(0u, handler.failures().size()); + EXPECT_EQUAL(1u, handler.headers().size()); + TEST_DO(checkMemory("data", handler.content())); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/http_connection/.gitignore b/vbench/src/tests/http_connection/.gitignore new file mode 100644 index 00000000000..9829af0bdba --- /dev/null +++ b/vbench/src/tests/http_connection/.gitignore @@ -0,0 +1 @@ +vbench_http_connection_test_app diff --git a/vbench/src/tests/http_connection/CMakeLists.txt b/vbench/src/tests/http_connection/CMakeLists.txt new file mode 100644 index 00000000000..8d33e8fb1e3 --- /dev/null +++ b/vbench/src/tests/http_connection/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_http_connection_test_app + SOURCES + http_connection_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_http_connection_test_app COMMAND vbench_http_connection_test_app) diff --git a/vbench/src/tests/http_connection/FILES b/vbench/src/tests/http_connection/FILES new file mode 100644 index 00000000000..de51725c9ac --- /dev/null +++ b/vbench/src/tests/http_connection/FILES @@ -0,0 +1 @@ +http_connection_test.cpp diff --git a/vbench/src/tests/http_connection/http_connection_test.cpp b/vbench/src/tests/http_connection/http_connection_test.cpp new file mode 100644 index 00000000000..dc03a9e7e8e --- /dev/null +++ b/vbench/src/tests/http_connection/http_connection_test.cpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("http connection") { + ServerSocket serverSocket; + HttpConnection client(ServerSpec("localhost", serverSocket.port())); + Stream::UP server = serverSocket.accept(); + EXPECT_TRUE(client.fresh()); + EXPECT_EQUAL("localhost", client.server().host); + EXPECT_FALSE(client.mayReuse(0.1)); // still fresh + client.touch(5.0); + EXPECT_FALSE(client.fresh()); + EXPECT_TRUE(client.mayReuse(5.1)); + server.reset(); + client.stream().obtain(1, 1); // trigger eof + EXPECT_FALSE(client.mayReuse(5.1)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/http_connection_pool/.gitignore b/vbench/src/tests/http_connection_pool/.gitignore new file mode 100644 index 00000000000..f9601f89d39 --- /dev/null +++ b/vbench/src/tests/http_connection_pool/.gitignore @@ -0,0 +1 @@ +vbench_http_connection_pool_test_app diff --git a/vbench/src/tests/http_connection_pool/CMakeLists.txt b/vbench/src/tests/http_connection_pool/CMakeLists.txt new file mode 100644 index 00000000000..6ebac01b954 --- /dev/null +++ b/vbench/src/tests/http_connection_pool/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_http_connection_pool_test_app + SOURCES + http_connection_pool_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_http_connection_pool_test_app COMMAND vbench_http_connection_pool_test_app) diff --git a/vbench/src/tests/http_connection_pool/FILES b/vbench/src/tests/http_connection_pool/FILES new file mode 100644 index 00000000000..1bc5c1cc463 --- /dev/null +++ b/vbench/src/tests/http_connection_pool/FILES @@ -0,0 +1 @@ +http_connection_pool_test.cpp diff --git a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp new file mode 100644 index 00000000000..06f158e7b84 --- /dev/null +++ b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> +#include <vespa/vespalib/util/sync.h> + +using namespace vbench; +using vespalib::CountDownLatch; + +TEST_MT_F("http connection pool", 2, ServerSocket()) { + if (thread_id == 0) { + for (; f1.accept().get() != 0; ) {} + } else { + Timer timer; + HttpConnection::UP conn; + HttpConnectionPool pool(timer); + conn = pool.getConnection(ServerSpec("localhost", f1.port())); + EXPECT_TRUE(conn.get() != 0); + pool.putConnection(std::move(conn)); + EXPECT_TRUE(conn.get() == 0); + conn = pool.getConnection(ServerSpec("localhost", f1.port())); + EXPECT_TRUE(conn.get() != 0); + conn->stream().obtain(1, 1); // trigger eof + pool.putConnection(std::move(conn)); + EXPECT_TRUE(conn.get() == 0); + conn = pool.getConnection(ServerSpec("localhost", f1.port())); + EXPECT_TRUE(conn.get() != 0); + f1.close(); + } +} + +TEST_MT_FFFF("stress http connection pool", 256, ServerSocket(), Timer(), HttpConnectionPool(f2), + CountDownLatch(num_threads-2)) +{ + if (thread_id == 0) { + for (; f1.accept().get() != 0; ) {} + } else { + while (f2.sample() < 5.0) { + HttpConnection::UP conn = f3.getConnection(ServerSpec("localhost", f1.port())); + EXPECT_TRUE(conn.get() != 0); + if (thread_id > (num_threads / 2)) { + conn->stream().obtain(1, 1); // trigger eof + } + f3.putConnection(std::move(conn)); + EXPECT_TRUE(conn.get() == 0); + } + if (thread_id == 1) { + f4.await(); + f1.close(); + } else { + f4.countDown(); + } + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/input_file_reader/.gitignore b/vbench/src/tests/input_file_reader/.gitignore new file mode 100644 index 00000000000..a8f6e9d2af4 --- /dev/null +++ b/vbench/src/tests/input_file_reader/.gitignore @@ -0,0 +1 @@ +vbench_input_file_reader_test_app diff --git a/vbench/src/tests/input_file_reader/CMakeLists.txt b/vbench/src/tests/input_file_reader/CMakeLists.txt new file mode 100644 index 00000000000..ac4ded756f9 --- /dev/null +++ b/vbench/src/tests/input_file_reader/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_input_file_reader_test_app + SOURCES + input_file_reader_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_input_file_reader_test_app COMMAND vbench_input_file_reader_test_app) diff --git a/vbench/src/tests/input_file_reader/FILES b/vbench/src/tests/input_file_reader/FILES new file mode 100644 index 00000000000..46b97e37a9d --- /dev/null +++ b/vbench/src/tests/input_file_reader/FILES @@ -0,0 +1,3 @@ +input_file_reader_test.cpp +simple_test_input.txt +hard_test_input.txt diff --git a/vbench/src/tests/input_file_reader/hard_test_input.txt b/vbench/src/tests/input_file_reader/hard_test_input.txt new file mode 100644 index 00000000000..8144450a5c7 --- /dev/null +++ b/vbench/src/tests/input_file_reader/hard_test_input.txt @@ -0,0 +1,5 @@ +foo + +bar
+baz +
\ No newline at end of file diff --git a/vbench/src/tests/input_file_reader/input_file_reader_test.cpp b/vbench/src/tests/input_file_reader/input_file_reader_test.cpp new file mode 100644 index 00000000000..2cf59dff1ec --- /dev/null +++ b/vbench/src/tests/input_file_reader/input_file_reader_test.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("input file reader") { + { + InputFileReader reader("not_found.txt"); + EXPECT_TRUE(reader.tainted()); + } + { + InputFileReader reader("simple_test_input.txt"); + EXPECT_TRUE(!reader.tainted()); + string line; + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("foo", line); + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("bar", line); + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("baz", line); + EXPECT_TRUE(!reader.readLine(line)); + TEST_FLUSH(); + } + { + InputFileReader reader("hard_test_input.txt"); + EXPECT_TRUE(!reader.tainted()); + string line; + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("foo", line); + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("bar", line); + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("baz", line); + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("\r", line); + EXPECT_TRUE(!reader.readLine(line)); + TEST_FLUSH(); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/input_file_reader/simple_test_input.txt b/vbench/src/tests/input_file_reader/simple_test_input.txt new file mode 100644 index 00000000000..86e041dad66 --- /dev/null +++ b/vbench/src/tests/input_file_reader/simple_test_input.txt @@ -0,0 +1,3 @@ +foo +bar +baz diff --git a/vbench/src/tests/input_reader/.gitignore b/vbench/src/tests/input_reader/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vbench/src/tests/input_reader/.gitignore diff --git a/vbench/src/tests/latency_analyzer/.gitignore b/vbench/src/tests/latency_analyzer/.gitignore new file mode 100644 index 00000000000..c82c48eb1f6 --- /dev/null +++ b/vbench/src/tests/latency_analyzer/.gitignore @@ -0,0 +1 @@ +vbench_latency_analyzer_test_app diff --git a/vbench/src/tests/latency_analyzer/CMakeLists.txt b/vbench/src/tests/latency_analyzer/CMakeLists.txt new file mode 100644 index 00000000000..01fbb5c3d5c --- /dev/null +++ b/vbench/src/tests/latency_analyzer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_latency_analyzer_test_app + SOURCES + latency_analyzer_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_latency_analyzer_test_app COMMAND vbench_latency_analyzer_test_app) diff --git a/vbench/src/tests/latency_analyzer/FILES b/vbench/src/tests/latency_analyzer/FILES new file mode 100644 index 00000000000..4d90bfb352d --- /dev/null +++ b/vbench/src/tests/latency_analyzer/FILES @@ -0,0 +1 @@ +latency_analyzer_test.cpp diff --git a/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp new file mode 100644 index 00000000000..39d12420415 --- /dev/null +++ b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +void post(double latency, Handler<Request> &handler, + double startTime = 0.0, Request::Status status = Request::STATUS_OK) +{ + Request::UP req(new Request()); + req->status(status).startTime(startTime).endTime(startTime + latency); + handler.handle(std::move(req)); +} + +TEST_FF("require that only OK requests are counted", RequestSink(), LatencyAnalyzer(f1)) { + post(1.0, f2); + post(2.0, f2, 3.0); + post(10.0, f2, 0.0, Request::STATUS_DROPPED); + post(20.0, f2, 0.0, Request::STATUS_FAILED); + EXPECT_APPROX(1.0, f2.getStats().min, 10e-6); + EXPECT_APPROX(1.5, f2.getStats().avg, 10e-6); + EXPECT_APPROX(2.0, f2.getStats().max, 10e-6); +} + +TEST_FF("verify percentiles", RequestSink(), LatencyAnalyzer(f1)) { + for (size_t i = 0; i <= 10000; ++i) { + post(0.001 * i, f2); + } + LatencyAnalyzer::Stats stats = f2.getStats(); + EXPECT_APPROX(5.0, stats.per50, 10e-6); + EXPECT_APPROX(9.5, stats.per95, 10e-6); + EXPECT_APPROX(9.9, stats.per99, 10e-6); + fprintf(stderr, "%s", stats.toString().c_str()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/line_reader/.gitignore b/vbench/src/tests/line_reader/.gitignore new file mode 100644 index 00000000000..01fa87fd5f8 --- /dev/null +++ b/vbench/src/tests/line_reader/.gitignore @@ -0,0 +1 @@ +vbench_line_reader_test_app diff --git a/vbench/src/tests/line_reader/CMakeLists.txt b/vbench/src/tests/line_reader/CMakeLists.txt new file mode 100644 index 00000000000..5235d4dfe12 --- /dev/null +++ b/vbench/src/tests/line_reader/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_line_reader_test_app + SOURCES + line_reader_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_line_reader_test_app COMMAND vbench_line_reader_test_app) diff --git a/vbench/src/tests/line_reader/FILES b/vbench/src/tests/line_reader/FILES new file mode 100644 index 00000000000..2611646033f --- /dev/null +++ b/vbench/src/tests/line_reader/FILES @@ -0,0 +1 @@ +line_reader_test.cpp diff --git a/vbench/src/tests/line_reader/line_reader_test.cpp b/vbench/src/tests/line_reader/line_reader_test.cpp new file mode 100644 index 00000000000..631b2ffc765 --- /dev/null +++ b/vbench/src/tests/line_reader/line_reader_test.cpp @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("line reader") { + SimpleBuffer buffer; + { + BufferedOutput dst(buffer, 64); + dst.append("foo\n"); + dst.append("bar\r\n"); + dst.append("\n"); + dst.append("\rbaz\n"); + dst.append("\r\n"); + dst.append("zzz"); + } + { + LineReader src(buffer, 3); + string str; + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("foo", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("bar", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("\rbaz", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("", str); + EXPECT_TRUE(src.readLine(str)); + EXPECT_EQUAL("zzz", str); + EXPECT_TRUE(!src.readLine(str)); + EXPECT_EQUAL("", str); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/mapped_file_input/.gitignore b/vbench/src/tests/mapped_file_input/.gitignore new file mode 100644 index 00000000000..99fcc66eeab --- /dev/null +++ b/vbench/src/tests/mapped_file_input/.gitignore @@ -0,0 +1 @@ +vbench_mapped_file_input_test_app diff --git a/vbench/src/tests/mapped_file_input/CMakeLists.txt b/vbench/src/tests/mapped_file_input/CMakeLists.txt new file mode 100644 index 00000000000..de7cbb7c89d --- /dev/null +++ b/vbench/src/tests/mapped_file_input/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_mapped_file_input_test_app + SOURCES + mapped_file_input_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_mapped_file_input_test_app COMMAND vbench_mapped_file_input_test_app) diff --git a/vbench/src/tests/mapped_file_input/FILES b/vbench/src/tests/mapped_file_input/FILES new file mode 100644 index 00000000000..55eddab2a7a --- /dev/null +++ b/vbench/src/tests/mapped_file_input/FILES @@ -0,0 +1,2 @@ +mapped_file_input_test.cpp +file.txt diff --git a/vbench/src/tests/mapped_file_input/file.txt b/vbench/src/tests/mapped_file_input/file.txt new file mode 100644 index 00000000000..dd59d098638 --- /dev/null +++ b/vbench/src/tests/mapped_file_input/file.txt @@ -0,0 +1 @@ +file content diff --git a/vbench/src/tests/mapped_file_input/mapped_file_input_test.cpp b/vbench/src/tests/mapped_file_input/mapped_file_input_test.cpp new file mode 100644 index 00000000000..5044f5401e1 --- /dev/null +++ b/vbench/src/tests/mapped_file_input/mapped_file_input_test.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("mapped file input") { + { + MappedFileInput file("not_found.txt"); + EXPECT_TRUE(file.tainted()); + } + { + MappedFileInput file("file.txt"); + EXPECT_TRUE(!file.tainted()); + LineReader reader(file, 3); + string line; + EXPECT_TRUE(reader.readLine(line)); + EXPECT_EQUAL("file content", line); + EXPECT_TRUE(!reader.readLine(line)); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/memory/.gitignore b/vbench/src/tests/memory/.gitignore new file mode 100644 index 00000000000..e1fd81a3e7c --- /dev/null +++ b/vbench/src/tests/memory/.gitignore @@ -0,0 +1 @@ +vbench_memory_test_app diff --git a/vbench/src/tests/memory/CMakeLists.txt b/vbench/src/tests/memory/CMakeLists.txt new file mode 100644 index 00000000000..bcdaf55f4a1 --- /dev/null +++ b/vbench/src/tests/memory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_memory_test_app + SOURCES + memory_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_memory_test_app COMMAND vbench_memory_test_app) diff --git a/vbench/src/tests/memory/FILES b/vbench/src/tests/memory/FILES new file mode 100644 index 00000000000..3627eec5ec7 --- /dev/null +++ b/vbench/src/tests/memory/FILES @@ -0,0 +1 @@ +memory_test.cpp diff --git a/vbench/src/tests/memory/memory_test.cpp b/vbench/src/tests/memory/memory_test.cpp new file mode 100644 index 00000000000..9504914d191 --- /dev/null +++ b/vbench/src/tests/memory/memory_test.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("empty memory") { + Memory m; + EXPECT_EQUAL((const char*)0, m.data); + EXPECT_EQUAL(0u, m.size); +} + +TEST("from string") { + string str("foo"); + Memory m1(str); + Memory m2 = str; + EXPECT_EQUAL(str.data(), m1.data); + EXPECT_EQUAL(str.size(), m1.size); + EXPECT_EQUAL(str.data(), m2.data); + EXPECT_EQUAL(str.size(), m2.size); +} + +TEST("from cstring") { + const char *str = "foo"; + Memory m1(str); + Memory m2 = str; + EXPECT_EQUAL(str, m1.data); + EXPECT_EQUAL(strlen(str), m1.size); + EXPECT_EQUAL(str, m2.data); + EXPECT_EQUAL(strlen(str), m2.size); +} + +TEST("from ptr and len") { + string str("foo"); + Memory m1(str.data(), str.size()); + EXPECT_EQUAL(str.data(), m1.data); + EXPECT_EQUAL(str.size(), m1.size); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/qps_analyzer/.gitignore b/vbench/src/tests/qps_analyzer/.gitignore new file mode 100644 index 00000000000..0651de34c3c --- /dev/null +++ b/vbench/src/tests/qps_analyzer/.gitignore @@ -0,0 +1 @@ +vbench_qps_analyzer_test_app diff --git a/vbench/src/tests/qps_analyzer/CMakeLists.txt b/vbench/src/tests/qps_analyzer/CMakeLists.txt new file mode 100644 index 00000000000..b4167511665 --- /dev/null +++ b/vbench/src/tests/qps_analyzer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_qps_analyzer_test_app + SOURCES + qps_analyzer_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_qps_analyzer_test_app COMMAND vbench_qps_analyzer_test_app) diff --git a/vbench/src/tests/qps_analyzer/FILES b/vbench/src/tests/qps_analyzer/FILES new file mode 100644 index 00000000000..978efc5eafd --- /dev/null +++ b/vbench/src/tests/qps_analyzer/FILES @@ -0,0 +1 @@ +qps_analyzer_test.cpp diff --git a/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp new file mode 100644 index 00000000000..4de340fcf29 --- /dev/null +++ b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +void post(double endTime, Handler<Request> &handler, + Request::Status status = Request::STATUS_OK) +{ + Request::UP req(new Request()); + req->status(status).endTime(endTime); + handler.handle(std::move(req)); +} + +TEST_FF("simulate 100 qps", RequestSink(), QpsAnalyzer(f1)) { + for (size_t i = 1; i < 10000; ++i) { + post(i * 0.01, f2); + post(i * 0.01, f2, Request::STATUS_DROPPED); + post(i * 0.01, f2, Request::STATUS_FAILED); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/qps_tagger/.gitignore b/vbench/src/tests/qps_tagger/.gitignore new file mode 100644 index 00000000000..91701521f51 --- /dev/null +++ b/vbench/src/tests/qps_tagger/.gitignore @@ -0,0 +1 @@ +vbench_qps_tagger_test_app diff --git a/vbench/src/tests/qps_tagger/CMakeLists.txt b/vbench/src/tests/qps_tagger/CMakeLists.txt new file mode 100644 index 00000000000..2538de07b8f --- /dev/null +++ b/vbench/src/tests/qps_tagger/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_qps_tagger_test_app + SOURCES + qps_tagger_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_qps_tagger_test_app COMMAND vbench_qps_tagger_test_app) diff --git a/vbench/src/tests/qps_tagger/FILES b/vbench/src/tests/qps_tagger/FILES new file mode 100644 index 00000000000..9fbf5764e52 --- /dev/null +++ b/vbench/src/tests/qps_tagger/FILES @@ -0,0 +1 @@ +qps_tagger_test.cpp diff --git a/vbench/src/tests/qps_tagger/qps_tagger_test.cpp b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp new file mode 100644 index 00000000000..34ae5082298 --- /dev/null +++ b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_FF("qps tagger", RequestReceptor(), QpsTagger(2.0, f1)) { + f2.handle(Request::UP(new Request())); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_APPROX(0.0, f1.request->scheduledTime(), 10e-6); + f2.handle(Request::UP(new Request())); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_APPROX(0.5, f1.request->scheduledTime(), 10e-6); + f2.handle(Request::UP(new Request())); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_APPROX(1.0, f1.request->scheduledTime(), 10e-6); + f2.handle(Request::UP(new Request())); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_APPROX(1.5, f1.request->scheduledTime(), 10e-6); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/request_dumper/.gitignore b/vbench/src/tests/request_dumper/.gitignore new file mode 100644 index 00000000000..dec2573b121 --- /dev/null +++ b/vbench/src/tests/request_dumper/.gitignore @@ -0,0 +1 @@ +vbench_request_dumper_test_app diff --git a/vbench/src/tests/request_dumper/CMakeLists.txt b/vbench/src/tests/request_dumper/CMakeLists.txt new file mode 100644 index 00000000000..08cd6f12ebe --- /dev/null +++ b/vbench/src/tests/request_dumper/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_request_dumper_test_app + SOURCES + request_dumper_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_request_dumper_test_app COMMAND vbench_request_dumper_test_app) diff --git a/vbench/src/tests/request_dumper/FILES b/vbench/src/tests/request_dumper/FILES new file mode 100644 index 00000000000..24fc66cc387 --- /dev/null +++ b/vbench/src/tests/request_dumper/FILES @@ -0,0 +1 @@ +request_dumper_test.cpp diff --git a/vbench/src/tests/request_dumper/request_dumper_test.cpp b/vbench/src/tests/request_dumper/request_dumper_test.cpp new file mode 100644 index 00000000000..4ca98bb4c90 --- /dev/null +++ b/vbench/src/tests/request_dumper/request_dumper_test.cpp @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_FF("dump request", RequestSink(), RequestDumper(f1)) { + f2.handle(Request::UP(new Request())); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/request_generator/.gitignore b/vbench/src/tests/request_generator/.gitignore new file mode 100644 index 00000000000..8ec28a2df32 --- /dev/null +++ b/vbench/src/tests/request_generator/.gitignore @@ -0,0 +1 @@ +vbench_request_generator_test_app diff --git a/vbench/src/tests/request_generator/CMakeLists.txt b/vbench/src/tests/request_generator/CMakeLists.txt new file mode 100644 index 00000000000..a59029055d0 --- /dev/null +++ b/vbench/src/tests/request_generator/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_request_generator_test_app + SOURCES + request_generator_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_request_generator_test_app COMMAND vbench_request_generator_test_app) diff --git a/vbench/src/tests/request_generator/FILES b/vbench/src/tests/request_generator/FILES new file mode 100644 index 00000000000..2afbcd6ec35 --- /dev/null +++ b/vbench/src/tests/request_generator/FILES @@ -0,0 +1,2 @@ +request_generator_test.cpp +input.txt diff --git a/vbench/src/tests/request_generator/input.txt b/vbench/src/tests/request_generator/input.txt new file mode 100644 index 00000000000..259b0f7250f --- /dev/null +++ b/vbench/src/tests/request_generator/input.txt @@ -0,0 +1 @@ +/this/is/url diff --git a/vbench/src/tests/request_generator/request_generator_test.cpp b/vbench/src/tests/request_generator/request_generator_test.cpp new file mode 100644 index 00000000000..52aca22954c --- /dev/null +++ b/vbench/src/tests/request_generator/request_generator_test.cpp @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_FF("generate request", RequestReceptor(), RequestGenerator("input.txt", f1)) { + f2.run(); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_EQUAL("/this/is/url", f1.request->url()); + EXPECT_FALSE(f2.tainted()); +} + +TEST_FF("input not found", RequestReceptor(), RequestGenerator("no_such_input.txt", f1)) { + f2.run(); + EXPECT_TRUE(f1.request.get() == 0); + EXPECT_TRUE(f2.tainted()); +} + +TEST_FF("abort request generation", RequestReceptor(), RequestGenerator("input.txt", f1)) { + f2.abort(); + f2.run(); + EXPECT_TRUE(f1.request.get() == 0); + EXPECT_FALSE(f2.tainted()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/request_sink/.gitignore b/vbench/src/tests/request_sink/.gitignore new file mode 100644 index 00000000000..82dd828954e --- /dev/null +++ b/vbench/src/tests/request_sink/.gitignore @@ -0,0 +1 @@ +vbench_request_sink_test_app diff --git a/vbench/src/tests/request_sink/CMakeLists.txt b/vbench/src/tests/request_sink/CMakeLists.txt new file mode 100644 index 00000000000..4b184c25f91 --- /dev/null +++ b/vbench/src/tests/request_sink/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_request_sink_test_app + SOURCES + request_sink_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_request_sink_test_app COMMAND vbench_request_sink_test_app) diff --git a/vbench/src/tests/request_sink/FILES b/vbench/src/tests/request_sink/FILES new file mode 100644 index 00000000000..77c3c4b8273 --- /dev/null +++ b/vbench/src/tests/request_sink/FILES @@ -0,0 +1 @@ +request_sink_test.cpp diff --git a/vbench/src/tests/request_sink/request_sink_test.cpp b/vbench/src/tests/request_sink/request_sink_test.cpp new file mode 100644 index 00000000000..aa281d2a3cb --- /dev/null +++ b/vbench/src/tests/request_sink/request_sink_test.cpp @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_F("put a request into the sink, where does it go, nobody cares", RequestSink()) { + f1.handle(Request::UP(new Request())); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/runnable_pair/.gitignore b/vbench/src/tests/runnable_pair/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vbench/src/tests/runnable_pair/.gitignore diff --git a/vbench/src/tests/server_socket/.gitignore b/vbench/src/tests/server_socket/.gitignore new file mode 100644 index 00000000000..e79511c3a0a --- /dev/null +++ b/vbench/src/tests/server_socket/.gitignore @@ -0,0 +1 @@ +vbench_server_socket_test_app diff --git a/vbench/src/tests/server_socket/CMakeLists.txt b/vbench/src/tests/server_socket/CMakeLists.txt new file mode 100644 index 00000000000..1077d7d0525 --- /dev/null +++ b/vbench/src/tests/server_socket/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_server_socket_test_app + SOURCES + server_socket_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_server_socket_test_app COMMAND vbench_server_socket_test_app) diff --git a/vbench/src/tests/server_socket/FILES b/vbench/src/tests/server_socket/FILES new file mode 100644 index 00000000000..47bee2d3769 --- /dev/null +++ b/vbench/src/tests/server_socket/FILES @@ -0,0 +1 @@ +server_socket_test.cpp diff --git a/vbench/src/tests/server_socket/server_socket_test.cpp b/vbench/src/tests/server_socket/server_socket_test.cpp new file mode 100644 index 00000000000..575c7dfaa02 --- /dev/null +++ b/vbench/src/tests/server_socket/server_socket_test.cpp @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> +#include <vespa/vespalib/util/thread.h> + +using namespace vbench; + +TEST_MT_F("require that close will interrupt accept", 2, ServerSocket()) { + if (thread_id == 0) { + for (;;) { + Stream::UP stream = f1.accept(); + if (stream.get() == 0) { + break; + } + } + Stream::UP s2 = f1.accept(); + EXPECT_TRUE(s2.get() == 0); + } else { + vespalib::Thread::sleep(20); + f1.close(); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/server_spec/.gitignore b/vbench/src/tests/server_spec/.gitignore new file mode 100644 index 00000000000..635947d28a8 --- /dev/null +++ b/vbench/src/tests/server_spec/.gitignore @@ -0,0 +1 @@ +vbench_server_spec_test_app diff --git a/vbench/src/tests/server_spec/CMakeLists.txt b/vbench/src/tests/server_spec/CMakeLists.txt new file mode 100644 index 00000000000..a43d2c80dde --- /dev/null +++ b/vbench/src/tests/server_spec/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_server_spec_test_app + SOURCES + server_spec_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_server_spec_test_app COMMAND vbench_server_spec_test_app) diff --git a/vbench/src/tests/server_spec/FILES b/vbench/src/tests/server_spec/FILES new file mode 100644 index 00000000000..34e52a9f86e --- /dev/null +++ b/vbench/src/tests/server_spec/FILES @@ -0,0 +1 @@ +server_spec_test.cpp diff --git a/vbench/src/tests/server_spec/server_spec_test.cpp b/vbench/src/tests/server_spec/server_spec_test.cpp new file mode 100644 index 00000000000..a254e5f8101 --- /dev/null +++ b/vbench/src/tests/server_spec/server_spec_test.cpp @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("empty") { + ServerSpec spec; + EXPECT_EQUAL("", spec.host); + EXPECT_EQUAL(0, spec.port); +} + +TEST("standard") { + ServerSpec spec("foo", 3); + EXPECT_EQUAL("foo", spec.host); + EXPECT_EQUAL(3, spec.port); +} + +TEST("compare") { + ServerSpec spec0("foo", 3); + ServerSpec spec1("foo", 3); + ServerSpec spec2("bar", 3); + ServerSpec spec3("foo", 4); + ServerSpec spec4("bar", 4); + EXPECT_TRUE(spec0 == spec1); + EXPECT_TRUE(spec1 == spec0); + EXPECT_FALSE(spec0 == spec2); + EXPECT_FALSE(spec2 == spec0); + EXPECT_TRUE(spec2 < spec0); + EXPECT_TRUE(spec0 < spec3); + EXPECT_TRUE(spec0 < spec4); + EXPECT_FALSE(spec0 < spec1); + EXPECT_FALSE(spec0 < spec2); + EXPECT_FALSE(spec3 < spec0); + EXPECT_FALSE(spec4 < spec0); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/server_tagger/.gitignore b/vbench/src/tests/server_tagger/.gitignore new file mode 100644 index 00000000000..3ade58a58d5 --- /dev/null +++ b/vbench/src/tests/server_tagger/.gitignore @@ -0,0 +1 @@ +vbench_server_tagger_test_app diff --git a/vbench/src/tests/server_tagger/CMakeLists.txt b/vbench/src/tests/server_tagger/CMakeLists.txt new file mode 100644 index 00000000000..d7cc0bdbea9 --- /dev/null +++ b/vbench/src/tests/server_tagger/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_server_tagger_test_app + SOURCES + server_tagger_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_server_tagger_test_app COMMAND vbench_server_tagger_test_app) diff --git a/vbench/src/tests/server_tagger/FILES b/vbench/src/tests/server_tagger/FILES new file mode 100644 index 00000000000..b632f6e6916 --- /dev/null +++ b/vbench/src/tests/server_tagger/FILES @@ -0,0 +1 @@ +server_tagger_test.cpp diff --git a/vbench/src/tests/server_tagger/server_tagger_test.cpp b/vbench/src/tests/server_tagger/server_tagger_test.cpp new file mode 100644 index 00000000000..461ddffc124 --- /dev/null +++ b/vbench/src/tests/server_tagger/server_tagger_test.cpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST_FF("server tagger", RequestReceptor(), ServerTagger(ServerSpec("host", 42), f1)) { + Request::UP req(new Request()); + EXPECT_EQUAL("", req->server().host); + EXPECT_EQUAL(0, req->server().port); + f2.handle(std::move(req)); + ASSERT_TRUE(f1.request.get() != 0); + EXPECT_EQUAL("host", f1.request->server().host); + EXPECT_EQUAL(42, f1.request->server().port); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/simple_buffer/.gitignore b/vbench/src/tests/simple_buffer/.gitignore new file mode 100644 index 00000000000..f40426a6708 --- /dev/null +++ b/vbench/src/tests/simple_buffer/.gitignore @@ -0,0 +1 @@ +vbench_simple_buffer_test_app diff --git a/vbench/src/tests/simple_buffer/CMakeLists.txt b/vbench/src/tests/simple_buffer/CMakeLists.txt new file mode 100644 index 00000000000..dd8da993c0b --- /dev/null +++ b/vbench/src/tests/simple_buffer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_simple_buffer_test_app + SOURCES + simple_buffer_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_simple_buffer_test_app COMMAND vbench_simple_buffer_test_app) diff --git a/vbench/src/tests/simple_buffer/FILES b/vbench/src/tests/simple_buffer/FILES new file mode 100644 index 00000000000..3669f575911 --- /dev/null +++ b/vbench/src/tests/simple_buffer/FILES @@ -0,0 +1 @@ +simple_buffer_test.cpp diff --git a/vbench/src/tests/simple_buffer/simple_buffer_test.cpp b/vbench/src/tests/simple_buffer/simple_buffer_test.cpp new file mode 100644 index 00000000000..f666438d721 --- /dev/null +++ b/vbench/src/tests/simple_buffer/simple_buffer_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +void checkMemory(const string &expect, const Memory &mem) { + EXPECT_EQUAL(expect, string(mem.data, mem.size)); +} + +void checkBuffer(const string &expect, const SimpleBuffer &buf) { + TEST_DO(checkMemory(expect, buf.get())); +} + +TEST("simple buffer") { + SimpleBuffer buf; + TEST_DO(checkBuffer("", buf)); + { // read from empty buffer + EXPECT_EQUAL(0u, buf.obtain(1, 1).size); + } + { // write to buffer + WritableMemory mem = buf.reserve(10); + TEST_DO(checkBuffer("", buf)); + EXPECT_EQUAL(10u, mem.size); + mem.data[0] = 'a'; + mem.data[1] = 'b'; + mem.data[2] = 'c'; + EXPECT_EQUAL(&buf, &buf.commit(3, 0)); + mem = buf.reserve(0); + TEST_DO(checkBuffer("abc", buf)); + EXPECT_EQUAL(0u, mem.size); + } + { // unaligned read across end (last byte not evicted) + Memory mem = buf.obtain(2, 1); + TEST_DO(checkBuffer("abc", buf)); + TEST_DO(checkMemory("ab", mem)); + EXPECT_EQUAL(&buf, &buf.evict(2)); + mem = buf.obtain(2, 1); + TEST_DO(checkBuffer("c", buf)); + TEST_DO(checkMemory("c", mem)); + mem = buf.obtain(2, 1); + TEST_DO(checkBuffer("c", buf)); + TEST_DO(checkMemory("c", mem)); + } + { // write more to buffer + WritableMemory mem = buf.reserve(10); + EXPECT_EQUAL(10u, mem.size); + TEST_DO(checkBuffer("c", buf)); + mem.data[0] = 'd'; + EXPECT_EQUAL(&buf, &buf.commit(1, 0)); + mem = buf.reserve(5); + TEST_DO(checkBuffer("cd", buf)); + EXPECT_EQUAL(5u, mem.size); + } + { // aligned read until end + Memory mem = buf.obtain(1, 1); + TEST_DO(checkBuffer("cd", buf)); + TEST_DO(checkMemory("c", mem)); + EXPECT_EQUAL(&buf, &buf.evict(1)); + mem = buf.obtain(1, 1); + TEST_DO(checkBuffer("d", buf)); + TEST_DO(checkMemory("d", mem)); + EXPECT_EQUAL(&buf, &buf.evict(1)); + mem = buf.obtain(1, 1); + TEST_DO(checkBuffer("", buf)); + TEST_DO(checkMemory("", mem)); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/socket/.gitignore b/vbench/src/tests/socket/.gitignore new file mode 100644 index 00000000000..da032fba4cc --- /dev/null +++ b/vbench/src/tests/socket/.gitignore @@ -0,0 +1 @@ +vbench_socket_test_app diff --git a/vbench/src/tests/socket/CMakeLists.txt b/vbench/src/tests/socket/CMakeLists.txt new file mode 100644 index 00000000000..6ede77e5d63 --- /dev/null +++ b/vbench/src/tests/socket/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_socket_test_app + SOURCES + socket_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_socket_test_app COMMAND vbench_socket_test_app) diff --git a/vbench/src/tests/socket/FILES b/vbench/src/tests/socket/FILES new file mode 100644 index 00000000000..d9051f5e248 --- /dev/null +++ b/vbench/src/tests/socket/FILES @@ -0,0 +1 @@ +socket_test.cpp diff --git a/vbench/src/tests/socket/socket_test.cpp b/vbench/src/tests/socket/socket_test.cpp new file mode 100644 index 00000000000..4e58465c5a8 --- /dev/null +++ b/vbench/src/tests/socket/socket_test.cpp @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +const size_t numLines = 100; + +struct Agent { + Stream::UP socket; + Agent(Stream::UP s) : socket(std::move(s)) {} + void write(const char *prefix) { + BufferedOutput out(*socket, 32); + for (size_t i = 0; i < numLines; ++i) { + out.printf("%s%zu\n", prefix, i); + } + out.append("\n"); + } + void read(const char *prefix) { + LineReader reader(*socket, 32); + for (size_t lines = 0; true; ++lines) { + string line; + reader.readLine(line); + if (line.empty()) { + EXPECT_EQUAL(numLines, lines); + break; + } + EXPECT_EQUAL(strfmt("%s%zu", prefix, lines), line); + } + } +}; + +struct Client : public Agent, public vespalib::Runnable { + Client(Stream::UP s) : Agent(std::move(s)) {} + virtual void run() { + TEST_THREAD("client"); + write("client-"); + read("server-"); + } +}; + +struct Server : public Agent, public vespalib::Runnable { + Server(Stream::UP s) : Agent(std::move(s)) {} + virtual void run() { + TEST_THREAD("server"); + read("client-"); + write("server-"); + } +}; + +TEST("socket") { + ServerSocket serverSocket; + Client client(Stream::UP(new Socket("localhost", serverSocket.port()))); + Server server(serverSocket.accept()); + vespalib::Thread clientThread(client); + vespalib::Thread serverThread(server); + clientThread.start(); + serverThread.start(); + clientThread.join(); + serverThread.join(); + { + server.socket.reset(); + LineReader reader(*client.socket, 32); + string line; + EXPECT_FALSE(reader.readLine(line)); + EXPECT_TRUE(line.empty()); + EXPECT_TRUE(client.socket->eof()); + EXPECT_FALSE(client.socket->tainted()); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/taint/.gitignore b/vbench/src/tests/taint/.gitignore new file mode 100644 index 00000000000..ed8a6bdc417 --- /dev/null +++ b/vbench/src/tests/taint/.gitignore @@ -0,0 +1 @@ +vbench_taint_test_app diff --git a/vbench/src/tests/taint/CMakeLists.txt b/vbench/src/tests/taint/CMakeLists.txt new file mode 100644 index 00000000000..7dd23e62020 --- /dev/null +++ b/vbench/src/tests/taint/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_taint_test_app + SOURCES + taint_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_taint_test_app COMMAND vbench_taint_test_app) diff --git a/vbench/src/tests/taint/FILES b/vbench/src/tests/taint/FILES new file mode 100644 index 00000000000..448832fd7dc --- /dev/null +++ b/vbench/src/tests/taint/FILES @@ -0,0 +1 @@ +taint_test.cpp diff --git a/vbench/src/tests/taint/taint_test.cpp b/vbench/src/tests/taint/taint_test.cpp new file mode 100644 index 00000000000..1d7c1cb0ef0 --- /dev/null +++ b/vbench/src/tests/taint/taint_test.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("untainted") { + Taint t; + bool fail = t; + EXPECT_FALSE(fail); + EXPECT_FALSE(t.taint()); + EXPECT_EQUAL("", t.reason()); +} + +TEST("Taintable::nil") { + const Taint &t = Taintable::nil().tainted(); + bool fail = t; + EXPECT_FALSE(fail); + EXPECT_FALSE(t.taint()); + EXPECT_EQUAL("", t.reason()); +} + +TEST("tainted") { + Taint t("argh"); + bool fail = t; + EXPECT_TRUE(fail); + EXPECT_TRUE(t.taint()); + EXPECT_EQUAL("argh", t.reason()); +} + +TEST("reset taint") { + Taint t; + EXPECT_FALSE(t.taint()); + EXPECT_EQUAL("", t.reason()); + t.reset("argh"); + EXPECT_TRUE(t.taint()); + EXPECT_EQUAL("argh", t.reason()); + t.reset(); + EXPECT_FALSE(t.taint()); + EXPECT_EQUAL("", t.reason()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/thread/.gitignore b/vbench/src/tests/thread/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vbench/src/tests/thread/.gitignore diff --git a/vbench/src/tests/time_queue/.gitignore b/vbench/src/tests/time_queue/.gitignore new file mode 100644 index 00000000000..c7cb2538d85 --- /dev/null +++ b/vbench/src/tests/time_queue/.gitignore @@ -0,0 +1 @@ +vbench_time_queue_test_app diff --git a/vbench/src/tests/time_queue/CMakeLists.txt b/vbench/src/tests/time_queue/CMakeLists.txt new file mode 100644 index 00000000000..a87a4c1982e --- /dev/null +++ b/vbench/src/tests/time_queue/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_time_queue_test_app + SOURCES + time_queue_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_time_queue_test_app COMMAND vbench_time_queue_test_app) diff --git a/vbench/src/tests/time_queue/FILES b/vbench/src/tests/time_queue/FILES new file mode 100644 index 00000000000..39406e0b322 --- /dev/null +++ b/vbench/src/tests/time_queue/FILES @@ -0,0 +1 @@ +time_queue_test.cpp diff --git a/vbench/src/tests/time_queue/time_queue_test.cpp b/vbench/src/tests/time_queue/time_queue_test.cpp new file mode 100644 index 00000000000..d6be9501e6d --- /dev/null +++ b/vbench/src/tests/time_queue/time_queue_test.cpp @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> +#include <vespa/vespalib/util/sync.h> + +using namespace vbench; + +TEST_MT_FFF("time queue", 2, TimeQueue<int>(10.0, 5.0), vespalib::Gate(), vespalib::Gate()) { + if (thread_id == 0) { + f1.insert(std::unique_ptr<int>(new int(1)), 1.0); + f1.insert(std::unique_ptr<int>(new int(2)), 3.0); + f1.insert(std::unique_ptr<int>(new int(3)), 2.0); + f2.countDown(); + f1.insert(std::unique_ptr<int>(new int(4)), 100.0); + f1.insert(std::unique_ptr<int>(new int(5)), 101.0); + f3.countDown(); + } else { + double delay; + std::vector<std::unique_ptr<int> > list; + EXPECT_TRUE(f2.await(20000)); + EXPECT_FALSE(f3.await(20)); + { + f1.extract(1.5, list, delay); + ASSERT_EQUAL(1u, list.size()); + EXPECT_EQUAL(1, *list[0]); + EXPECT_EQUAL(0.5, delay); + list.clear(); + } + { + f1.extract(10.0, list, delay); + ASSERT_EQUAL(2u, list.size()); + EXPECT_EQUAL(3, *list[0]); + EXPECT_EQUAL(2, *list[1]); + EXPECT_EQUAL(5.0, delay); + list.clear(); + } + { + f1.extract(99.25, list, delay); + EXPECT_EQUAL(0u, list.size()); + EXPECT_EQUAL(5.0, delay); + } + EXPECT_TRUE(f3.await(20000)); + { + f1.extract(99.25, list, delay); + EXPECT_EQUAL(0u, list.size()); + EXPECT_EQUAL(0.75, delay); + } + f1.discard(); + { + f1.extract(101.5, list, delay); + EXPECT_EQUAL(0u, list.size()); + EXPECT_EQUAL(5.0, delay); + } + f1.close(); + f1.insert(std::unique_ptr<int>(new int(6)), 102.0); + f1.insert(std::unique_ptr<int>(new int(7)), 103.0); + { + f1.extract(103.5, list, delay); + EXPECT_EQUAL(0u, list.size()); + EXPECT_EQUAL(5.0, delay); + } + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/timer/.gitignore b/vbench/src/tests/timer/.gitignore new file mode 100644 index 00000000000..cd37c01446c --- /dev/null +++ b/vbench/src/tests/timer/.gitignore @@ -0,0 +1 @@ +vbench_timer_test_app diff --git a/vbench/src/tests/timer/CMakeLists.txt b/vbench/src/tests/timer/CMakeLists.txt new file mode 100644 index 00000000000..fc50c222ba0 --- /dev/null +++ b/vbench/src/tests/timer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_timer_test_app + SOURCES + timer_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_timer_test_app COMMAND vbench_timer_test_app) diff --git a/vbench/src/tests/timer/FILES b/vbench/src/tests/timer/FILES new file mode 100644 index 00000000000..a90924ff2d1 --- /dev/null +++ b/vbench/src/tests/timer/FILES @@ -0,0 +1 @@ +timer_test.cpp diff --git a/vbench/src/tests/timer/timer_test.cpp b/vbench/src/tests/timer/timer_test.cpp new file mode 100644 index 00000000000..150a8d847d6 --- /dev/null +++ b/vbench/src/tests/timer/timer_test.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +IGNORE_TEST("timer") { + Timer timer; + EXPECT_APPROX(0.0, timer.sample(), 0.1); + vespalib::Thread::sleep(1000); + EXPECT_APPROX(1.0, timer.sample(), 0.1); + timer.reset(); + EXPECT_APPROX(0.0, timer.sample(), 0.1); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/true/.gitignore b/vbench/src/tests/true/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vbench/src/tests/true/.gitignore diff --git a/vbench/src/tests/writable_memory/.gitignore b/vbench/src/tests/writable_memory/.gitignore new file mode 100644 index 00000000000..00061ea18f7 --- /dev/null +++ b/vbench/src/tests/writable_memory/.gitignore @@ -0,0 +1 @@ +vbench_writable_memory_test_app diff --git a/vbench/src/tests/writable_memory/CMakeLists.txt b/vbench/src/tests/writable_memory/CMakeLists.txt new file mode 100644 index 00000000000..4011ee64f63 --- /dev/null +++ b/vbench/src/tests/writable_memory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vbench_writable_memory_test_app + SOURCES + writable_memory_test.cpp + DEPENDS + vbench_test + vbench +) +vespa_add_test(NAME vbench_writable_memory_test_app COMMAND vbench_writable_memory_test_app) diff --git a/vbench/src/tests/writable_memory/FILES b/vbench/src/tests/writable_memory/FILES new file mode 100644 index 00000000000..14718444a2e --- /dev/null +++ b/vbench/src/tests/writable_memory/FILES @@ -0,0 +1 @@ +writable_memory_test.cpp diff --git a/vbench/src/tests/writable_memory/writable_memory_test.cpp b/vbench/src/tests/writable_memory/writable_memory_test.cpp new file mode 100644 index 00000000000..2be9d6b5153 --- /dev/null +++ b/vbench/src/tests/writable_memory/writable_memory_test.cpp @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vbench/test/all.h> + +using namespace vbench; + +TEST("empty") { + WritableMemory wm; + EXPECT_EQUAL((char*)0, wm.data); + EXPECT_EQUAL(0u, wm.size); +} + +TEST("from buffer") { + char buf[3]; + WritableMemory wm(buf, 3); + EXPECT_EQUAL(buf, wm.data); + EXPECT_EQUAL(3u, wm.size); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/vbench/.gitignore b/vbench/src/vbench/.gitignore new file mode 100644 index 00000000000..657fed7b72a --- /dev/null +++ b/vbench/src/vbench/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/libvbench.so.5.1 diff --git a/vbench/src/vbench/CMakeLists.txt b/vbench/src/vbench/CMakeLists.txt new file mode 100644 index 00000000000..ce67bf427b4 --- /dev/null +++ b/vbench/src/vbench/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vbench + SOURCES + $<TARGET_OBJECTS:vbench_core> + $<TARGET_OBJECTS:vbench_http> + $<TARGET_OBJECTS:vbench_vbench_vbench> + INSTALL lib64 + DEPENDS +) diff --git a/vbench/src/vbench/core/.gitignore b/vbench/src/vbench/core/.gitignore new file mode 100644 index 00000000000..6241ac5efbe --- /dev/null +++ b/vbench/src/vbench/core/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/html diff --git a/vbench/src/vbench/core/CMakeLists.txt b/vbench/src/vbench/core/CMakeLists.txt new file mode 100644 index 00000000000..3817a4c6950 --- /dev/null +++ b/vbench/src/vbench/core/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vbench_core OBJECT + SOURCES + buffered_output.cpp + byte_input.cpp + closeable.cpp + dispatcher.cpp + handler.cpp + handler_thread.cpp + input.cpp + input_file_reader.cpp + line_reader.cpp + mapped_file_input.cpp + memory.cpp + output.cpp + provider.cpp + simple_buffer.cpp + socket.cpp + stream.cpp + string.cpp + taint.cpp + taintable.cpp + time_queue.cpp + timer.cpp + writable_memory.cpp + DEPENDS +) diff --git a/vbench/src/vbench/core/Doxyfile b/vbench/src/vbench/core/Doxyfile new file mode 100644 index 00000000000..ed6fda1135f --- /dev/null +++ b/vbench/src/vbench/core/Doxyfile @@ -0,0 +1,1162 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Doxyfile 1.3.9.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = vbench_core + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of source +# files, where putting all generated files in the same directory would otherwise +# cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, +# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, +# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, +# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, +# Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# This tag can be used to specify the encoding used in the generated output. +# The encoding is not always determined by the language that is chosen, +# but also whether or not the output is meant for Windows or non-Windows users. +# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES +# forces the Windows encoding (this is the default for the Windows binary), +# whereas setting the tag to NO uses a Unix-style encoding (the default for +# all platforms other than Windows). + +USE_WINDOWS_ENCODING = NO + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is used +# as the annotated text. Otherwise, the brief description is used as-is. If left +# blank, the following values are used ("$name" is automatically replaced with the +# name of the entity): "The $name class" "The $name widget" "The $name file" +# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited +# members of a class in the documentation of that class as if those members were +# ordinary class members. Constructors, destructors and assignment operators of +# the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like the Qt-style comments (thus requiring an +# explicit @brief command for a brief description. + +JAVADOC_AUTOBRIEF = YES + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources +# only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. + +SHOW_DIRECTORIES = YES + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp +# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories +# that are symbolic links (a Unix filesystem feature) are excluded from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. + +EXCLUDE_PATTERNS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_PREDEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse the +# parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or +# super classes. Setting the tag to NO turns the diagrams off. Note that this +# option is superseded by the HAVE_DOT option below. This is only a fallback. It is +# recommended to install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = YES + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = YES + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = YES + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found on the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_WIDTH = 4096 + +# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_HEIGHT = 4096 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes that +# lay further from the root node will be omitted. Note that setting this option to +# 1 or 2 may greatly reduce the computation time needed for large code bases. Also +# note that a graph may be further truncated if the graph's image dimensions are +# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). +# If 0 is used for the depth value (the default), the graph is not depth-constrained. + +MAX_DOT_GRAPH_DEPTH = 0 + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO diff --git a/vbench/src/vbench/core/buffered_output.cpp b/vbench/src/vbench/core/buffered_output.cpp new file mode 100644 index 00000000000..36e01e55fd5 --- /dev/null +++ b/vbench/src/vbench/core/buffered_output.cpp @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "buffered_output.h" + +namespace vbench { + +BufferedOutput & +BufferedOutput::printf(const char *fmt, ...) +{ + ensureFree(256); + int space = (_data.size - _pos); + int size; + va_list ap; + va_start(ap, fmt); + size = vsnprintf(_data.data + _pos, space, fmt, ap); + va_end(ap); + assert(size >= 0); + if (size >= space) { + space = size + 1; + ensureFree(space); + va_start(ap, fmt); + size = vsnprintf(_data.data + _pos, space, fmt, ap); + va_end(ap); + assert((size + 1) == space); + } + _pos += size; + return *this; +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/buffered_output.h b/vbench/src/vbench/core/buffered_output.h new file mode 100644 index 00000000000..f274ce9f71c --- /dev/null +++ b/vbench/src/vbench/core/buffered_output.h @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "output.h" +#include "string.h" + +namespace vbench { + +/** + * Concrete utility class used to write small amounts of data to an + * output efficiently by buffering the output into larger chunks. + **/ +class BufferedOutput +{ +private: + Output &_output; + WritableMemory _data; + size_t _pos; + size_t _chunkSize; + + void ensureFree(size_t bytes) { + if ((_pos + bytes) > _data.size) { + _data = _output.commit(_pos, 0).reserve(std::max(bytes, _chunkSize)); + _pos = 0; + } + } + +public: + BufferedOutput(Output &output, size_t chunkSize) + : _output(output), _data(), _pos(), _chunkSize(chunkSize) {} + ~BufferedOutput() { _output.commit(_pos, 0); } + + BufferedOutput &append(char c) { + ensureFree(1); + _data.data[_pos++] = c; + return *this; + } + + BufferedOutput &append(const char *data, size_t size) { + ensureFree(size); + memcpy(_data.data + _pos, data, size); + _pos += size; + return *this; + } + + BufferedOutput &append(const string &str) { + return append(str.data(), str.size()); + } + + BufferedOutput &append(const char *str) { + return append(str, strlen(str)); + } + + BufferedOutput &printf(const char *fmt, ...) +#ifdef __GNUC__ + // Add printf format checks with gcc + __attribute__ ((format (printf,2,3))) +#endif + ; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/byte_input.cpp b/vbench/src/vbench/core/byte_input.cpp new file mode 100644 index 00000000000..d6753e2cce5 --- /dev/null +++ b/vbench/src/vbench/core/byte_input.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "byte_input.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/byte_input.h b/vbench/src/vbench/core/byte_input.h new file mode 100644 index 00000000000..34468a2ffeb --- /dev/null +++ b/vbench/src/vbench/core/byte_input.h @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "input.h" + +namespace vbench { + +/** + * Concrete utility class used to read input data one byte at a time + * by wrapping a generic Input implementation. + **/ +class ByteInput +{ +private: + Input &_input; + Memory _data; + size_t _pos; + size_t _chunkSize; + +public: + /** + * Wrap an Input to read one byte at a time. + * + * @param input the underlying Input + * @param chunkSize how much data to request from the input per transaction + **/ + ByteInput(Input &input, size_t chunkSize) + : _input(input), _data(), _pos(0), _chunkSize(chunkSize) {} + ~ByteInput() { _input.evict(_pos); } + + /** + * Read the next byte of input. + * + * @return next byte of input, or -1 if no more input is available + **/ + int get() { + if (_pos < _data.size) { + return (_data.data[_pos++] & 0xff); + } else { + _data = _input.evict(_pos).obtain(_chunkSize, 1); + if ((_pos = 0) < _data.size) { + return (_data.data[_pos++] & 0xff); + } + } + return -1; + } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/closeable.cpp b/vbench/src/vbench/core/closeable.cpp new file mode 100644 index 00000000000..1a61f1a386e --- /dev/null +++ b/vbench/src/vbench/core/closeable.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "closeable.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/closeable.h b/vbench/src/vbench/core/closeable.h new file mode 100644 index 00000000000..012eaef5112 --- /dev/null +++ b/vbench/src/vbench/core/closeable.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vbench { + +/** + * Something that can be closed to disrupt its steady-state behavior. + **/ +struct Closeable +{ + virtual void close() = 0; + virtual ~Closeable() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/dispatcher.cpp b/vbench/src/vbench/core/dispatcher.cpp new file mode 100644 index 00000000000..b787ad2ae9f --- /dev/null +++ b/vbench/src/vbench/core/dispatcher.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "dispatcher.h" + +namespace vbench { + +namespace { + +struct DummyItem {}; + +} // namespace vbench::<unnamed> + +template class Dispatcher<DummyItem>; + +} // namespace vbench diff --git a/vbench/src/vbench/core/dispatcher.h b/vbench/src/vbench/core/dispatcher.h new file mode 100644 index 00000000000..fbf3e86d124 --- /dev/null +++ b/vbench/src/vbench/core/dispatcher.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/sync.h> + +#include "handler.h" +#include "provider.h" +#include "closeable.h" + +namespace vbench { + +/** + * Dispatch objects between threads. Objects received through the + * Handler interface will be passed along to Components requesting + * objects through the Provider interface. If there are no components + * currently waiting for objects, the objects will be passed along to + * a predefined fallback handler instead. A closed dispatcher will + * provide nil objects and handle incoming objects by deleting them. + **/ +template <typename T> +class Dispatcher : public Handler<T>, + public Provider<T>, + public Closeable +{ +private: + struct ThreadState { + std::unique_ptr<T> object; + vespalib::Gate gate; + }; + + Handler<T> &_fallback; + vespalib::Lock _lock; + std::vector<ThreadState*> _threads; + bool _closed; + +public: + Dispatcher(Handler<T> &fallback); + bool waitForThreads(size_t threads, size_t pollCnt) const; + virtual void close(); + virtual void handle(std::unique_ptr<T> obj); + virtual std::unique_ptr<T> provide(); +}; + +} // namespace vbench + +#include "dispatcher.hpp" + diff --git a/vbench/src/vbench/core/dispatcher.hpp b/vbench/src/vbench/core/dispatcher.hpp new file mode 100644 index 00000000000..b2c14d4ea7a --- /dev/null +++ b/vbench/src/vbench/core/dispatcher.hpp @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/thread.h> + +namespace vbench { + +template <typename T> +Dispatcher<T>::Dispatcher(Handler<T> &fallback) + : _fallback(fallback), + _lock(), + _threads(), + _closed(false) +{ +} + +template <typename T> +bool +Dispatcher<T>::waitForThreads(size_t threads, size_t pollCnt) const +{ + for (size_t i = 0; i < pollCnt; ++i) { + if (i != 0) { + vespalib::Thread::sleep(20); + } + { + vespalib::LockGuard guard(_lock); + if (_threads.size() >= threads) { + return true; + } + } + } + return false; +} + +template <typename T> +void +Dispatcher<T>::close() +{ + std::vector<ThreadState*> threads; + { + vespalib::LockGuard guard(_lock); + std::swap(_threads, threads); + _closed = true; + } + for (size_t i = 0; i < threads.size(); ++i) { + threads[i]->gate.countDown(); + } +} + +template <typename T> +void +Dispatcher<T>::handle(std::unique_ptr<T> obj) +{ + vespalib::LockGuard guard(_lock); + if (!_threads.empty()) { + ThreadState *state = _threads.back(); + _threads.pop_back(); + guard.unlock(); + state->object = std::move(obj); + state->gate.countDown(); + } else { + bool closed = _closed; + guard.unlock(); + if (!closed) { + _fallback.handle(std::move(obj)); + } + } +} + +template <typename T> +std::unique_ptr<T> +Dispatcher<T>::provide() +{ + ThreadState state; + { + vespalib::LockGuard guard(_lock); + if (!_closed) { + _threads.push_back(&state); + guard.unlock(); + state.gate.await(); + } + } + return std::move(state.object); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/handler.cpp b/vbench/src/vbench/core/handler.cpp new file mode 100644 index 00000000000..345d4131a05 --- /dev/null +++ b/vbench/src/vbench/core/handler.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "handler.h" + +namespace vbench { + +namespace { + +struct DummyItem {}; + +} // namespace vbench::<unnamed> + +template class Handler<DummyItem>; + +} // namespace vbench diff --git a/vbench/src/vbench/core/handler.h b/vbench/src/vbench/core/handler.h new file mode 100644 index 00000000000..962e6c09ad7 --- /dev/null +++ b/vbench/src/vbench/core/handler.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace vbench { + +/** + * A Handler is a component to whom you can pass an object. + **/ +template <typename T> +struct Handler +{ + virtual void handle(std::unique_ptr<T> obj) = 0; + virtual ~Handler() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/handler_thread.cpp b/vbench/src/vbench/core/handler_thread.cpp new file mode 100644 index 00000000000..d0333327c79 --- /dev/null +++ b/vbench/src/vbench/core/handler_thread.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "handler_thread.h" + +namespace vbench { + +namespace { + +struct DummyItem {}; + +} // namespace vbench::<unnamed> + +template class HandlerThread<DummyItem>; + +} // namespace vbench diff --git a/vbench/src/vbench/core/handler_thread.h b/vbench/src/vbench/core/handler_thread.h new file mode 100644 index 00000000000..cd7b53a6109 --- /dev/null +++ b/vbench/src/vbench/core/handler_thread.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/arrayqueue.hpp> +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/util/runnable.h> +#include <vespa/vespalib/util/joinable.h> + +#include "handler.h" + +namespace vbench { + +/** + * A Handler that will forward incoming objects to another handler in + * a separate thread. All objects are forwarded using the same thread, + * reducing the need for synchronization in the handler forwarded + * to. A call to join will wait until all queued object have been + * forwarded. Object obtained after join is invoked will be discarded. + **/ +template <typename T> +class HandlerThread : public Handler<T>, + public vespalib::Runnable, + public vespalib::Joinable +{ +private: + vespalib::Monitor _monitor; + vespalib::ArrayQueue<std::unique_ptr<T> > _queue; + Handler<T> &_next; + vespalib::Thread _thread; + bool _done; + + virtual void run(); + +public: + HandlerThread(Handler<T> &next); + virtual ~HandlerThread(); + virtual void handle(std::unique_ptr<T> obj); + virtual void join(); +}; + +} // namespace vbench + +#include "handler_thread.hpp" + diff --git a/vbench/src/vbench/core/handler_thread.hpp b/vbench/src/vbench/core/handler_thread.hpp new file mode 100644 index 00000000000..b1d87128c72 --- /dev/null +++ b/vbench/src/vbench/core/handler_thread.hpp @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace vbench { + +template <typename T> +void +HandlerThread<T>::run() +{ + for (;;) { + vespalib::MonitorGuard guard(_monitor); + while (!_done && _queue.empty()) { + guard.wait(); + } + if (_done && _queue.empty()) { + return; + } + assert(!_queue.empty()); + std::unique_ptr<T> obj(std::move(_queue.access(0))); + _queue.pop(); + guard.unlock(); // UNLOCK + _next.handle(std::move(obj)); + } +} + +template <typename T> +HandlerThread<T>::HandlerThread(Handler<T> &next) + : _monitor(), + _queue(), + _next(next), + _thread(*this), + _done(false) +{ + _thread.start(); +} + +template <typename T> +HandlerThread<T>::~HandlerThread() +{ + join(); + assert(_queue.empty()); +} + +template <typename T> +void +HandlerThread<T>::handle(std::unique_ptr<T> obj) +{ + vespalib::MonitorGuard guard(_monitor); + if (!_done) { + if (_queue.empty()) { + guard.signal(); + } + _queue.push(std::move(obj)); + } +} + +template <typename T> +void +HandlerThread<T>::join() +{ + { + vespalib::MonitorGuard guard(_monitor); + _done = true; + guard.signal(); + } + _thread.join(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/input.cpp b/vbench/src/vbench/core/input.cpp new file mode 100644 index 00000000000..b6ffda1f255 --- /dev/null +++ b/vbench/src/vbench/core/input.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "input.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/input.h b/vbench/src/vbench/core/input.h new file mode 100644 index 00000000000..6b767ca1997 --- /dev/null +++ b/vbench/src/vbench/core/input.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "memory.h" + +namespace vbench { + +/** + * Interface used to abstract a source of input data. Note that the + * input data itself is owned by the object implementing this + * interface. + **/ +struct Input +{ + /** + * Obtain more input data. You will never obtain more data than + * requested, but you may obtain less. + * + * @return the obtained input data + * @param bytes the number of bytes requested + * @param lowMark minimum bytes in byffer before refilling + **/ + virtual Memory obtain(size_t bytes, size_t lowMark) = 0; + + /** + * Evict processed input data. Never evict more data than you have + * obtained. + * + * @return this object, for chaining + * @param bytes the number of bytes to evict + **/ + virtual Input &evict(size_t bytes) = 0; + + virtual ~Input() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/input_file_reader.cpp b/vbench/src/vbench/core/input_file_reader.cpp new file mode 100644 index 00000000000..324951a9ae2 --- /dev/null +++ b/vbench/src/vbench/core/input_file_reader.cpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "input_file_reader.h" + +namespace vbench { + +bool +InputFileReader::readLine(string &dst) +{ + while (_lines.readLine(dst) && dst.empty()) { + // skip empty lines + } + return !dst.empty(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/input_file_reader.h b/vbench/src/vbench/core/input_file_reader.h new file mode 100644 index 00000000000..de80a023719 --- /dev/null +++ b/vbench/src/vbench/core/input_file_reader.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "string.h" +#include "taint.h" +#include "mapped_file_input.h" +#include "byte_input.h" +#include "line_reader.h" + +namespace vbench { + +/** + * Read non-empty lines from an input file. This class is implemented + * in terms of the MappedFileInput and LineReader classes. + **/ +class InputFileReader : public Taintable +{ +private: + MappedFileInput _file; + LineReader _lines; + +public: + InputFileReader(const string &name) + : _file(name), _lines(_file, 4096) {} + + /** + * Read a single line from the input file and put it into + * 'dst'. Empty lines and '\r' directly before '\n' will be + * ignored. Lines are terminated by '\n' or EOF. + * + * @return true if dst is non-empty + * @param dst place to put read line + **/ + bool readLine(string &dst); + + virtual const Taint &tainted() const { return _file.tainted(); } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/line_reader.cpp b/vbench/src/vbench/core/line_reader.cpp new file mode 100644 index 00000000000..5bc8160ccd2 --- /dev/null +++ b/vbench/src/vbench/core/line_reader.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "line_reader.h" + +namespace vbench { + +namespace { +void stripCR(string &dst) { + if (!dst.empty() && dst[dst.size() - 1] == '\r') { + dst.resize(dst.size() - 1); + } +} +} // namespace vbench::<unnamed> + +LineReader::LineReader(Input &input, size_t chunkSize) + : _input(input, chunkSize) +{ +} + +bool +LineReader::readLine(string &dst) +{ + dst.clear(); + for (int c = _input.get(); c >= 0; c = _input.get()) { + if (c != '\n') { + dst.push_back(c); + } else { + stripCR(dst); + return true; + } + } + return !dst.empty(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/line_reader.h b/vbench/src/vbench/core/line_reader.h new file mode 100644 index 00000000000..c26dea96db2 --- /dev/null +++ b/vbench/src/vbench/core/line_reader.h @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "byte_input.h" +#include "string.h" + +namespace vbench { + +/** + * Concrete utility class used to read individual lines of text from + * an underlying input. This class is implemented in terms of the + * ByteInput class. + **/ +class LineReader +{ +private: + ByteInput _input; + +public: + /** + * Wrap an Input to read one line at a time. + * + * @param input the underlying Input + * @param chunkSize how much data to request from the input per transaction + **/ + LineReader(Input &input, size_t chunkSize); + + /** + * Read the next line of input. Lines are separated by '\n'. '\r' + * appearing directly in from of '\n' will be stripped. Empty + * lines will be returned. + * + * @return true if a line could be read, + * false if no more data was available + * @param dst where to store the line that was read + **/ + bool readLine(string &dst); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/mapped_file_input.cpp b/vbench/src/vbench/core/mapped_file_input.cpp new file mode 100644 index 00000000000..c4edc619e5d --- /dev/null +++ b/vbench/src/vbench/core/mapped_file_input.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "mapped_file_input.h" + +namespace vbench { + +MappedFileInput::MappedFileInput(const string &name) + : _file(open(name.c_str(), O_RDONLY)), + _data(static_cast<char*>(MAP_FAILED)), + _size(0), + _taint(strfmt("could not open file: %s", name.c_str())), + _pos(0) +{ + struct stat info; + if (_file >= 0 && fstat(_file, &info) == 0) { + _data = static_cast<char*>(mmap(0, info.st_size, + PROT_READ, MAP_SHARED, _file, 0)); + if (_data != MAP_FAILED) { + _size = info.st_size; + madvise(_data, _size, MADV_SEQUENTIAL); + _taint.reset(); + } + } +} + +Memory +MappedFileInput::obtain(size_t bytes, size_t) +{ + return Memory(_data + _pos, std::min(bytes, (_size - _pos))); +} + +Input & +MappedFileInput::evict(size_t bytes) +{ + assert(bytes <= (_size - _pos)); + _pos += bytes; + return *this; +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/mapped_file_input.h b/vbench/src/vbench/core/mapped_file_input.h new file mode 100644 index 00000000000..94f0e21dbf2 --- /dev/null +++ b/vbench/src/vbench/core/mapped_file_input.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "input.h" +#include "taintable.h" + +namespace vbench { + +/** + * A Taintable Input implementation reading sequentially from a memory + * mapped file. + **/ +class MappedFileInput : public Input, + public Taintable +{ +private: + int _file; + char *_data; + size_t _size; + Taint _taint; + size_t _pos; + +public: + MappedFileInput(const string &name); + Memory get() const { return Memory(_data, _size); } + virtual Memory obtain(size_t bytes, size_t lowMark); + virtual Input &evict(size_t bytes); + virtual const Taint &tainted() const { return _taint; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/memory.cpp b/vbench/src/vbench/core/memory.cpp new file mode 100644 index 00000000000..75d2c6cc903 --- /dev/null +++ b/vbench/src/vbench/core/memory.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "memory.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/memory.h b/vbench/src/vbench/core/memory.h new file mode 100644 index 00000000000..749e7daef15 --- /dev/null +++ b/vbench/src/vbench/core/memory.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "string.h" + +namespace vbench { + +/** + * Simple wrapper referencing a read-only region of memory. + **/ +struct Memory +{ + const char *data; + size_t size; + Memory() : data(0), size(0) {} + Memory(const char *d, size_t s) : data(d), size(s) {} + Memory(const char *str) : data(str), size(strlen(str)) {} + Memory(const string &str) : data(str.data()), size(str.size()) {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/output.cpp b/vbench/src/vbench/core/output.cpp new file mode 100644 index 00000000000..71f09ccab22 --- /dev/null +++ b/vbench/src/vbench/core/output.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "output.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/output.h b/vbench/src/vbench/core/output.h new file mode 100644 index 00000000000..410e0f72643 --- /dev/null +++ b/vbench/src/vbench/core/output.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "writable_memory.h" + +namespace vbench { + +/** + * Interface used to abstract a destination for output data. Note that + * the output data itself is owned by the object implementing this + * interface. + **/ +struct Output +{ + /** + * Reserve space for more output data. + * + * @return the reserved output data + * @param bytes number of bytes to reserve + **/ + virtual WritableMemory reserve(size_t bytes) = 0; + + /** + * Commit produced output data. Never commit more data than you + * have reserved. + * + * @return this object, for chaining + * @param bytes number of bytes to commit + * @param hiMark maximum number of unflushed bytes after commit + **/ + virtual Output &commit(size_t bytes, size_t hiMark) = 0; + + virtual ~Output() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/provider.cpp b/vbench/src/vbench/core/provider.cpp new file mode 100644 index 00000000000..0cd3c10f0da --- /dev/null +++ b/vbench/src/vbench/core/provider.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "provider.h" + +namespace vbench { + +namespace { + +struct DummyItem {}; + +} // namespace vbench::<unnamed> + +template class Provider<DummyItem>; + +} // namespace vbench diff --git a/vbench/src/vbench/core/provider.h b/vbench/src/vbench/core/provider.h new file mode 100644 index 00000000000..c72079c828e --- /dev/null +++ b/vbench/src/vbench/core/provider.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace vbench { + +/** + * A Provider is a component from which you can request an object. + **/ +template <typename T> +struct Provider +{ + virtual std::unique_ptr<T> provide() = 0; + virtual ~Provider() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/simple_buffer.cpp b/vbench/src/vbench/core/simple_buffer.cpp new file mode 100644 index 00000000000..1e1d0a8e4ae --- /dev/null +++ b/vbench/src/vbench/core/simple_buffer.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "simple_buffer.h" + +namespace vbench { + +SimpleBuffer::SimpleBuffer() + : _data(), + _used(0) +{ +} + +Memory +SimpleBuffer::obtain(size_t bytes, size_t) +{ + return Memory(&_data[0], std::min(bytes, _used)); +} + +Input & +SimpleBuffer::evict(size_t bytes) +{ + assert(bytes <= _used); + _data.erase(_data.begin(), _data.begin() + bytes); + _used -= bytes; + return *this; +} + +WritableMemory +SimpleBuffer::reserve(size_t bytes) +{ + _data.resize(_used + bytes, char(0x55)); + return WritableMemory(&_data[_used], bytes); +} + +Output & +SimpleBuffer::commit(size_t bytes, size_t) +{ + assert(bytes <= (_data.size() - _used)); + _used += bytes; + return *this; +} + +bool +SimpleBuffer::operator==(const SimpleBuffer &rhs) const +{ + Memory a = get(); + Memory b = rhs.get(); + if (a.size != b.size) { + return false; + } + for (size_t i = 0; i < a.size; ++i) { + if (a.data[i] != b.data[i]) { + return false; + } + } + return true; +} + +std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf) { + Memory memory = buf.get(); + uint32_t written = 0; + uint32_t hexCount = 25; + os << "size: " << memory.size << "(bytes)" << std::endl; + for (size_t i = 0; i < memory.size; ++i, ++written) { + if (written > hexCount) { + os << std::endl; + written = 0; + } + os << strfmt("0x%02x ", memory.data[i] & 0xff); + } + if (written > 0) { + os << std::endl; + } + return os; +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/simple_buffer.h b/vbench/src/vbench/core/simple_buffer.h new file mode 100644 index 00000000000..173d774daf6 --- /dev/null +++ b/vbench/src/vbench/core/simple_buffer.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "memory.h" +#include "input.h" +#include "output.h" +#include <ostream> + +namespace vbench { + +/** + * Simple buffer class that implements the Input/Output + * interfaces. Requesting the memory region of this buffer or + * comparing buffers will only look at the data conceptually contained + * in the buffer, ignoring evicted data and reserved data not yet + * committed. + **/ +class SimpleBuffer : public Input, + public Output +{ +private: + std::vector<char> _data; + size_t _used; + +public: + SimpleBuffer(); + Memory get() const { return Memory(&_data[0], _used); } + virtual Memory obtain(size_t bytes, size_t lowMark); + virtual Input &evict(size_t bytes); + virtual WritableMemory reserve(size_t bytes); + virtual Output &commit(size_t bytes, size_t hiMark); + bool operator==(const SimpleBuffer &rhs) const; +}; + +std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf); + +} // namespace vbench + diff --git a/vbench/src/vbench/core/socket.cpp b/vbench/src/vbench/core/socket.cpp new file mode 100644 index 00000000000..265f41c3040 --- /dev/null +++ b/vbench/src/vbench/core/socket.cpp @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "socket.h" + +namespace vbench { + +Socket::Socket(std::unique_ptr<FastOS_SocketInterface> socket) + : _socket(std::move(socket)), + _input(), + _output(), + _taint(), + _eof(false) +{ +} + +Socket::Socket(const string host, int port) + : _socket(new FastOS_Socket()), + _input(), + _output(), + _taint(), + _eof(false) +{ + if (!_socket->SetAddressByHostName(port, host.c_str()) || + !_socket->SetSoBlocking(true) || + !_socket->Connect() || + !_socket->SetSoLinger(false, 0)) + { + _socket->Close(); + _taint.reset(strfmt("socket connect failed: host: %s, port: %d", + host.c_str(), port)); + } +} + +Socket::~Socket() +{ + _socket->Close(); +} + +Memory +Socket::obtain(size_t bytes, size_t lowMark) +{ + if (_input.get().size < bytes && _input.get().size < lowMark && !_eof && !_taint) { + WritableMemory buf = _input.reserve(bytes - _input.get().size); + ssize_t res = _socket->Read(buf.data, buf.size); + if (res > 0) { + _input.commit(res, 0); + } else if (res < 0) { + _taint.reset("socket read error"); + } else { + _eof = true; + } + } + return _input.obtain(bytes, 1); +} + +Input & +Socket::evict(size_t bytes) +{ + _input.evict(bytes); + return *this; +} + +WritableMemory +Socket::reserve(size_t bytes) +{ + return _output.reserve(bytes); +} + +Output & +Socket::commit(size_t bytes, size_t hiMark) +{ + _output.commit(bytes, 0); + while (_output.get().size > hiMark && !_taint) { + Memory buf = _output.obtain(_output.get().size, 1); + ssize_t res = _socket->Write(buf.data, buf.size); + if (res > 0) { + _output.evict(res); + } else { + _taint.reset("socket write error"); + } + } + return *this; +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h new file mode 100644 index 00000000000..4a281555bec --- /dev/null +++ b/vbench/src/vbench/core/socket.h @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "string.h" +#include "stream.h" +#include "simple_buffer.h" +#include <memory> + +namespace vbench { + +class Socket : public Stream +{ +private: + std::unique_ptr<FastOS_SocketInterface> _socket; + SimpleBuffer _input; + SimpleBuffer _output; + Taint _taint; + bool _eof; + +public: + Socket(std::unique_ptr<FastOS_SocketInterface> socket); + Socket(const string host, int port); + virtual ~Socket(); + virtual bool eof() const { return _eof; } + virtual Memory obtain(size_t bytes, size_t lowMark); + virtual Input &evict(size_t bytes); + virtual WritableMemory reserve(size_t bytes); + virtual Output &commit(size_t bytes, size_t hiMark); + virtual const Taint &tainted() const { return _taint; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/stream.cpp b/vbench/src/vbench/core/stream.cpp new file mode 100644 index 00000000000..84eac5feeaf --- /dev/null +++ b/vbench/src/vbench/core/stream.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "stream.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/stream.h b/vbench/src/vbench/core/stream.h new file mode 100644 index 00000000000..8a678647e6f --- /dev/null +++ b/vbench/src/vbench/core/stream.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "input.h" +#include "output.h" +#include "taintable.h" +#include <memory> + +namespace vbench { + +/** + * A Stream is an abstract taintable entity that can act as both input + * and output. + **/ +struct Stream : public Input, + public Output, + public Taintable +{ + typedef std::unique_ptr<Stream> UP; + virtual bool eof() const = 0; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/string.cpp b/vbench/src/vbench/core/string.cpp new file mode 100644 index 00000000000..35f1113d7bc --- /dev/null +++ b/vbench/src/vbench/core/string.cpp @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "string.h" + +#if USE_VESPA_STRING +#include <vespa/vespalib/util/vstringfmt.h> +#else +#include <vespa/vespalib/util/stringfmt.h> +#endif + +namespace vbench { + +string strfmt(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); +#if USE_VESPA_STRING + string ret = vespalib::make_vespa_string_va(fmt, ap); +#else + string ret = vespalib::make_string_va(fmt, ap); +#endif + va_end(ap); + return ret; +} + +size_t splitstr(const string &str, const string &sep, + std::vector<string> &dst) +{ + dst.clear(); + string tmp; + for (size_t i = 0; i < str.size(); ++i) { + if (sep.find(str[i]) != string::npos) { + if (!tmp.empty()) { + dst.push_back(tmp); + tmp.clear(); + } + } else { + tmp.push_back(str[i]); + } + } + if (!tmp.empty()) { + dst.push_back(tmp); + } + return dst.size(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/string.h b/vbench/src/vbench/core/string.h new file mode 100644 index 00000000000..61a2b3b442c --- /dev/null +++ b/vbench/src/vbench/core/string.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#define USE_VESPA_STRING 1 + +#if USE_VESPA_STRING +#include <vespa/vespalib/stllike/string.h> +#else +#include <string> +#endif + +namespace vbench { + +// define which string class to use +#if USE_VESPA_STRING +typedef vespalib::string string; +#else +typedef std::string string; +#endif + +extern string strfmt(const char *fmt, ...) +#ifdef __GNUC__ + // Add printf format checks with gcc + __attribute__ ((format (printf,1,2))) +#endif + ; + +extern size_t splitstr(const string &str, const string &sep, + std::vector<string> &dst); + +} // namespace vbench + diff --git a/vbench/src/vbench/core/taint.cpp b/vbench/src/vbench/core/taint.cpp new file mode 100644 index 00000000000..afb28f48d88 --- /dev/null +++ b/vbench/src/vbench/core/taint.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "taint.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/taint.h b/vbench/src/vbench/core/taint.h new file mode 100644 index 00000000000..8e35257b2ba --- /dev/null +++ b/vbench/src/vbench/core/taint.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "string.h" + +namespace vbench { + +/** + * A Taint indicates whether something has gone wrong. It may also + * contain a textual reason. + **/ +class Taint +{ +private: + bool _taint; + string _reason; + +public: + Taint() : _taint(false), _reason() {} + Taint(const string &r) : _taint(true), _reason(r) {} + void reset() { _taint = false; _reason.clear(); } + void reset(const string &r) { _taint = true; _reason = r; } + bool taint() const { return _taint; } + const string &reason() const { return _reason; } + operator bool() const { return taint(); } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/taintable.cpp b/vbench/src/vbench/core/taintable.cpp new file mode 100644 index 00000000000..5409379e02c --- /dev/null +++ b/vbench/src/vbench/core/taintable.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "taintable.h" + +namespace vbench { + +namespace { + +struct Untaintable : Taintable { + Taint taint; + virtual const Taint &tainted() const { return taint; } + virtual ~Untaintable() {} +}; + +Untaintable untaintable; + +} // namespace vbench::<unnamed> + +const Taintable & +Taintable::nil() +{ + return untaintable; +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/taintable.h b/vbench/src/vbench/core/taintable.h new file mode 100644 index 00000000000..2702c2e2736 --- /dev/null +++ b/vbench/src/vbench/core/taintable.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "taint.h" + +namespace vbench { + +/** + * Interface used to report what went wrong. Implementing this + * interface indicates that something could go wrong. + **/ +struct Taintable +{ + static const Taintable &nil(); + virtual const Taint &tainted() const = 0; + virtual ~Taintable() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/time_queue.cpp b/vbench/src/vbench/core/time_queue.cpp new file mode 100644 index 00000000000..a207b91a7c6 --- /dev/null +++ b/vbench/src/vbench/core/time_queue.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "time_queue.h" + +namespace vbench { + +namespace { + +struct DummyItem {}; + +} // namespace vbench::<unnamed> + +template class TimeQueue<DummyItem>; + +} // namespace vbench diff --git a/vbench/src/vbench/core/time_queue.h b/vbench/src/vbench/core/time_queue.h new file mode 100644 index 00000000000..b7c287dbdc4 --- /dev/null +++ b/vbench/src/vbench/core/time_queue.h @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/priority_queue.h> +#include <vespa/vespalib/util/sync.h> +#include <memory> + +#include "closeable.h" + +namespace vbench { + +/** + * A thread-safe priority queue keeping track of objects queued + * according to an abstract time line. After a time queue is closed, + * all incoming objects will be deleted. + **/ +template <typename T> +class TimeQueue : public Closeable +{ +private: + struct Entry { + std::unique_ptr<T> object; + double time; + Entry(std::unique_ptr<T> obj, double t) : object(std::move(obj)), time(t) {} + Entry(Entry &&rhs) : object(std::move(rhs.object)), time(rhs.time) {} + Entry &operator=(Entry &&rhs) { + object = std::move(rhs.object); + time = rhs.time; + return *this; + } + bool operator<(const Entry &rhs) const { + return (time < rhs.time); + } + }; + + vespalib::Monitor _monitor; + double _time; + double _window; + double _tick; + vespalib::PriorityQueue<Entry> _queue; + bool _closed; + +public: + TimeQueue(double window, double tick); + virtual void close(); + void discard(); + void insert(std::unique_ptr<T> obj, double time); + bool extract(double time, std::vector<std::unique_ptr<T> > &list, double &delay); +}; + +} // namespace vbench + +#include "time_queue.hpp" + diff --git a/vbench/src/vbench/core/time_queue.hpp b/vbench/src/vbench/core/time_queue.hpp new file mode 100644 index 00000000000..f84f319d038 --- /dev/null +++ b/vbench/src/vbench/core/time_queue.hpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace vbench { + +template <typename T> +TimeQueue<T>::TimeQueue(double window, double tick) + : _monitor(), + _time(0.0), + _window(window), + _tick(tick), + _queue(), + _closed(false) +{ +} + +template <typename T> +void +TimeQueue<T>::close() +{ + vespalib::MonitorGuard guard(_monitor); + _closed = true; + guard.broadcast(); +} + +template <typename T> +void +TimeQueue<T>::discard() +{ + vespalib::MonitorGuard guard(_monitor); + while (!_queue.empty()) { + _queue.pop_any(); + } + guard.broadcast(); +} + +template <typename T> +void +TimeQueue<T>::insert(std::unique_ptr<T> obj, double time) +{ + vespalib::MonitorGuard guard(_monitor); + while (time > (_time + _window) && !_closed) { + guard.wait(); + } + if (!_closed) { + _queue.push(Entry(std::move(obj), time)); + } +} + +template <typename T> +bool +TimeQueue<T>::extract(double time, std::vector<std::unique_ptr<T> > &list, double &delay) +{ + vespalib::MonitorGuard guard(_monitor); + _time = time; + while (!_queue.empty() && _queue.front().time <= time) { + list.push_back(std::move(_queue.front().object)); + _queue.pop_front(); + } + guard.broadcast(); + delay = _queue.empty() ? _tick : (_queue.front().time - time); + return (!_closed || !_queue.empty()); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/timer.cpp b/vbench/src/vbench/core/timer.cpp new file mode 100644 index 00000000000..8f262d3f0cd --- /dev/null +++ b/vbench/src/vbench/core/timer.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "timer.h" + +namespace vbench { + +Timer::Timer() + : _time() +{ + reset(); +} + +void +Timer::reset() +{ + _time.SetNow(); +} + +double +Timer::sample() const +{ + return (_time.MilliSecsToNow() / 1000.0); +} + +} // namespace vbench diff --git a/vbench/src/vbench/core/timer.h b/vbench/src/vbench/core/timer.h new file mode 100644 index 00000000000..e1b1132089b --- /dev/null +++ b/vbench/src/vbench/core/timer.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vbench { + +/** + * Simple utility class used to handle low-level time sampling. + **/ +class Timer +{ +private: + FastOS_Time _time; + +public: + Timer(); + void reset(); + double sample() const; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/core/writable_memory.cpp b/vbench/src/vbench/core/writable_memory.cpp new file mode 100644 index 00000000000..2da24dba10d --- /dev/null +++ b/vbench/src/vbench/core/writable_memory.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "writable_memory.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/core/writable_memory.h b/vbench/src/vbench/core/writable_memory.h new file mode 100644 index 00000000000..3dbf85bdfee --- /dev/null +++ b/vbench/src/vbench/core/writable_memory.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vbench { + +/** + * Simple wrapper referencing a writable region of memory. This class + * does not have ownership of the referenced memory region. + **/ +struct WritableMemory { + char *data; + size_t size; + WritableMemory() : data(0), size(0) {} + WritableMemory(char *d, size_t s) : data(d), size(s) {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/.gitignore b/vbench/src/vbench/http/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/vbench/src/vbench/http/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/vbench/src/vbench/http/CMakeLists.txt b/vbench/src/vbench/http/CMakeLists.txt new file mode 100644 index 00000000000..3838d7c8248 --- /dev/null +++ b/vbench/src/vbench/http/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vbench_http OBJECT + SOURCES + benchmark_headers.cpp + hex_number.cpp + http_client.cpp + http_connection.cpp + http_connection_pool.cpp + http_result_handler.cpp + server_spec.cpp + DEPENDS +) diff --git a/vbench/src/vbench/http/benchmark_headers.cpp b/vbench/src/vbench/http/benchmark_headers.cpp new file mode 100644 index 00000000000..df96f6a9dc7 --- /dev/null +++ b/vbench/src/vbench/http/benchmark_headers.cpp @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "benchmark_headers.h" +#include <map> + +namespace vbench { + +namespace benchmark_headers { +const std::string NUM_HITS = "X-Yahoo-Vespa-NumHits"; +const std::string NUM_FASTHITS = "X-Yahoo-Vespa-NumFastHits"; +const std::string NUM_GROUPHITS = "X-Yahoo-Vespa-NumGroupHits"; +const std::string NUM_ERRORS = "X-Yahoo-Vespa-NumErrors"; +const std::string TOTAL_HIT_COUNT = "X-Yahoo-Vespa-TotalHitCount"; +const std::string NUM_DOCSUMS = "X-Yahoo-Vespa-NumDocsums"; +const std::string QUERY_HITS = "X-Yahoo-Vespa-QueryHits"; +const std::string QUERY_OFFSET = "X-Yahoo-Vespa-QueryOffset"; +const std::string SEARCH_TIME = "X-Yahoo-Vespa-SearchTime"; +const std::string ATTR_TIME = "X-Yahoo-Vespa-AttributeFetchTime"; +const std::string FILL_TIME = "X-Yahoo-Vespa-FillTime"; +const std::string DOCS_SEARCHED = "X-Yahoo-Vespa-DocsSearched"; +const std::string NODES_SEARCHED = "X-Yahoo-Vespa-NodesSearched"; +const std::string FULL_COVERAGE = "X-Yahoo-Vespa-FullCoverage"; +struct HeaderTraverser { + virtual ~HeaderTraverser() { } + virtual void header(const string &name, double value) = 0; +}; +struct HeaderMapper { + typedef BenchmarkHeaders::Value BenchmarkHeaders::*ValueRef; + typedef std::map<string,ValueRef> HeaderMap; + typedef std::map<string,ValueRef>::iterator HeaderEntry; + HeaderMap map; + HeaderMapper() : map() { + map[NUM_HITS] = &BenchmarkHeaders::num_hits; + map[NUM_FASTHITS] = &BenchmarkHeaders::num_fasthits; + map[NUM_GROUPHITS] = &BenchmarkHeaders::num_grouphits; + map[NUM_ERRORS] = &BenchmarkHeaders::num_errors; + map[TOTAL_HIT_COUNT] = &BenchmarkHeaders::total_hit_count; + map[NUM_DOCSUMS] = &BenchmarkHeaders::num_docsums; + map[QUERY_HITS] = &BenchmarkHeaders::query_hits; + map[QUERY_OFFSET] = &BenchmarkHeaders::query_offset; + map[SEARCH_TIME] = &BenchmarkHeaders::search_time; + map[ATTR_TIME] = &BenchmarkHeaders::attr_time; + map[FILL_TIME] = &BenchmarkHeaders::fill_time; + map[DOCS_SEARCHED] = &BenchmarkHeaders::docs_searched; + map[NODES_SEARCHED] = &BenchmarkHeaders::nodes_searched; + map[FULL_COVERAGE] = &BenchmarkHeaders::full_coverage; + } + void apply(BenchmarkHeaders &headers, const string &name, const string &string_value) { + HeaderEntry entry = map.find(name); + if (entry != map.end()) { + (headers.*(entry->second)).set(string_value); + } + } + void traverse(const BenchmarkHeaders &headers, HeaderTraverser &traverser) { + for (HeaderEntry entry = map.begin(); entry != map.end(); ++entry) { + if ((headers.*(entry->second)).is_set) { + traverser.header(entry->first, (headers.*(entry->second)).value); + } + } + } +}; +HeaderMapper header_mapper; +} // namespace vbench::benchmark_headers + +void +BenchmarkHeaders::handleHeader(const string &name, const string &string_value) +{ + benchmark_headers::header_mapper.apply(*this, name, string_value); +} + +string +BenchmarkHeaders::toString() const +{ + string str = ""; + struct HeaderToString : benchmark_headers::HeaderTraverser { + string &str; + HeaderToString(string &s) : str(s) {} + virtual void header(const string &name, double value) { + str += strfmt(" %s: %g\n", name.c_str(), value); + } + } headerToString(str); + benchmark_headers::header_mapper.traverse(*this, headerToString); + return str; +} + +} // namespace vbench diff --git a/vbench/src/vbench/http/benchmark_headers.h b/vbench/src/vbench/http/benchmark_headers.h new file mode 100644 index 00000000000..317eb2fb2d4 --- /dev/null +++ b/vbench/src/vbench/http/benchmark_headers.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/string.h> + +namespace vbench { + +/** + * Special benchmark headers that can be returned from the QRS. All + * values are converted to double. They are also bundled with a flag + * indicating whether they have been set or not. + **/ +struct BenchmarkHeaders +{ + struct Value { + double value; + bool is_set; + Value() : value(0.0), is_set(false) {} + void set(const string &string_value) { + char *end; + errno = 0; + double val = strtod(string_value.c_str(), &end); + if (errno == 0 && *end == '\0') { + value = val; + is_set = true; + } + } + }; + Value num_hits; + Value num_fasthits; + Value num_grouphits; + Value num_errors; + Value total_hit_count; + Value num_docsums; + Value query_hits; + Value query_offset; + Value search_time; + Value attr_time; + Value fill_time; + Value docs_searched; + Value nodes_searched; + Value full_coverage; + void handleHeader(const string &name, const string &string_value); + string toString() const; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/hex_number.cpp b/vbench/src/vbench/http/hex_number.cpp new file mode 100644 index 00000000000..54bd2c9b660 --- /dev/null +++ b/vbench/src/vbench/http/hex_number.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "hex_number.h" + +namespace vbench { + +HexNumber::HexNumber(const char *str) + : _value(0), + _length(0) +{ + while (*str != '\0') { + switch (*str++) { + case '0': _value = (_value << 4) + 0; break; + case '1': _value = (_value << 4) + 1; break; + case '2': _value = (_value << 4) + 2; break; + case '3': _value = (_value << 4) + 3; break; + case '4': _value = (_value << 4) + 4; break; + case '5': _value = (_value << 4) + 5; break; + case '6': _value = (_value << 4) + 6; break; + case '7': _value = (_value << 4) + 7; break; + case '8': _value = (_value << 4) + 8; break; + case '9': _value = (_value << 4) + 9; break; + case 'a': case 'A': _value = (_value << 4) + 10; break; + case 'b': case 'B': _value = (_value << 4) + 11; break; + case 'c': case 'C': _value = (_value << 4) + 12; break; + case 'd': case 'D': _value = (_value << 4) + 13; break; + case 'e': case 'E': _value = (_value << 4) + 14; break; + case 'f': case 'F': _value = (_value << 4) + 15; break; + default: return; + } + ++_length; + } +} + +} // namespace vbench diff --git a/vbench/src/vbench/http/hex_number.h b/vbench/src/vbench/http/hex_number.h new file mode 100644 index 00000000000..bc78ff72582 --- /dev/null +++ b/vbench/src/vbench/http/hex_number.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +namespace vbench { + +/** + * Utility class used to work with hex number used in HTTP chunks. + **/ +class HexNumber +{ +private: + size_t _value; + size_t _length; + +public: + HexNumber(const char *str); + size_t value() const { return _value; } + size_t length() const { return _length; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/http_client.cpp b/vbench/src/vbench/http/http_client.cpp new file mode 100644 index 00000000000..a83ad41b41a --- /dev/null +++ b/vbench/src/vbench/http/http_client.cpp @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "http_client.h" +#include "hex_number.h" +#include <vbench/core/buffered_output.h> +#include <vbench/core/line_reader.h> +#include <algorithm> + +namespace vbench { + +void +HttpClient::writeRequest() { + BufferedOutput dst(_conn->stream(), WRITE_SIZE); + dst.printf("GET %s HTTP/1.1\r\n", _url.c_str()); + dst.printf("Host: %s\r\n", _conn->server().host.c_str()); + dst.append("User-Agent: vbench\r\n"); + dst.append("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); + dst.append("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); + dst.append("\r\n"); +} + +bool +HttpClient::readStatus() +{ + LineReader reader(_conn->stream(), READ_SIZE); + if (reader.readLine(_line) && (splitstr(_line, "\t ", _split) >= 2)) { + if (_split[0] == "HTTP/1.0") { + _header.version = 0; + } else if (_split[0] == "HTTP/1.1") { + _header.version = 1; + } else { + _handler.handleFailure(strfmt("unknown HTTP version: '%s'", _split[0].c_str())); + return false; + } + _header.status = atoi(_split[1].c_str()); + if (_header.status != 200) { + _handler.handleFailure(strfmt("HTTP status not 200: '%s'", _split[1].c_str())); + return false; + } + return true; + } + if (_conn->stream().tainted()) { + _handler.handleFailure(strfmt("Connection error: %s", + _conn->stream().tainted().reason().c_str())); + } else { + _handler.handleFailure(strfmt("could not parse HTTP status line: '%s'", _line.c_str())); + } + return false; +} + +bool +HttpClient::readHeaders() +{ + LineReader reader(_conn->stream(), READ_SIZE); + while (reader.readLine(_line)) { + if (_line.empty()) { + return true; + } + if ((_line[0] == ' ') || (_line[0] == '\t')) { + // ignore continuation headers + } else if (_line.find("X-Yahoo-Vespa-") == 0) { + if (splitstr(_line, ":\t ", _split) == 2) { + _handler.handleHeader(_split[0], _split[1]); + } + } else { + if (splitstr(_line, ":\t ", _split) > 1) { + if (strcasecmp(_split[0].c_str(), "connection") == 0) { + for (size_t i = 1; i < _split.size(); ++i) { + if (strcasecmp(_split[i].c_str(), "keep-alive") == 0) { + _handler.handleHeader(_split[0], _split[i]); + _header.keepAliveGiven = true; + } else if (strcasecmp(_split[i].c_str(), "close") == 0) { + _handler.handleHeader(_split[0], _split[i]); + _header.connectionCloseGiven = true; + } + } + } else if (strcasecmp(_split[0].c_str(), "content-length") == 0 && + _split.size() == 2) + { + _handler.handleHeader(_split[0], _split[1]); + _header.contentLengthGiven = true; + _header.contentLength = atoi(_split[1].c_str()); + } else if (strcasecmp(_split[0].c_str(), "transfer-encoding") == 0 && + strcasecmp(_split[1].c_str(), "chunked") == 0) + { + _handler.handleHeader(_split[0], _split[1]); + _header.chunkedEncodingGiven = true; + } + } + } + } + _handler.handleFailure("HTTP header did not end in empty line"); + return false; +} + +bool +HttpClient::readContent(size_t len) { + Input &input = _conn->stream(); + while (len > 0) { + Memory mem = input.obtain(READ_SIZE, 1); + mem.size = std::min(len, mem.size); + if (mem.size == 0) { + _handler.handleFailure(strfmt("short read: missing %zu bytes", len)); + return false; + } + _handler.handleContent(mem); + input.evict(mem.size); + len -= mem.size; + } + return true; +} + +bool +HttpClient::readChunkSize(bool first, size_t &size) +{ + LineReader reader(_conn->stream(), READ_SIZE); + if (!first && (!reader.readLine(_line) || !_line.empty())) { + return false; + } + if (!reader.readLine(_line)) { + return false; + } + HexNumber hex(_line.c_str()); + size = hex.value(); + return (hex.length() > 0); +} + +bool +HttpClient::skipTrailers() +{ + LineReader reader(_conn->stream(), READ_SIZE); + while (reader.readLine(_line)) { + if (_line.empty()) { + return true; + } + } + return false; +} + +bool +HttpClient::readContent() +{ + if (_header.contentLengthGiven) { + return readContent(_header.contentLength); + } else if (_header.chunkedEncodingGiven) { + size_t chunkSize = 0; + for (bool first = true; readChunkSize(first, chunkSize); first = false) { + if (chunkSize == 0) { + return skipTrailers(); + } + if (!readContent(chunkSize)) { + return false; + } + } + _handler.handleFailure("error reading HTTP chunk size"); + return false; + } else { // data terminated by eof + if (serverKeepAlive()) { + _handler.handleFailure("server indicated keep-alive, " + "but we need eof to terminate data"); + return false; + } + Input &input = _conn->stream(); + for (;;) { + Memory mem = input.obtain(READ_SIZE, 1); + if (mem.size == 0) { + if (_conn->stream().tainted()) { + _handler.handleFailure(strfmt("read error: '%s'", + _conn->stream().tainted().reason().c_str())); + } + return true; + } + _handler.handleContent(mem); + input.evict(mem.size); + } + } +} + +bool +HttpClient::perform() +{ + writeRequest(); + if (!_conn->fresh() && (_conn->stream().obtain(READ_SIZE, 1).size == 0)) { + _conn.reset(new HttpConnection(_conn->server())); + writeRequest(); + } + return (readStatus() && readHeaders() && readContent()); +} + +} // namespace vbench diff --git a/vbench/src/vbench/http/http_client.h b/vbench/src/vbench/http/http_client.h new file mode 100644 index 00000000000..b80dca2c819 --- /dev/null +++ b/vbench/src/vbench/http/http_client.h @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/socket.h> +#include "http_connection.h" +#include "http_connection_pool.h" +#include "http_result_handler.h" + +namespace vbench { + +/** + * This class handles sequential HTTP requests against a single + * server. + **/ +class HttpClient +{ +private: + static const size_t READ_SIZE = 8000; + static const size_t WRITE_SIZE = 2000; + + struct HeaderInfo { + bool connectionCloseGiven; + bool contentLengthGiven; + bool chunkedEncodingGiven; + bool keepAliveGiven; + uint32_t status; + uint32_t version; + size_t contentLength; + HeaderInfo() : connectionCloseGiven(false), contentLengthGiven(false), + chunkedEncodingGiven(false), keepAliveGiven(false), + status(0), version(0) {} + }; + + HttpConnection::UP _conn; + string _url; + HttpResultHandler &_handler; + HeaderInfo _header; + + // scratch data used for parsing + string _line; + std::vector<string> _split; + + HttpClient(HttpConnection::UP conn, const string &url, HttpResultHandler &handler) + : _conn(std::move(conn)), _url(url), _handler(handler), _header() {} + + bool serverKeepAlive() const { + return ((_header.version == 1 && !_header.connectionCloseGiven) || + (_header.version == 0 && _header.keepAliveGiven)); + } + + void writeRequest(); + bool readStatus(); + bool readHeaders(); + bool readContent(size_t len); + bool readChunkSize(bool first, size_t &size); + bool skipTrailers(); + bool readContent(); + bool perform(); + +public: + static bool fetch(const ServerSpec &server, const string &url, + HttpResultHandler &handler) + { + HttpClient client(HttpConnection::UP(new HttpConnection(server)), url, handler); + return client.perform(); + } + static bool fetch(HttpConnectionPool &pool, + const ServerSpec &server, const string &url, + HttpResultHandler &handler) + { + HttpClient client(pool.getConnection(server), url, handler); + if (client.perform()) { + if (client.serverKeepAlive()) { + pool.putConnection(std::move(client._conn)); + } + return true; + } + return false; + } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/http_connection.cpp b/vbench/src/vbench/http/http_connection.cpp new file mode 100644 index 00000000000..e76ae44fd6e --- /dev/null +++ b/vbench/src/vbench/http/http_connection.cpp @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "http_connection.h" + +namespace vbench { + +HttpConnection::HttpConnection(const ServerSpec &s) + : _server(s), + _socket(s.host, s.port), + _lastUsed(-1000.0) +{ +} + +bool +HttpConnection::mayReuse(double now) const +{ + return (((now - _lastUsed) < 1.0) && + !_socket.eof() && + !_socket.tainted()); +} + +} // namespace vbench diff --git a/vbench/src/vbench/http/http_connection.h b/vbench/src/vbench/http/http_connection.h new file mode 100644 index 00000000000..e390765f928 --- /dev/null +++ b/vbench/src/vbench/http/http_connection.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> +#include <vbench/core/socket.h> +#include "server_spec.h" + +namespace vbench { + +/** + * A connection to a specific server that can be reused at a later + * time to support persistent connections. + **/ +class HttpConnection +{ +private: + ServerSpec _server; + Socket _socket; + double _lastUsed; + +public: + typedef std::unique_ptr<HttpConnection> UP; + + HttpConnection(const ServerSpec &server); + bool fresh() const { return (_lastUsed < 0); } + const ServerSpec &server() const { return _server; } + Stream &stream() { return _socket; } + void touch(double now) { _lastUsed = now; } + bool mayReuse(double now) const; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/http_connection_pool.cpp b/vbench/src/vbench/http/http_connection_pool.cpp new file mode 100644 index 00000000000..dd1c8d9488b --- /dev/null +++ b/vbench/src/vbench/http/http_connection_pool.cpp @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "http_connection_pool.h" + +namespace vbench { + +HttpConnectionPool::HttpConnectionPool(Timer &timer) + : _lock(), + _map(), + _store(), + _timer(timer) +{ +} + +HttpConnection::UP +HttpConnectionPool::getConnection(const ServerSpec &server) +{ + double now = _timer.sample(); + vespalib::LockGuard guard(_lock); + auto res = _map.insert(std::make_pair(server, _store.size())); + if (res.second) { + _store.emplace_back(); + } + Queue &queue = _store[res.first->second]; + while (!queue.empty() && !queue.front()->mayReuse(now)) { + queue.pop(); + } + if (!queue.empty()) { + HttpConnection::UP ret = std::move(queue.access(0)); + queue.pop(); + return ret; + } + return HttpConnection::UP(new HttpConnection(server)); +} + +void +HttpConnectionPool::putConnection(HttpConnection::UP conn) +{ + double now = _timer.sample(); + vespalib::LockGuard guard(_lock); + conn->touch(now); + size_t idx = _map[conn->server()]; + assert(idx < _store.size()); + _store[idx].push(std::move(conn)); +} + +} // namespace vbench diff --git a/vbench/src/vbench/http/http_connection_pool.h b/vbench/src/vbench/http/http_connection_pool.h new file mode 100644 index 00000000000..95a069ef6f1 --- /dev/null +++ b/vbench/src/vbench/http/http_connection_pool.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <map> +#include <vespa/vespalib/util/arrayqueue.hpp> +#include <vespa/vespalib/util/sync.h> +#include <vbench/core/timer.h> +#include "http_connection.h" + +namespace vbench { + +/** + * A pool of http connections used to support persistent + * connections. The pool is shared between threads to reduce the + * number of needed connections when using many servers. + **/ +class HttpConnectionPool +{ +private: + typedef vespalib::ArrayQueue<HttpConnection::UP> Queue; + typedef std::map<ServerSpec, size_t> Map; + + vespalib::Lock _lock; + Map _map; + std::vector<Queue> _store; + Timer &_timer; + +public: + HttpConnectionPool(Timer &timer); + HttpConnection::UP getConnection(const ServerSpec &server); + void putConnection(HttpConnection::UP conn); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/http_result_handler.cpp b/vbench/src/vbench/http/http_result_handler.cpp new file mode 100644 index 00000000000..2dedb58ad05 --- /dev/null +++ b/vbench/src/vbench/http/http_result_handler.cpp @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "http_result_handler.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/http/http_result_handler.h b/vbench/src/vbench/http/http_result_handler.h new file mode 100644 index 00000000000..9c6abb24131 --- /dev/null +++ b/vbench/src/vbench/http/http_result_handler.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/string.h> +#include <vbench/core/memory.h> + +namespace vbench { + +/** + * Callback interface that must be implemented in order to use the + * http client. + **/ +struct HttpResultHandler +{ + virtual void handleHeader(const string &name, const string &value) = 0; + virtual void handleContent(const Memory &data) = 0; + virtual void handleFailure(const string &reason) = 0; + virtual ~HttpResultHandler() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/http/server_spec.cpp b/vbench/src/vbench/http/server_spec.cpp new file mode 100644 index 00000000000..dae981a3ded --- /dev/null +++ b/vbench/src/vbench/http/server_spec.cpp @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "server_spec.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/http/server_spec.h b/vbench/src/vbench/http/server_spec.h new file mode 100644 index 00000000000..efe72f27b5b --- /dev/null +++ b/vbench/src/vbench/http/server_spec.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/string.h> + +namespace vbench { + +/** + * Simple wrapper specifying the host and port of a server. This will + * typically be a HTTP server. + **/ +struct ServerSpec +{ + string host; + int port; + + ServerSpec() : host(), port(0) {} + ServerSpec(const string &h, int p) : host(h), port(p) {} + bool operator==(const ServerSpec &rhs) const { + return (port == rhs.port && host == rhs.host); + } + bool operator<(const ServerSpec &rhs) const { + if (port == rhs.port) { + return (host < rhs.host); + } + return (port < rhs.port); + } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/test/.gitignore b/vbench/src/vbench/test/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/vbench/src/vbench/test/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/vbench/src/vbench/test/CMakeLists.txt b/vbench/src/vbench/test/CMakeLists.txt new file mode 100644 index 00000000000..ddd37822969 --- /dev/null +++ b/vbench/src/vbench/test/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vbench_test STATIC + SOURCES + request_receptor.cpp + server_socket.cpp + simple_http_result_handler.cpp + DEPENDS +) diff --git a/vbench/src/vbench/test/all.h b/vbench/src/vbench/test/all.h new file mode 100644 index 00000000000..37f0a4d2ac7 --- /dev/null +++ b/vbench/src/vbench/test/all.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// This file was generated by create-all-h.sh + +#pragma once + +#include <vbench/vbench/qps_analyzer.h> +#include <vbench/vbench/request_dumper.h> +#include <vbench/vbench/request_scheduler.h> +#include <vbench/vbench/request_sink.h> +#include <vbench/vbench/qps_tagger.h> +#include <vbench/vbench/dropped_tagger.h> +#include <vbench/vbench/worker.h> +#include <vbench/vbench/vbench.h> +#include <vbench/vbench/request_generator.h> +#include <vbench/vbench/server_tagger.h> +#include <vbench/vbench/request.h> +#include <vbench/vbench/latency_analyzer.h> +#include <vbench/core/input_file_reader.h> +#include <vbench/core/line_reader.h> +#include <vbench/core/string.h> +#include <vbench/core/taint.h> +#include <vbench/core/taintable.h> +#include <vbench/core/output.h> +#include <vbench/core/memory.h> +#include <vbench/core/mapped_file_input.h> +#include <vbench/core/time_queue.h> +#include <vbench/core/buffered_output.h> +#include <vbench/core/socket.h> +#include <vbench/core/handler_thread.h> +#include <vbench/core/closeable.h> +#include <vbench/core/handler.h> +#include <vbench/core/timer.h> +#include <vbench/core/writable_memory.h> +#include <vbench/core/simple_buffer.h> +#include <vbench/core/provider.h> +#include <vbench/core/byte_input.h> +#include <vbench/core/dispatcher.h> +#include <vbench/core/stream.h> +#include <vbench/core/input.h> +#include <vbench/test/simple_http_result_handler.h> +#include <vbench/test/server_socket.h> +#include <vbench/test/request_receptor.h> +#include <vbench/http/hex_number.h> +#include <vbench/http/http_result_handler.h> +#include <vbench/http/http_client.h> +#include <vbench/http/http_connection.h> +#include <vbench/http/server_spec.h> +#include <vbench/http/http_connection_pool.h> + diff --git a/vbench/src/vbench/test/create-all-h.sh b/vbench/src/vbench/test/create-all-h.sh new file mode 100644 index 00000000000..01ab856193d --- /dev/null +++ b/vbench/src/vbench/test/create-all-h.sh @@ -0,0 +1,24 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#!/bin/sh +cd $(dirname $0) +out="all.h" +exec > $out +year=$(date +%Y) +cat <<EOF +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// This file was generated by $(basename $0) + + +EOF + +( + cd ../.. + find vbench -name "*.h" | grep -v "vbench/test/$out" | while read name; do + echo "#include <$name>" + done +) + +cat <<EOF + +EOF diff --git a/vbench/src/vbench/test/request_receptor.cpp b/vbench/src/vbench/test/request_receptor.cpp new file mode 100644 index 00000000000..b0407a9ce78 --- /dev/null +++ b/vbench/src/vbench/test/request_receptor.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request_receptor.h" + +namespace vbench { + +void +RequestReceptor::handle(Request::UP req) +{ + request = std::move(req); +} + +} // namespace vbench diff --git a/vbench/src/vbench/test/request_receptor.h b/vbench/src/vbench/test/request_receptor.h new file mode 100644 index 00000000000..738f1336bc6 --- /dev/null +++ b/vbench/src/vbench/test/request_receptor.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> +#include <vbench/vbench/request.h> + +namespace vbench { + +struct RequestReceptor : public Handler<Request> { + Request::UP request; + RequestReceptor() : request() {} + virtual void handle(Request::UP req); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/test/server_socket.cpp b/vbench/src/vbench/test/server_socket.cpp new file mode 100644 index 00000000000..b87a245dc88 --- /dev/null +++ b/vbench/src/vbench/test/server_socket.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "server_socket.h" +#include <vbench/core/socket.h> +#include <vespa/vespalib/util/thread.h> + +namespace vbench { + +ServerSocket::ServerSocket() + : _serverSocket(0, 500, 0, 0), + _closed(false) +{ + _serverSocket.SetSoBlocking(false); + _serverSocket.Listen(); +} + +Stream::UP +ServerSocket::accept() +{ + while (!_closed) { + std::unique_ptr<FastOS_SocketInterface> socket(_serverSocket.Accept()); + if (socket.get() != 0) { + socket->SetSoBlocking(true); + return Stream::UP(new Socket(std::move(socket))); + } else { + int error = FastOS_Socket::GetLastError(); + if (error == FastOS_Socket::ERR_WOULDBLOCK) { + vespalib::Thread::sleep(10); + } else { + return Stream::UP(); + } + } + } + return Stream::UP(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/test/server_socket.h b/vbench/src/vbench/test/server_socket.h new file mode 100644 index 00000000000..af3a3365309 --- /dev/null +++ b/vbench/src/vbench/test/server_socket.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/stream.h> + +namespace vbench { + +/** + * Simple server socket listening to a random port. + **/ +class ServerSocket +{ +private: + FastOS_ServerSocket _serverSocket; + volatile bool _closed; + +public: + ServerSocket(); + Stream::UP accept(); + int port() { return _serverSocket.GetLocalPort(); } + void close() { _closed = true; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/test/simple_http_result_handler.cpp b/vbench/src/vbench/test/simple_http_result_handler.cpp new file mode 100644 index 00000000000..a0896381f33 --- /dev/null +++ b/vbench/src/vbench/test/simple_http_result_handler.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "simple_http_result_handler.h" + +namespace vbench { + +SimpleHttpResultHandler::SimpleHttpResultHandler() + : _headers(), + _content(), + _failures() +{ +} + +void +SimpleHttpResultHandler::handleHeader(const string &name, const string &value) +{ + _headers.push_back(std::make_pair(name, value)); +} + +void +SimpleHttpResultHandler::handleContent(const Memory &data) +{ + WritableMemory wm = _content.reserve(data.size); + memcpy(wm.data, data.data, data.size); + _content.commit(data.size, 0); +} + +void +SimpleHttpResultHandler::handleFailure(const string &reason) +{ + _failures.push_back(reason); +} + +} // namespace vbench diff --git a/vbench/src/vbench/test/simple_http_result_handler.h b/vbench/src/vbench/test/simple_http_result_handler.h new file mode 100644 index 00000000000..4a6fd04b852 --- /dev/null +++ b/vbench/src/vbench/test/simple_http_result_handler.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/http/http_result_handler.h> +#include <vbench/core/simple_buffer.h> + +namespace vbench { + +class SimpleHttpResultHandler : public HttpResultHandler +{ +private: + std::vector<std::pair<string, string> > _headers; + SimpleBuffer _content; + std::vector<string> _failures; + +public: + SimpleHttpResultHandler(); + virtual void handleHeader(const string &name, const string &value); + virtual void handleContent(const Memory &data); + virtual void handleFailure(const string &reason); + const std::vector<std::pair<string, string> > &headers() const { + return _headers; + } + Memory content() const { return _content.get(); } + const std::vector<string> &failures() const { return _failures; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/.gitignore b/vbench/src/vbench/vbench/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/vbench/src/vbench/vbench/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/vbench/src/vbench/vbench/CMakeLists.txt b/vbench/src/vbench/vbench/CMakeLists.txt new file mode 100644 index 00000000000..b1d6ec7ebc6 --- /dev/null +++ b/vbench/src/vbench/vbench/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vbench_vbench_vbench OBJECT + SOURCES + analyzer.cpp + dropped_tagger.cpp + generator.cpp + ignore_before.cpp + latency_analyzer.cpp + native_factory.cpp + qps_analyzer.cpp + qps_tagger.cpp + request.cpp + request_dumper.cpp + request_generator.cpp + request_scheduler.cpp + request_sink.cpp + server_tagger.cpp + tagger.cpp + vbench.cpp + worker.cpp + DEPENDS +) diff --git a/vbench/src/vbench/vbench/analyzer.cpp b/vbench/src/vbench/vbench/analyzer.cpp new file mode 100644 index 00000000000..e92ce82c29f --- /dev/null +++ b/vbench/src/vbench/vbench/analyzer.cpp @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "analyzer.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/analyzer.h b/vbench/src/vbench/vbench/analyzer.h new file mode 100644 index 00000000000..e1bd8e23919 --- /dev/null +++ b/vbench/src/vbench/vbench/analyzer.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> +#include <vbench/core/handler.h> + +#include "request.h" + +namespace vbench { + +struct Analyzer : public Handler<Request> +{ + typedef std::unique_ptr<Analyzer> UP; + virtual void report() = 0; + virtual ~Analyzer() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/dropped_tagger.cpp b/vbench/src/vbench/vbench/dropped_tagger.cpp new file mode 100644 index 00000000000..888b2204bf1 --- /dev/null +++ b/vbench/src/vbench/vbench/dropped_tagger.cpp @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "dropped_tagger.h" + +namespace vbench { + +DroppedTagger::DroppedTagger(Handler<Request> &next) + : _next(next) +{ +} + +void +DroppedTagger::handle(Request::UP request) +{ + request->status(Request::STATUS_DROPPED); + _next.handle(std::move(request)); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/dropped_tagger.h b/vbench/src/vbench/vbench/dropped_tagger.h new file mode 100644 index 00000000000..6c05dcdaf10 --- /dev/null +++ b/vbench/src/vbench/vbench/dropped_tagger.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> + +#include "request.h" + +namespace vbench { + +/** + * Tags a request as dropped before passing it along. + **/ +class DroppedTagger : public Handler<Request> +{ +private: + Handler<Request> &_next; + +public: + DroppedTagger(Handler<Request> &next); + virtual void handle(Request::UP request); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/generator.cpp b/vbench/src/vbench/vbench/generator.cpp new file mode 100644 index 00000000000..55769e16cad --- /dev/null +++ b/vbench/src/vbench/vbench/generator.cpp @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "generator.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/generator.h b/vbench/src/vbench/vbench/generator.h new file mode 100644 index 00000000000..5eb8e056265 --- /dev/null +++ b/vbench/src/vbench/vbench/generator.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> +#include <vespa/vespalib/util/runnable.h> +#include <vbench/core/taintable.h> + +namespace vbench { + +struct Generator : public vespalib::Runnable, + public Taintable +{ + typedef std::unique_ptr<Generator> UP; + virtual void abort() = 0; + virtual ~Generator() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/ignore_before.cpp b/vbench/src/vbench/vbench/ignore_before.cpp new file mode 100644 index 00000000000..6459b872fdf --- /dev/null +++ b/vbench/src/vbench/vbench/ignore_before.cpp @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "ignore_before.h" + +namespace vbench { + +IgnoreBefore::IgnoreBefore(double time, Handler<Request> &next) + : _next(next), + _time(time), + _ignored(0) +{ +} + +void +IgnoreBefore::handle(Request::UP request) +{ + if (request->startTime() < _time) { + ++_ignored; + return; + } + _next.handle(std::move(request)); +} + +void +IgnoreBefore::report() +{ + fprintf(stdout, "ignored %zu requests\n", _ignored); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/ignore_before.h b/vbench/src/vbench/vbench/ignore_before.h new file mode 100644 index 00000000000..0be0add60c4 --- /dev/null +++ b/vbench/src/vbench/vbench/ignore_before.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> +#include "request.h" +#include "analyzer.h" + +namespace vbench { + +/** + * Component ignoring (discarding) requests that have start times + * before a specific time. + **/ +class IgnoreBefore : public Analyzer +{ +private: + Handler<Request> &_next; + double _time; + size_t _ignored; + +public: + IgnoreBefore(double time, Handler<Request> &next); + virtual void handle(Request::UP request); + virtual void report(); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/latency_analyzer.cpp b/vbench/src/vbench/vbench/latency_analyzer.cpp new file mode 100644 index 00000000000..34b99c76b30 --- /dev/null +++ b/vbench/src/vbench/vbench/latency_analyzer.cpp @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "latency_analyzer.h" +#include <math.h> + +namespace vbench { + +double +LatencyAnalyzer::getN(size_t n) const +{ + size_t acc = 0; + for (size_t i = 0; i < _hist.size(); ++i) { + acc += _hist[i]; + if (acc > n) { + return (((double)i) / 1000.0); + } + } + return _max; +} + +double +LatencyAnalyzer::getPercentile(double per) const +{ + double target = std::max((((double)(_cnt - 1)) * (per / 100.0)), 0.0); + size_t before = (size_t)floor(target); + size_t after = (size_t)ceil(target); + double factor = ceil(target) - target; + return (factor * getN(before) + (1.0 - factor) * getN(after)); +} + +string +LatencyAnalyzer::Stats::toString() const +{ + string str = "Latency {\n"; + str += strfmt(" min: %g\n", min); + str += strfmt(" avg: %g\n", avg); + str += strfmt(" max: %g\n", max); + str += strfmt(" 50%%: %g\n", per50); + str += strfmt(" 95%%: %g\n", per95); + str += strfmt(" 99%%: %g\n", per99); + str += "}\n"; + return str; +} + +LatencyAnalyzer::LatencyAnalyzer(Handler<Request> &next) + : _next(next), + _cnt(0), + _min(0.0), + _max(0.0), + _total(0.0), + _hist(10000, 0) +{ +} + +void +LatencyAnalyzer::handle(Request::UP request) +{ + if (request->status() == Request::STATUS_OK) { + addLatency(request->latency()); + } + _next.handle(std::move(request)); +} + +void +LatencyAnalyzer::report() +{ + fprintf(stdout, "%s\n", getStats().toString().c_str()); +} + +void +LatencyAnalyzer::addLatency(double latency) +{ + if (_cnt == 0 || latency < _min) { + _min = latency; + } + if (_cnt == 0 || latency > _max) { + _max = latency; + } + ++_cnt; + _total += latency; + size_t idx = (size_t)(latency * 1000.0 + 0.5); + if (idx < _hist.size()) { + ++_hist[idx]; + } +} + +LatencyAnalyzer::Stats +LatencyAnalyzer::getStats() const +{ + Stats stats; + stats.min = _min; + if (_cnt > 0) { + stats.avg = (_total / (double)_cnt); + } + stats.max = _max; + stats.per50 = getPercentile(50.0); + stats.per95 = getPercentile(95.0); + stats.per99 = getPercentile(99.0); + return stats; +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/latency_analyzer.h b/vbench/src/vbench/vbench/latency_analyzer.h new file mode 100644 index 00000000000..d69c85d0360 --- /dev/null +++ b/vbench/src/vbench/vbench/latency_analyzer.h @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> +#include <vbench/core/string.h> +#include "request.h" +#include "analyzer.h" + +namespace vbench { + +/** + * Component picking up the latency of successful requests and + * calculating relevant aggregated values. + **/ +class LatencyAnalyzer : public Analyzer +{ +private: + Handler<Request> &_next; + size_t _cnt; + double _min; + double _max; + double _total; + std::vector<size_t> _hist; + + double getN(size_t n) const; + double getPercentile(double per) const; + +public: + struct Stats { + double min; + double avg; + double max; + double per50; + double per95; + double per99; + Stats() : min(0), avg(0), max(0), per50(0), per95(0), per99(0) {} + string toString() const; + }; + LatencyAnalyzer(Handler<Request> &next); + virtual void handle(Request::UP request); + virtual void report(); + void addLatency(double latency); + Stats getStats() const; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/native_factory.cpp b/vbench/src/vbench/vbench/native_factory.cpp new file mode 100644 index 00000000000..95b0b7e243f --- /dev/null +++ b/vbench/src/vbench/vbench/native_factory.cpp @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "native_factory.h" +#include "request_generator.h" +#include "server_tagger.h" +#include "qps_tagger.h" +#include "latency_analyzer.h" +#include "qps_analyzer.h" +#include "request_dumper.h" +#include "ignore_before.h" + +namespace vbench { + +Generator::UP +NativeFactory::createGenerator(const vespalib::slime::Inspector &spec, + Handler<Request> &next) +{ + std::string type = spec["type"].asString().make_string(); + if (type == "RequestGenerator") { + return Generator::UP(new RequestGenerator(spec["file"].asString().make_string(), next)); + } + return Generator::UP(); +} + +Tagger::UP +NativeFactory::createTagger(const vespalib::slime::Inspector &spec, + Handler<Request> &next) +{ + std::string type = spec["type"].asString().make_string(); + if (type == "ServerTagger") { + return Tagger::UP(new ServerTagger(ServerSpec(spec["host"].asString().make_string(), + spec["port"].asLong()), next)); + } + if (type == "QpsTagger") { + return Tagger::UP(new QpsTagger(spec["qps"].asLong(), next)); + } + return Tagger::UP(); +} + +Analyzer::UP +NativeFactory::createAnalyzer(const vespalib::slime::Inspector &spec, + Handler<Request> &next) +{ + std::string type = spec["type"].asString().make_string(); + if (type == "LatencyAnalyzer") { + return Analyzer::UP(new LatencyAnalyzer(next)); + } + if (type == "QpsAnalyzer") { + return Analyzer::UP(new QpsAnalyzer(next)); + } + if (type == "RequestDumper") { + return Analyzer::UP(new RequestDumper(next)); + } + if (type == "IgnoreBefore") { + return Analyzer::UP(new IgnoreBefore(spec["time"].asDouble(), next)); + } + return Analyzer::UP(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/native_factory.h b/vbench/src/vbench/vbench/native_factory.h new file mode 100644 index 00000000000..9d5226161d2 --- /dev/null +++ b/vbench/src/vbench/vbench/native_factory.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include "generator.h" +#include "tagger.h" +#include "analyzer.h" + +namespace vbench { + +struct NativeFactory { + Generator::UP createGenerator(const vespalib::slime::Inspector &spec, + Handler<Request> &next); + Tagger::UP createTagger(const vespalib::slime::Inspector &spec, + Handler<Request> &next); + Analyzer::UP createAnalyzer(const vespalib::slime::Inspector &spec, + Handler<Request> &next); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/qps_analyzer.cpp b/vbench/src/vbench/vbench/qps_analyzer.cpp new file mode 100644 index 00000000000..4989edae057 --- /dev/null +++ b/vbench/src/vbench/vbench/qps_analyzer.cpp @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "qps_analyzer.h" +#include <math.h> + +namespace vbench { + +QpsAnalyzer::QpsAnalyzer(Handler<Request> &next) + : _next(next), + _qps(0), + _samples(0), + _begin(0), + _cnt(0) +{ +} + +void +QpsAnalyzer::handle(Request::UP request) +{ + if (request->status() == Request::STATUS_OK) { + addEndTime(request->endTime()); + } + _next.handle(std::move(request)); +} + +void +QpsAnalyzer::report() +{ + fprintf(stdout, "end qps: %g\n", _qps); +} + +void +QpsAnalyzer::addEndTime(double end) +{ + ++_cnt; + if (end < _begin) { + _begin = end; + } + if ((end - _begin) > 5.0) { + double newQps = ((double)_cnt) / (end - _begin); + double factor = (_samples == 0) ? 1.0 : 0.75; + _qps = (((1 - factor) * _qps) + (factor * newQps)); + ++_samples; + _begin = end; + _cnt = 0; + fprintf(stderr, "qps: %g\n", _qps); + } +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/qps_analyzer.h b/vbench/src/vbench/vbench/qps_analyzer.h new file mode 100644 index 00000000000..bdc8b4cfe57 --- /dev/null +++ b/vbench/src/vbench/vbench/qps_analyzer.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> +#include "request.h" +#include "analyzer.h" + +namespace vbench { + +/** + * Component calculating the rate of successful requests based on end + * time. + **/ +class QpsAnalyzer : public Analyzer +{ +private: + Handler<Request> &_next; + double _qps; + size_t _samples; + double _begin; + size_t _cnt; + +public: + QpsAnalyzer(Handler<Request> &next); + virtual void handle(Request::UP request); + virtual void report(); + void addEndTime(double end); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/qps_tagger.cpp b/vbench/src/vbench/vbench/qps_tagger.cpp new file mode 100644 index 00000000000..d4a00bcc3bf --- /dev/null +++ b/vbench/src/vbench/vbench/qps_tagger.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "qps_tagger.h" + +namespace vbench { + +QpsTagger::QpsTagger(double qps, Handler<Request> &next) + : _invQps(1.0/qps), + _count(0), + _next(next) +{ +} + +void +QpsTagger::handle(Request::UP request) +{ + request->scheduledTime(((double)(_count++)) * _invQps); + _next.handle(std::move(request)); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/qps_tagger.h b/vbench/src/vbench/vbench/qps_tagger.h new file mode 100644 index 00000000000..fcd80b364bf --- /dev/null +++ b/vbench/src/vbench/vbench/qps_tagger.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> + +#include "request.h" +#include "tagger.h" + +namespace vbench { + +/** + * Sets the start time of requests based on a given qps. + **/ +class QpsTagger : public Tagger +{ +private: + double _invQps; + size_t _count; + Handler<Request> &_next; + +public: + QpsTagger(double qps, Handler<Request> &next); + virtual void handle(Request::UP request); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/request.cpp b/vbench/src/vbench/vbench/request.cpp new file mode 100644 index 00000000000..fa5e253922f --- /dev/null +++ b/vbench/src/vbench/vbench/request.cpp @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request.h" + +namespace vbench { + +Request::Request() + : _url(), + _server(), + _scheduledTime(), + _status(STATUS_OK), + _startTime(), + _endTime(), + _size(0) +{ +} + +void +Request::handleHeader(const string &name, const string &value) +{ + _headers.handleHeader(name, value); +} + +void +Request::handleContent(const Memory &data) +{ + _size += data.size; +} + +void +Request::handleFailure(const string &) +{ + _status = STATUS_FAILED; +} + +string +Request::toString() const +{ + string str; + str += "Request {\n"; + str += strfmt(" url: %s\n", _url.c_str()); + str += strfmt(" server.host: %s\n", _server.host.c_str()); + str += strfmt(" server.port: %d\n", _server.port); + str += strfmt(" scheduledTime: %g\n", _scheduledTime); + str += strfmt(" status: %s\n", + ((_status == STATUS_OK) ? "OK" + : (_status == STATUS_DROPPED) ? "DROPPED" + : (_status == STATUS_FAILED) ? "FAILED" + : "UNKNOWN")); + str += strfmt(" startTime: %g\n", _startTime); + str += strfmt(" endTime: %g\n", _endTime); + str += strfmt(" latency: %g\n", latency()); + str += strfmt(" size: %zu\n", _size); + str += _headers.toString(); + str += "}\n"; + return str; +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/request.h b/vbench/src/vbench/vbench/request.h new file mode 100644 index 00000000000..7da731d746f --- /dev/null +++ b/vbench/src/vbench/vbench/request.h @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/string.h> +#include <vbench/http/benchmark_headers.h> +#include <vbench/http/server_spec.h> +#include <vbench/http/http_result_handler.h> +#include <memory> + +namespace vbench { + +/** + * Encapsulates all known information about a single query. This + * object will flow through the system. + **/ +class Request : public HttpResultHandler +{ +public: + typedef std::unique_ptr<Request> UP; + + enum Status { + STATUS_OK = 0, + STATUS_DROPPED = 1, + STATUS_FAILED = 2 + }; + +private: + // parameters to request scheduler + string _url; + ServerSpec _server; + double _scheduledTime; + + // results filled in by request scheduler + Status _status; + double _startTime; + double _endTime; + size_t _size; + + // benchmark headers from QRS + BenchmarkHeaders _headers; + +public: + Request(); + + //--- parameters + + const string &url() const { return _url; } + Request &url(const string &value) { _url = value; return *this; } + + const ServerSpec &server() const { return _server; } + Request &server(const ServerSpec &value) { _server = value; return *this; } + + double scheduledTime() const { return _scheduledTime; } + Request &scheduledTime(double value) { _scheduledTime = value; return *this; } + + //--- results + + Status status() const { return _status; } + Request &status(Status value) { _status = value; return *this; } + + double startTime() const { return _startTime; } + Request &startTime(double value) { _startTime = value; return *this; } + + double endTime() const { return _endTime; } + Request &endTime(double value) { _endTime = value; return *this; } + + double latency() const { return (_endTime - _startTime); } + + virtual void handleHeader(const string &name, const string &value); + virtual void handleContent(const Memory &data); + virtual void handleFailure(const string &reason); + + const BenchmarkHeaders &headers() const { return _headers; } + + string toString() const; +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/request_dumper.cpp b/vbench/src/vbench/vbench/request_dumper.cpp new file mode 100644 index 00000000000..04a29522f87 --- /dev/null +++ b/vbench/src/vbench/vbench/request_dumper.cpp @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request_dumper.h" +#include <vbench/core/string.h> + +namespace vbench { + +RequestDumper::RequestDumper(Handler<Request> &next) + : _next(next) +{ +} + +void +RequestDumper::handle(Request::UP request) +{ + string dump = request->toString(); + fprintf(stderr, "%s\n", dump.c_str()); +} + +void +RequestDumper::report() +{ +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/request_dumper.h b/vbench/src/vbench/vbench/request_dumper.h new file mode 100644 index 00000000000..7005f71fbdf --- /dev/null +++ b/vbench/src/vbench/vbench/request_dumper.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/core/handler.h> + +#include "request.h" +#include "analyzer.h" + +namespace vbench { + +/** + * Dumps the textual representation of a request to standard + * output. Intended for debugging purposes. + **/ +class RequestDumper : public Analyzer +{ +private: + Handler<Request> &_next; + +public: + RequestDumper(Handler<Request> &_next); + virtual void handle(Request::UP request); + virtual void report(); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/request_generator.cpp b/vbench/src/vbench/vbench/request_generator.cpp new file mode 100644 index 00000000000..69985f1d6f6 --- /dev/null +++ b/vbench/src/vbench/vbench/request_generator.cpp @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request_generator.h" + +namespace vbench { + +RequestGenerator::RequestGenerator(const string &inputFile, + Handler<Request> &next) + : _input(inputFile), + _next(next), + _aborted(false) +{ +} + +void +RequestGenerator::abort() +{ + _aborted = true; +} + +void +RequestGenerator::run() +{ + string line; + while (!_aborted && _input.readLine(line)) { + Request::UP request(new Request()); + request->url(line); + _next.handle(std::move(request)); + } +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/request_generator.h b/vbench/src/vbench/vbench/request_generator.h new file mode 100644 index 00000000000..5559a8ffcf1 --- /dev/null +++ b/vbench/src/vbench/vbench/request_generator.h @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vespa/vespalib/util/runnable.h> +#include <vbench/core/input_file_reader.h> +#include <vbench/core/taintable.h> +#include <vbench/core/handler.h> + +#include "request.h" +#include "generator.h" + +namespace vbench { + +/** + * Reads lines from an input file and generates requests that are + * passed to a request handler. + **/ +class RequestGenerator : public Generator +{ +private: + InputFileReader _input; + Handler<Request> &_next; + bool _aborted; + +public: + RequestGenerator(const string &inputFile, + Handler<Request> &next); + void abort(); + virtual void run(); + virtual const Taint &tainted() const { return _input.tainted(); } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/request_scheduler.cpp b/vbench/src/vbench/vbench/request_scheduler.cpp new file mode 100644 index 00000000000..de46e7b7942 --- /dev/null +++ b/vbench/src/vbench/vbench/request_scheduler.cpp @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request_scheduler.h" + +#include <vbench/core/timer.h> + +namespace vbench { + +void +RequestScheduler::run() +{ + double sleepTime; + std::vector<Request::UP> list; + vespalib::Thread &thread = vespalib::Thread::currentThread(); + while (_queue.extract(_timer.sample(), list, sleepTime)) { + for (size_t i = 0; i < list.size(); ++i) { + Request::UP request = Request::UP(list[i].release()); + _dispatcher.handle(std::move(request)); + } + list.clear(); + thread.slumber(sleepTime); + } +} + +RequestScheduler::RequestScheduler(Handler<Request> &next, size_t numWorkers) + : _timer(), + _proxy(next), + _queue(10.0, 0.020), + _droppedTagger(_proxy), + _dispatcher(_droppedTagger), + _thread(*this), + _connectionPool(_timer), + _workers() +{ + for (size_t i = 0; i < numWorkers; ++i) { + _workers.push_back(std::unique_ptr<Worker>(new Worker(_dispatcher, _proxy, _connectionPool, _timer))); + } + _dispatcher.waitForThreads(numWorkers, 256); +} + +void +RequestScheduler::abort() +{ + _queue.close(); + _queue.discard(); + _thread.stop(); +} + +void +RequestScheduler::handle(Request::UP request) +{ + _queue.insert(std::move(request), request->scheduledTime()); +} + +void +RequestScheduler::start() +{ + _timer.reset(); + _thread.start(); +} + +RequestScheduler & +RequestScheduler::stop() +{ + _queue.close(); + _thread.stop(); + return *this; +} + +void +RequestScheduler::join() +{ + _thread.join(); + _dispatcher.close(); + for (size_t i = 0; i < _workers.size(); ++i) { + _workers[i]->join(); + } + _proxy.join(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/request_scheduler.h b/vbench/src/vbench/vbench/request_scheduler.h new file mode 100644 index 00000000000..09c7b7f0f34 --- /dev/null +++ b/vbench/src/vbench/vbench/request_scheduler.h @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> + +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/util/runnable.h> +#include <vespa/vespalib/util/active.h> + +#include <vbench/core/handler.h> +#include <vbench/core/time_queue.h> +#include <vbench/core/dispatcher.h> +#include <vbench/core/handler_thread.h> + +#include "request.h" +#include "worker.h" +#include "dropped_tagger.h" + +namespace vbench { + +/** + * Component responsible for dispatching requests to workers at the + * appropriate time based on what start time the requests are tagged + * with. + **/ +class RequestScheduler : public Handler<Request>, + public vespalib::Runnable, + public vespalib::Active +{ +private: + Timer _timer; + HandlerThread<Request> _proxy; + TimeQueue<Request> _queue; + DroppedTagger _droppedTagger; + Dispatcher<Request> _dispatcher; + vespalib::Thread _thread; + HttpConnectionPool _connectionPool; + std::vector<Worker::UP> _workers; + + virtual void run(); +public: + typedef std::unique_ptr<RequestScheduler> UP; + RequestScheduler(Handler<Request> &next, size_t numWorkers); + void abort(); + virtual void handle(Request::UP request); + virtual void start(); + virtual RequestScheduler &stop(); + virtual void join(); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/request_sink.cpp b/vbench/src/vbench/vbench/request_sink.cpp new file mode 100644 index 00000000000..38468d39e94 --- /dev/null +++ b/vbench/src/vbench/vbench/request_sink.cpp @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "request_sink.h" +#include <vbench/core/string.h> + +namespace vbench { + +RequestSink::RequestSink() +{ +} + +void +RequestSink::handle(Request::UP request) +{ + request.reset(); +} + +void +RequestSink::report() +{ +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/request_sink.h b/vbench/src/vbench/vbench/request_sink.h new file mode 100644 index 00000000000..27be9e7942e --- /dev/null +++ b/vbench/src/vbench/vbench/request_sink.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include "analyzer.h" + +namespace vbench { + +/** + * Deletes incoming request object. This handler is used to terminate + * a chain of handlers. + **/ +class RequestSink : public Analyzer +{ +public: + RequestSink(); + virtual void handle(Request::UP request); + virtual void report(); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/server_tagger.cpp b/vbench/src/vbench/vbench/server_tagger.cpp new file mode 100644 index 00000000000..b42ee3896ac --- /dev/null +++ b/vbench/src/vbench/vbench/server_tagger.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "server_tagger.h" + +namespace vbench { + +ServerTagger::ServerTagger(const ServerSpec &server, + Handler<Request> &next) + : _server(server), + _next(next) +{ +} + +void +ServerTagger::handle(Request::UP request) +{ + request->server(_server); + _next.handle(std::move(request)); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/server_tagger.h b/vbench/src/vbench/vbench/server_tagger.h new file mode 100644 index 00000000000..c104b15bd53 --- /dev/null +++ b/vbench/src/vbench/vbench/server_tagger.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <vbench/http/server_spec.h> +#include <vbench/core/handler.h> + +#include "request.h" +#include "tagger.h" + +namespace vbench { + +/** + * Sets the target server for requests. + **/ +class ServerTagger : public Tagger +{ +private: + ServerSpec _server; + Handler<Request> &_next; + +public: + ServerTagger(const ServerSpec &server, + Handler<Request> &next); + virtual void handle(Request::UP request); +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/tagger.cpp b/vbench/src/vbench/vbench/tagger.cpp new file mode 100644 index 00000000000..780cf4c3aa7 --- /dev/null +++ b/vbench/src/vbench/vbench/tagger.cpp @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "tagger.h" + +namespace vbench { + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/tagger.h b/vbench/src/vbench/vbench/tagger.h new file mode 100644 index 00000000000..c19319e39e2 --- /dev/null +++ b/vbench/src/vbench/vbench/tagger.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> +#include <vbench/core/handler.h> + +#include "request.h" + +namespace vbench { + +struct Tagger : public Handler<Request> +{ + typedef std::unique_ptr<Tagger> UP; + virtual ~Tagger() {} +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp new file mode 100644 index 00000000000..56b949d4caf --- /dev/null +++ b/vbench/src/vbench/vbench/vbench.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "vbench.h" +#include <vespa/vespalib/util/thread.h> + +namespace vbench { + +VBench::VBench(const vespalib::Slime &cfg) + : _factory(), + _analyzers(), + _scheduler(), + _inputs(), + _taint() +{ + _analyzers.push_back(Analyzer::UP(new RequestSink())); + vespalib::slime::Inspector &analyzers = cfg.get()["analyze"]; + for (size_t i = analyzers.children(); i-- > 0; ) { + Analyzer::UP obj = _factory.createAnalyzer(analyzers[i], *_analyzers.back()); + if (obj.get() != 0) { + _analyzers.push_back(Analyzer::UP(obj.release())); + } + } + _scheduler.reset(new RequestScheduler(*_analyzers.back(), + cfg.get()["http_threads"].asLong())); + vespalib::slime::Inspector &inputs = cfg.get()["inputs"]; + for (size_t i = inputs.children(); i-- > 0; ) { + vespalib::slime::Inspector &input = inputs[i]; + vespalib::slime::Inspector &taggers = input["prepare"]; + vespalib::slime::Inspector &generator = input["source"]; + InputChain::UP inputChain(new InputChain()); + for (size_t j = taggers.children(); j-- > 0; ) { + Handler<Request> &next = (j == (taggers.children() - 1)) + ? ((Handler<Request>&)*_scheduler) + : ((Handler<Request>&)*inputChain->taggers.back()); + Tagger::UP obj = _factory.createTagger(taggers[j], next); + if (obj.get() != 0) { + inputChain->taggers.push_back(Tagger::UP(obj.release())); + } + } + inputChain->generator = _factory.createGenerator(generator, *inputChain->taggers.back()); + if (inputChain->generator.get() != 0) { + inputChain->thread.reset(new vespalib::Thread(*inputChain->generator)); + _inputs.push_back(std::move(inputChain)); + } + } +} + +void +VBench::abort() +{ + fprintf(stderr, "aborting...\n"); + for (size_t i = 0; i < _inputs.size(); ++i) { + _inputs[i]->generator->abort(); + } + _scheduler->abort(); +} + +void +VBench::run() +{ + _scheduler->start(); + for (size_t i = 0; i < _inputs.size(); ++i) { + _inputs[i]->thread->start(); + } + for (size_t i = 0; i < _inputs.size(); ++i) { + _inputs[i]->thread->join(); + } + _scheduler->stop().join(); + for (size_t i = 0; i < _inputs.size(); ++i) { + if (_inputs[i]->generator->tainted()) { + _taint = _inputs[i]->generator->tainted(); + } + } + for (size_t i = 0; i < _analyzers.size(); ++i) { + _analyzers[i]->report(); + } +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/vbench.h b/vbench/src/vbench/vbench/vbench.h new file mode 100644 index 00000000000..f17e5d4c6b1 --- /dev/null +++ b/vbench/src/vbench/vbench/vbench.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> + +#include <vespa/vespalib/util/runnable.h> +#include <vbench/core/taintable.h> +#include <vespa/vespalib/data/slime/slime.h> + +#include "analyzer.h" +#include "generator.h" +#include "latency_analyzer.h" +#include "native_factory.h" +#include "qps_analyzer.h" +#include "qps_tagger.h" +#include "request_generator.h" +#include "request_scheduler.h" +#include "request_sink.h" +#include "server_tagger.h" +#include "tagger.h" + +namespace vbench { + +class VBench : public vespalib::Runnable, + public Taintable +{ +private: + struct InputChain { + typedef std::unique_ptr<InputChain> UP; + std::vector<Tagger::UP> taggers; + Generator::UP generator; + std::unique_ptr<vespalib::Thread> thread; + }; + NativeFactory _factory; + std::vector<Analyzer::UP> _analyzers; + RequestScheduler::UP _scheduler; + std::vector<InputChain::UP> _inputs; + Taint _taint; + +public: + VBench(const vespalib::Slime &cfg); + void abort(); + virtual void run(); + virtual const Taint &tainted() const { return _taint; } +}; + +} // namespace vbench + diff --git a/vbench/src/vbench/vbench/worker.cpp b/vbench/src/vbench/vbench/worker.cpp new file mode 100644 index 00000000000..467c47e2704 --- /dev/null +++ b/vbench/src/vbench/vbench/worker.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include "worker.h" +#include <vbench/http/http_client.h> + +namespace vbench { + +void +Worker::run() +{ + for (;;) { + Request::UP request = _provider.provide(); + if (request.get() == 0) { + break; + } + request->startTime(_timer.sample()); + HttpClient::fetch(_pool, request->server(), request->url(), *request); + request->endTime(_timer.sample()); + _next.handle(std::move(request)); + } +} + +Worker::Worker(Provider<Request> &provider, Handler<Request> &next, + HttpConnectionPool &pool, Timer &timer) + : _thread(*this), + _provider(provider), + _next(next), + _pool(pool), + _timer(timer) +{ + _thread.start(); +} + +} // namespace vbench diff --git a/vbench/src/vbench/vbench/worker.h b/vbench/src/vbench/vbench/worker.h new file mode 100644 index 00000000000..14e79312ce2 --- /dev/null +++ b/vbench/src/vbench/vbench/worker.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#pragma once + +#include <memory> + +#include <vespa/vespalib/util/runnable.h> +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/util/joinable.h> + +#include <vbench/core/provider.h> +#include <vbench/core/handler.h> +#include <vbench/http/http_connection_pool.h> + +#include "request.h" + +namespace vbench { + +/** + * Obtains requests from a request provider, performs the requests and + * passes the requests along to a request handler. Runs its own + * internal thread that will stop when the request provider starts + * handing out empty requests. + **/ +class Worker : public vespalib::Runnable, + public vespalib::Joinable +{ +private: + vespalib::Thread _thread; + Provider<Request> &_provider; + Handler<Request> &_next; + HttpConnectionPool &_pool; + Timer &_timer; + + virtual void run(); +public: + typedef std::unique_ptr<Worker> UP; + Worker(Provider<Request> &provider, Handler<Request> &next, + HttpConnectionPool &pool, Timer &timer); + virtual void join() { _thread.join(); } +}; + +} // namespace vbench + diff --git a/vbench/testrun/.gitignore b/vbench/testrun/.gitignore new file mode 100644 index 00000000000..81df10612ec --- /dev/null +++ b/vbench/testrun/.gitignore @@ -0,0 +1,8 @@ +/test-report.html +/test-report.html.* +/test.*.*.desc +/test.*.*.file.* +/test.*.*.files.html +/test.*.*.log +/tmp.* +/test.*.*.result |