diff options
author | HÃ¥vard Pettersen <3535158+havardpe@users.noreply.github.com> | 2020-12-09 14:12:26 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-12-09 14:12:26 +0100 |
commit | 82c41ca9a1ebe439e4498734a974dc362764ba89 (patch) | |
tree | 345615d1071ef36eac85764c6365d58def81459a /eval | |
parent | 581f4e28e3b72707c1f45a294c41f6101966e306 (diff) | |
parent | 6931b0f071472d852dc216bf014808f850db58b5 (diff) |
Merge pull request #15756 from vespa-engine/havardpe/ghost-mode
ghost support in instruction benchmark
Diffstat (limited to 'eval')
-rw-r--r-- | eval/src/tests/tensor/instruction_benchmark/.gitignore | 2 | ||||
-rw-r--r-- | eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp | 63 |
2 files changed, 57 insertions, 8 deletions
diff --git a/eval/src/tests/tensor/instruction_benchmark/.gitignore b/eval/src/tests/tensor/instruction_benchmark/.gitignore index dc5c408cf29..5b3eab59ff6 100644 --- a/eval/src/tests/tensor/instruction_benchmark/.gitignore +++ b/eval/src/tests/tensor/instruction_benchmark/.gitignore @@ -1 +1,3 @@ vespa-tensor-instructions-benchmark +/result.json +/ghost.json diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 3bddf69f53f..618715a885f 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -39,6 +39,10 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/io/mapped_file_input.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <optional> #include <algorithm> @@ -48,6 +52,8 @@ using namespace vespalib::tensor; using namespace vespalib::eval::instruction; using vespalib::make_string_short::fmt; +using vespalib::slime::JsonFormat; + using Instruction = InterpretedFunction::Instruction; using EvalSingle = InterpretedFunction::EvalSingle; @@ -256,6 +262,8 @@ Impl optimized_fast_value_impl(0, " Optimized FastValue", " Impl fast_value_impl(1, " FastValue", " FastV", FastValueBuilderFactory::get(), false); Impl simple_value_impl(2, " SimpleValue", " SimpleV", SimpleValueBuilderFactory::get(), false); vespalib::string short_header("--------"); +vespalib::string ghost_name(" loaded from ghost.json"); +vespalib::string ghost_short_name(" ghost"); constexpr double budget = 5.0; constexpr double best_limit = 0.95; // everything within 95% of best performance gets a star @@ -266,6 +274,10 @@ std::vector<CREF<Impl>> impl_list = {simple_value_impl, optimized_fast_value_impl, fast_value_impl}; +Slime ghost; // loaded from 'ghost.json' +bool has_ghost = false; +Slime prod_result; // saved to 'result.json' + //----------------------------------------------------------------------------- struct BenchmarkHeader { @@ -275,6 +287,9 @@ struct BenchmarkHeader { for (const Impl &impl: impl_list) { short_names[impl.order] = impl.short_name; } + if (has_ghost) { + short_names.push_back(ghost_short_name); + } } void print_header(const vespalib::string &desc) const { for (const auto &name: short_names) { @@ -300,12 +315,17 @@ struct BenchmarkResult { ~BenchmarkResult(); void sample(size_t order, double time) { relative_perf[order] = time; - if (order == 1) { - if (ref_time.has_value()) { - ref_time = std::min(ref_time.value(), time); - } else { - ref_time = time; + if (order == 0) { + prod_result.get().setDouble(desc, time); + if (has_ghost && (relative_perf.size() == impl_list.size())) { + double ghost_time = ghost.get()[desc].asDouble(); + size_t ghost_order = relative_perf.size(); + fprintf(stderr, " %s(%s): %10.3f us\n", ghost_name.c_str(), ghost_short_name.c_str(), ghost_time); + relative_perf.resize(ghost_order + 1); + return sample(ghost_order, ghost_time); } + } else if (order == 1) { + ref_time = time; } } void normalize() { @@ -333,6 +353,23 @@ std::vector<BenchmarkResult> benchmark_results; //----------------------------------------------------------------------------- +void load_ghost(const vespalib::string &file_name) { + MappedFileInput input(file_name); + has_ghost = JsonFormat::decode(input, ghost); +} + +void save_result(const vespalib::string &file_name) { + SmartBuffer output(4096); + JsonFormat::encode(prod_result, output, false); + Memory memory = output.obtain(); + File file(file_name); + file.open(File::CREATE | File::TRUNC); + file.write(memory.data, memory.size, 0); + file.close(); +} + +//----------------------------------------------------------------------------- + struct MyParam : LazyParams { Value::UP my_value; MyParam() : my_value() {} @@ -433,8 +470,8 @@ void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list } for (const auto &eval: list) { double time = eval->estimate_cost_us(loop_cnt[eval->impl.order], loop_cnt[1]); - result.sample(eval->impl.order, time); fprintf(stderr, " %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time); + result.sample(eval->impl.order, time); } result.normalize(); benchmark_results.push_back(result); @@ -674,9 +711,9 @@ void benchmark_encode_decode(const vespalib::string &desc, const TensorSpec &pro } double encode_us = encode_timer.min_time() * 1000.0 * 1000.0 / double(loop_cnt); double decode_us = decode_timer.min_time() * 1000.0 * 1000.0 / double(loop_cnt); - fprintf(stderr, " %s (%s) <encode>: %10.3f us\n", impl.name.c_str(), impl.short_name.c_str(), encode_us); - fprintf(stderr, " %s (%s) <decode>: %10.3f us\n", impl.name.c_str(), impl.short_name.c_str(), decode_us); + fprintf(stderr, " %s(%s): %10.3f us <encode>\n", impl.name.c_str(), impl.short_name.c_str(), encode_us); encode_result.sample(impl.order, encode_us); + fprintf(stderr, " %s(%s): %10.3f us <decode>\n", impl.name.c_str(), impl.short_name.c_str(), decode_us); decode_result.sample(impl.order, decode_us); } encode_result.normalize(); @@ -1077,16 +1114,26 @@ void print_summary() { } int main(int argc, char **argv) { + prod_result.setObject(); + load_ghost("ghost.json"); const std::string run_only_prod_option = "--limit-implementations"; + const std::string ghost_mode_option = "--ghost-mode"; if ((argc > 1) && (argv[1] == run_only_prod_option )) { impl_list.clear(); impl_list.push_back(optimized_fast_value_impl); impl_list.push_back(fast_value_impl); ++argv; --argc; + } else if ((argc > 1) && (argv[1] == ghost_mode_option )) { + impl_list.clear(); + impl_list.push_back(optimized_fast_value_impl); + has_ghost = true; + ++argv; + --argc; } ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); + save_result("result.json"); print_summary(); return result; } |