diff options
author | Håvard Pettersen <havardpe@oath.com> | 2017-10-03 13:11:24 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2017-10-03 13:11:24 +0000 |
commit | 38e1daf1c98882a1f75a345bcf5eca84c222fa89 (patch) | |
tree | 1215a6496d5539ae1d87670b539b371ec17e8a13 /eval/src/apps | |
parent | 81b9d23e0e85e73626ff6edea446741bb263b465 (diff) |
wire in spec-based tensor conformance testing
verify conformance spec generation
verify C++ default expression evaluation
bonus: verify binary format test spec generation
Diffstat (limited to 'eval/src/apps')
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.h | 6 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 170 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/test_spec.json | 44 |
3 files changed, 111 insertions, 109 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index d20d085f00c..0f74ce924b3 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -7,13 +7,13 @@ struct TestBuilder { using TensorSpec = vespalib::eval::TensorSpec; - // add test with undefined expected result - virtual void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) = 0; // add test with pre-defined expected result virtual void add(const vespalib::string &expression, const std::map<vespalib::string,TensorSpec> &inputs, const TensorSpec &expect) = 0; + // add test with undefined expected result + virtual void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) = 0; virtual ~TestBuilder() {} }; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 593e4439b0a..cfe9542ecda 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -25,23 +25,6 @@ using slime::JsonFormat; using tensor::DefaultTensorEngine; constexpr size_t CHUNK_SIZE = 16384; -constexpr bool not_compact = false; - -//----------------------------------------------------------------------------- - -size_t num_tests = 0; -std::map<vespalib::string,size_t> result_map; - -vespalib::string result_stats() { - vespalib::string stats; - for (const auto &entry: result_map) { - if (!stats.empty()) { - stats += ", "; - } - stats += make_string("%s: %zu", entry.first.c_str(), entry.second); - } - return stats; -} //----------------------------------------------------------------------------- @@ -85,6 +68,16 @@ public: } }; +void write_compact(const Slime &slime, Output &out) { + JsonFormat::encode(slime, out, true); + out.reserve(1).data[0] = '\n'; + out.commit(1); +} + +void write_readable(const Slime &slime, Output &out) { + JsonFormat::encode(slime, out, false); +} + //----------------------------------------------------------------------------- uint8_t unhex(char c) { @@ -193,93 +186,139 @@ std::vector<vespalib::string> extract_fields(const Inspector &object) { return std::move(extractor.result); }; -void dump_test(const Inspector &test) { - fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); - for (const auto &input: extract_fields(test["inputs"])) { - auto value = extract_value(test["inputs"][input]); - fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); - } -} - //----------------------------------------------------------------------------- class MyTestBuilder : public TestBuilder { private: Output &_out; - void build_test(Cursor &test, const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &input_map) + size_t _num_tests; + void make_test(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &input_map, + const TensorSpec *expect = nullptr) { + Slime slime; + Cursor &test = slime.setObject(); test.setString("expression", expression); Cursor &inputs = test.setObject("inputs"); for (const auto &input: input_map) { insert_value(inputs, input.first, input.second); } + if (expect != nullptr) { + insert_value(test.setObject("result"), "expect", *expect); + } else { + insert_value(test.setObject("result"), "expect", + eval_expr(slime.get(), SimpleTensorEngine::ref())); + } + write_compact(slime, _out); + ++_num_tests; } public: - MyTestBuilder(Output &out) : _out(out) {} + MyTestBuilder(Output &out) : _out(out), _num_tests(0) {} void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) override + const std::map<vespalib::string,TensorSpec> &inputs, + const TensorSpec &expect) override { - Slime slime; - build_test(slime.setObject(), expression, inputs); - insert_value(slime.get().setObject("result"), "expect", - eval_expr(slime.get(), SimpleTensorEngine::ref())); - JsonFormat::encode(slime, _out, not_compact); - ++num_tests; + make_test(expression, inputs, &expect); } void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs, - const TensorSpec &expect) override + const std::map<vespalib::string,TensorSpec> &inputs) override { + make_test(expression, inputs); + } + void make_summary() { Slime slime; - build_test(slime.setObject(), expression, inputs); - insert_value(slime.get().setObject("result"), "expect", expect); - if (!EXPECT_EQUAL(eval_expr(slime.get(), SimpleTensorEngine::ref()), expect)) { - dump_test(slime.get()); - } - JsonFormat::encode(slime, _out, not_compact); - ++num_tests; + Cursor &summary = slime.setObject(); + summary.setLong("num_tests", _num_tests); + write_compact(slime, _out); } }; void generate(Output &out) { MyTestBuilder my_test_builder(out); Generator::generate(my_test_builder); + my_test_builder.make_summary(); } //----------------------------------------------------------------------------- -void evaluate(Input &in, Output &out) { +void for_each_test(Input &in, + const std::function<void(Slime&)> &handle_test, + const std::function<void(Slime&)> &handle_summary) +{ + size_t num_tests = 0; + bool got_summary = false; while (in.obtain().size > 0) { Slime slime; if (JsonFormat::decode(in, slime)) { - ++num_tests; - insert_value(slime.get()["result"], "prod_cpp", - eval_expr(slime.get(), DefaultTensorEngine::ref())); - JsonFormat::encode(slime, out, not_compact); + bool is_test = slime.get()["expression"].valid(); + bool is_summary = slime.get()["num_tests"].valid(); + ASSERT_TRUE(is_test != is_summary); + if (is_test) { + ++num_tests; + ASSERT_TRUE(!got_summary); + handle_test(slime); + } else { + got_summary = true; + ASSERT_EQUAL(slime.get()["num_tests"].asLong(), int64_t(num_tests)); + handle_summary(slime); + } + } else { + ASSERT_EQUAL(in.obtain().size, 0u); } } + ASSERT_TRUE(got_summary); } //----------------------------------------------------------------------------- -void verify(Input &in) { - while (in.obtain().size > 0) { - Slime slime; - if (JsonFormat::decode(in, slime)) { - ++num_tests; - TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref()); - for (const auto &result: extract_fields(slime.get()["result"])) { - ++result_map[result]; - TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); - if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) { - dump_test(slime.get()); - } - } - } +void evaluate(Input &in, Output &out) { + auto handle_test = [&out](Slime &slime) + { + insert_value(slime.get()["result"], "prod_cpp", + eval_expr(slime.get(), DefaultTensorEngine::ref())); + write_compact(slime, out); + }; + auto handle_summary = [&out](Slime &slime) + { + write_compact(slime, out); + }; + for_each_test(in, handle_test, handle_summary); +} + +//----------------------------------------------------------------------------- + +void dump_test(const Inspector &test) { + fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); + for (const auto &input: extract_fields(test["inputs"])) { + auto value = extract_value(test["inputs"][input]); + fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); } } +void verify(Input &in, Output &out) { + std::map<vespalib::string,size_t> result_map; + auto handle_test = [&out,&result_map](Slime &slime) + { + TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref()); + for (const auto &result: extract_fields(slime.get()["result"])) { + ++result_map[result]; + TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); + if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) { + dump_test(slime.get()); + } + } + }; + auto handle_summary = [&out,&result_map](Slime &slime) + { + Cursor &stats = slime.get().setObject("stats"); + for (const auto &entry: result_map) { + stats.setLong(entry.first, entry.second); + } + write_readable(slime, out); + }; + for_each_test(in, handle_test, handle_summary); +} + //----------------------------------------------------------------------------- int usage(const char *self) { @@ -304,13 +343,10 @@ int main(int argc, char **argv) { TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str()); if (mode == "generate") { generate(std_out); - fprintf(stderr, "generated %zu test cases\n", num_tests); } else if (mode == "evaluate") { evaluate(std_in, std_out); - fprintf(stderr, "evaluated %zu test cases\n", num_tests); } else if (mode == "verify") { - verify(std_in); - fprintf(stderr, "verified %zu test cases (%s)\n", num_tests, result_stats().c_str()); + verify(std_in, std_out); } else { TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str()); } diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json index a7c906cfb85..c66931c2df8 100644 --- a/eval/src/apps/tensor_conformance/test_spec.json +++ b/eval/src/apps/tensor_conformance/test_spec.json @@ -1,39 +1,5 @@ -{ - "expression": "a+a", - "inputs": { - "a": "0x02004000000000000000" - }, - "result": { - "expect": "0x02004010000000000000" - } -} -{ - "expression": "a*b", - "inputs": { - "a": "0x02004000000000000000", - "b": "0x02004008000000000000" - }, - "result": { - "expect": "0x02004018000000000000" - } -} -{ - "expression": "(a+b)*(a-b)", - "inputs": { - "a": "0x02004014000000000000", - "b": "0x02004000000000000000" - }, - "result": { - "expect": "0x02004035000000000000" - } -} -{ - "expression": "(a-b)/(a+b)", - "inputs": { - "a": "0x02004014000000000000", - "b": "0x02004000000000000000" - }, - "result": { - "expect": "0x02003FDB6DB6DB6DB6DB" - } -} +{"expression":"a+a","inputs":{"a":"0x02004000000000000000"},"result":{"expect":"0x02004010000000000000"}} +{"expression":"a*b","inputs":{"a":"0x02004000000000000000","b":"0x02004008000000000000"},"result":{"expect":"0x02004018000000000000"}} +{"expression":"(a+b)*(a-b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02004035000000000000"}} +{"expression":"(a-b)/(a+b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02003FDB6DB6DB6DB6DB"}} +{"num_tests":4} |