diff options
5 files changed, 135 insertions, 109 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index d20d085f00c..0f74ce924b3 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -7,13 +7,13 @@ struct TestBuilder { using TensorSpec = vespalib::eval::TensorSpec; - // add test with undefined expected result - virtual void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) = 0; // add test with pre-defined expected result virtual void add(const vespalib::string &expression, const std::map<vespalib::string,TensorSpec> &inputs, const TensorSpec &expect) = 0; + // add test with undefined expected result + virtual void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) = 0; virtual ~TestBuilder() {} }; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 593e4439b0a..cfe9542ecda 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -25,23 +25,6 @@ using slime::JsonFormat; using tensor::DefaultTensorEngine; constexpr size_t CHUNK_SIZE = 16384; -constexpr bool not_compact = false; - -//----------------------------------------------------------------------------- - -size_t num_tests = 0; -std::map<vespalib::string,size_t> result_map; - -vespalib::string result_stats() { - vespalib::string stats; - for (const auto &entry: result_map) { - if (!stats.empty()) { - stats += ", "; - } - stats += make_string("%s: %zu", entry.first.c_str(), entry.second); - } - return stats; -} //----------------------------------------------------------------------------- @@ -85,6 +68,16 @@ public: } }; +void write_compact(const Slime &slime, Output &out) { + JsonFormat::encode(slime, out, true); + out.reserve(1).data[0] = '\n'; + out.commit(1); +} + +void write_readable(const Slime &slime, Output &out) { + JsonFormat::encode(slime, out, false); +} + //----------------------------------------------------------------------------- uint8_t unhex(char c) { @@ -193,93 +186,139 @@ std::vector<vespalib::string> extract_fields(const Inspector &object) { return std::move(extractor.result); }; -void dump_test(const Inspector &test) { - fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); - for (const auto &input: extract_fields(test["inputs"])) { - auto value = extract_value(test["inputs"][input]); - fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); - } -} - //----------------------------------------------------------------------------- class MyTestBuilder : public TestBuilder { private: Output &_out; - void build_test(Cursor &test, const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &input_map) + size_t _num_tests; + void make_test(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &input_map, + const TensorSpec *expect = nullptr) { + Slime slime; + Cursor &test = slime.setObject(); test.setString("expression", expression); Cursor &inputs = test.setObject("inputs"); for (const auto &input: input_map) { insert_value(inputs, input.first, input.second); } + if (expect != nullptr) { + insert_value(test.setObject("result"), "expect", *expect); + } else { + insert_value(test.setObject("result"), "expect", + eval_expr(slime.get(), SimpleTensorEngine::ref())); + } + write_compact(slime, _out); + ++_num_tests; } public: - MyTestBuilder(Output &out) : _out(out) {} + MyTestBuilder(Output &out) : _out(out), _num_tests(0) {} void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) override + const std::map<vespalib::string,TensorSpec> &inputs, + const TensorSpec &expect) override { - Slime slime; - build_test(slime.setObject(), expression, inputs); - insert_value(slime.get().setObject("result"), "expect", - eval_expr(slime.get(), SimpleTensorEngine::ref())); - JsonFormat::encode(slime, _out, not_compact); - ++num_tests; + make_test(expression, inputs, &expect); } void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs, - const TensorSpec &expect) override + const std::map<vespalib::string,TensorSpec> &inputs) override { + make_test(expression, inputs); + } + void make_summary() { Slime slime; - build_test(slime.setObject(), expression, inputs); - insert_value(slime.get().setObject("result"), "expect", expect); - if (!EXPECT_EQUAL(eval_expr(slime.get(), SimpleTensorEngine::ref()), expect)) { - dump_test(slime.get()); - } - JsonFormat::encode(slime, _out, not_compact); - ++num_tests; + Cursor &summary = slime.setObject(); + summary.setLong("num_tests", _num_tests); + write_compact(slime, _out); } }; void generate(Output &out) { MyTestBuilder my_test_builder(out); Generator::generate(my_test_builder); + my_test_builder.make_summary(); } //----------------------------------------------------------------------------- -void evaluate(Input &in, Output &out) { +void for_each_test(Input &in, + const std::function<void(Slime&)> &handle_test, + const std::function<void(Slime&)> &handle_summary) +{ + size_t num_tests = 0; + bool got_summary = false; while (in.obtain().size > 0) { Slime slime; if (JsonFormat::decode(in, slime)) { - ++num_tests; - insert_value(slime.get()["result"], "prod_cpp", - eval_expr(slime.get(), DefaultTensorEngine::ref())); - JsonFormat::encode(slime, out, not_compact); + bool is_test = slime.get()["expression"].valid(); + bool is_summary = slime.get()["num_tests"].valid(); + ASSERT_TRUE(is_test != is_summary); + if (is_test) { + ++num_tests; + ASSERT_TRUE(!got_summary); + handle_test(slime); + } else { + got_summary = true; + ASSERT_EQUAL(slime.get()["num_tests"].asLong(), int64_t(num_tests)); + handle_summary(slime); + } + } else { + ASSERT_EQUAL(in.obtain().size, 0u); } } + ASSERT_TRUE(got_summary); } //----------------------------------------------------------------------------- -void verify(Input &in) { - while (in.obtain().size > 0) { - Slime slime; - if (JsonFormat::decode(in, slime)) { - ++num_tests; - TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref()); - for (const auto &result: extract_fields(slime.get()["result"])) { - ++result_map[result]; - TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); - if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) { - dump_test(slime.get()); - } - } - } +void evaluate(Input &in, Output &out) { + auto handle_test = [&out](Slime &slime) + { + insert_value(slime.get()["result"], "prod_cpp", + eval_expr(slime.get(), DefaultTensorEngine::ref())); + write_compact(slime, out); + }; + auto handle_summary = [&out](Slime &slime) + { + write_compact(slime, out); + }; + for_each_test(in, handle_test, handle_summary); +} + +//----------------------------------------------------------------------------- + +void dump_test(const Inspector &test) { + fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); + for (const auto &input: extract_fields(test["inputs"])) { + auto value = extract_value(test["inputs"][input]); + fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); } } +void verify(Input &in, Output &out) { + std::map<vespalib::string,size_t> result_map; + auto handle_test = [&out,&result_map](Slime &slime) + { + TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref()); + for (const auto &result: extract_fields(slime.get()["result"])) { + ++result_map[result]; + TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); + if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) { + dump_test(slime.get()); + } + } + }; + auto handle_summary = [&out,&result_map](Slime &slime) + { + Cursor &stats = slime.get().setObject("stats"); + for (const auto &entry: result_map) { + stats.setLong(entry.first, entry.second); + } + write_readable(slime, out); + }; + for_each_test(in, handle_test, handle_summary); +} + //----------------------------------------------------------------------------- int usage(const char *self) { @@ -304,13 +343,10 @@ int main(int argc, char **argv) { TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str()); if (mode == "generate") { generate(std_out); - fprintf(stderr, "generated %zu test cases\n", num_tests); } else if (mode == "evaluate") { evaluate(std_in, std_out); - fprintf(stderr, "evaluated %zu test cases\n", num_tests); } else if (mode == "verify") { - verify(std_in); - fprintf(stderr, "verified %zu test cases (%s)\n", num_tests, result_stats().c_str()); + verify(std_in, std_out); } else { TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str()); } diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json index a7c906cfb85..c66931c2df8 100644 --- a/eval/src/apps/tensor_conformance/test_spec.json +++ b/eval/src/apps/tensor_conformance/test_spec.json @@ -1,39 +1,5 @@ -{ - "expression": "a+a", - "inputs": { - "a": "0x02004000000000000000" - }, - "result": { - "expect": "0x02004010000000000000" - } -} -{ - "expression": "a*b", - "inputs": { - "a": "0x02004000000000000000", - "b": "0x02004008000000000000" - }, - "result": { - "expect": "0x02004018000000000000" - } -} -{ - "expression": "(a+b)*(a-b)", - "inputs": { - "a": "0x02004014000000000000", - "b": "0x02004000000000000000" - }, - "result": { - "expect": "0x02004035000000000000" - } -} -{ - "expression": "(a-b)/(a+b)", - "inputs": { - "a": "0x02004014000000000000", - "b": "0x02004000000000000000" - }, - "result": { - "expect": "0x02003FDB6DB6DB6DB6DB" - } -} +{"expression":"a+a","inputs":{"a":"0x02004000000000000000"},"result":{"expect":"0x02004010000000000000"}} +{"expression":"a*b","inputs":{"a":"0x02004000000000000000","b":"0x02004008000000000000"},"result":{"expect":"0x02004018000000000000"}} +{"expression":"(a+b)*(a-b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02004035000000000000"}} +{"expression":"(a-b)/(a+b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02003FDB6DB6DB6DB6DB"}} +{"num_tests":4} diff --git a/eval/src/tests/tensor/tensor_conformance/.gitignore b/eval/src/tests/tensor/tensor_conformance/.gitignore new file mode 100644 index 00000000000..60177365cf7 --- /dev/null +++ b/eval/src/tests/tensor/tensor_conformance/.gitignore @@ -0,0 +1,2 @@ +/binary_test_spec.json +/conformance_test_spec.json diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index daf641143e8..ad21a50cf1d 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -3,10 +3,12 @@ #include <vespa/eval/eval/test/tensor_conformance.h> #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/vespalib/util/stringfmt.h> using vespalib::eval::SimpleTensorEngine; using vespalib::eval::test::TensorConformance; using vespalib::tensor::DefaultTensorEngine; +using vespalib::make_string; vespalib::string module_path(TEST_PATH("../../../../")); @@ -19,4 +21,24 @@ TEST("require that production tensor implementation passes all conformance tests TEST_DO(TensorConformance::run_tests(module_path, DefaultTensorEngine::ref())); } +TEST("require that tensor serialization test spec can be generated") { + vespalib::string spec = module_path + "src/apps/make_tensor_binary_format_test_spec/test_spec.json"; + vespalib::string binary = module_path + "src/apps/make_tensor_binary_format_test_spec/eval_make_tensor_binary_format_test_spec_app"; + EXPECT_EQUAL(system(make_string("%s > binary_test_spec.json", binary.c_str()).c_str()), 0); + EXPECT_EQUAL(system(make_string("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0); +} + +TEST("require that cross-language tensor conformance test spec can be generated") { + vespalib::string spec = module_path + "src/apps/tensor_conformance/test_spec.json"; + vespalib::string binary = module_path + "src/apps/tensor_conformance/vespa-tensor-conformance"; + EXPECT_EQUAL(system(make_string("%s generate > conformance_test_spec.json", binary.c_str()).c_str()), 0); + EXPECT_EQUAL(system(make_string("diff -u %s conformance_test_spec.json", spec.c_str()).c_str()), 0); +} + +TEST("require that cross-language tensor conformance tests pass with production C++ expression evaluation") { + vespalib::string spec = module_path + "src/apps/tensor_conformance/test_spec.json"; + vespalib::string binary = module_path + "src/apps/tensor_conformance/vespa-tensor-conformance"; + EXPECT_EQUAL(system(make_string("cat %s | %s evaluate | %s verify", spec.c_str(), binary.c_str(), binary.c_str()).c_str()), 0); +} + TEST_MAIN() { TEST_RUN_ALL(); } |