diff options
author | Håvard Pettersen <havardpe@oath.com> | 2019-09-02 12:18:29 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2019-09-02 12:18:29 +0000 |
commit | b93a431d9b94623332b30ee2e2509b1eea4d1043 (patch) | |
tree | ef3e1d4878126f7bfacd67ec889782a38df825d3 /eval/src/apps/tensor_conformance | |
parent | f1991f6ef4901dc3d8099e779c5f629450d9c3ed (diff) |
more robust verification of tensor conformance test generation
Diffstat (limited to 'eval/src/apps/tensor_conformance')
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 102 |
1 files changed, 101 insertions, 1 deletions
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index f727f623d3f..1a760a436b6 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/tensor_spec.h> @@ -16,6 +17,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/test_io.h> #include <unistd.h> +#include <functional> #include "generate.h" @@ -23,8 +25,11 @@ using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::eval::test; using namespace vespalib::slime::convenience; +using vespalib::slime::inject; +using vespalib::slime::SlimeInserter; using slime::JsonFormat; using tensor::DefaultTensorEngine; +using namespace std::placeholders; //----------------------------------------------------------------------------- @@ -213,8 +218,84 @@ void verify(Input &in, Output &out) { //----------------------------------------------------------------------------- +struct TestList { + std::vector<Slime> list; + void add_test(Slime &slime) { + list.emplace_back(); + inject(slime.get(), SlimeInserter(list.back())); + } +}; + +struct TestSpec { + vespalib::string expression; + std::vector<TensorSpec> inputs; + TensorSpec result; + TestSpec() : expression(), inputs(), result("error") {} + ~TestSpec(); + void decode(const Inspector &test) { + const auto &my_expression = test["expression"]; + ASSERT_TRUE(my_expression.valid()); + expression = my_expression.asString().make_string(); + Function fun = Function::parse(expression); + ASSERT_TRUE(!fun.has_error()); + ASSERT_EQUAL(fun.num_params(), test["inputs"].fields()); + for (size_t i = 0; i < fun.num_params(); ++i) { + TEST_STATE(make_string("input #%zu", i).c_str()); + const auto &my_input = test["inputs"][fun.param_name(i)]; + ASSERT_TRUE(my_input.valid()); + inputs.push_back(extract_value(my_input)); + } + const auto &my_result = test["result"]["expect"]; + ASSERT_TRUE(my_result.valid()); + result = extract_value(my_result); + } +}; +TestSpec::~TestSpec() = default; + +void compare_test(const Inspector &expect_in, const Inspector &actual_in) { + TestSpec expect; + TestSpec actual; + { + TEST_STATE("decoding expected test case"); + expect.decode(expect_in); + } + { + TEST_STATE("decoding actual test case"); + actual.decode(actual_in); + } + { + TEST_STATE("comparing test cases"); + ASSERT_EQUAL(expect.expression, actual.expression); + ASSERT_EQUAL(expect.inputs.size(), actual.inputs.size()); + for (size_t i = 0; i < expect.inputs.size(); ++i) { + TEST_STATE(make_string("input #%zu", i).c_str()); + ASSERT_EQUAL(expect.inputs[i], actual.inputs[i]); + } + ASSERT_EQUAL(expect.result, actual.result); + } +} + +void compare(Input &expect, Input &actual) { + TestList expect_tests; + TestList actual_tests; + for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &){}); + for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &){}); + ASSERT_TRUE(!expect_tests.list.empty()); + ASSERT_TRUE(!actual_tests.list.empty()); + ASSERT_EQUAL(expect_tests.list.size(), actual_tests.list.size()); + size_t num_tests = expect_tests.list.size(); + fprintf(stderr, "...found %zu test cases to compare...\n", num_tests); + for (size_t i = 0; i < num_tests; ++i) { + TEST_STATE(make_string("test case #%zu", i).c_str()); + compare_test(expect_tests.list[i].get(), actual_tests.list[i].get()); + } +} + +//----------------------------------------------------------------------------- + int usage(const char *self) { fprintf(stderr, "usage: %s <mode>\n", self); + fprintf(stderr, "usage: %s compare <expect> <actual>\n", self); fprintf(stderr, " <mode>: which mode to activate\n"); fprintf(stderr, " 'generate': write test cases to stdout\n"); fprintf(stderr, " 'evaluate': read test cases from stdin, annotate them with\n"); @@ -222,13 +303,15 @@ int usage(const char *self) { fprintf(stderr, " them to stdout\n"); fprintf(stderr, " 'verify': read annotated test cases from stdin and verify\n"); fprintf(stderr, " that all results are as expected\n"); + fprintf(stderr, " 'compare': read test cases from two separate files and\n"); + fprintf(stderr, " compare them to verify equivalence\n"); return 1; } int main(int argc, char **argv) { StdIn std_in; StdOut std_out; - if (argc != 2) { + if (argc < 2) { return usage(argv[0]); } vespalib::string mode = argv[1]; @@ -239,6 +322,23 @@ int main(int argc, char **argv) { evaluate(std_in, std_out); } else if (mode == "verify") { verify(std_in, std_out); + } else if (mode == "compare") { + if (argc == 4) { + MappedFileInput expect(argv[2]); + MappedFileInput actual(argv[3]); + if (expect.valid() && actual.valid()) { + compare(expect, actual); + } else { + if (!expect.valid()) { + TEST_ERROR(make_string("could not read file: %s", argv[2]).c_str()); + } + if (!actual.valid()) { + TEST_ERROR(make_string("could not read file: %s", argv[3]).c_str()); + } + } + } else { + TEST_ERROR("wrong number of parameters for 'compare'\n"); + } } else { TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str()); } |