more robust verification of tensor conformance test generation

2 files changed, 102 insertions, 2 deletions

diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index f727f623d3f..1a760a436b6 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/data/slime/json_format.h>
+#include <vespa/vespalib/io/mapped_file_input.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/eval/eval/tensor_spec.h>
@@ -16,6 +17,7 @@
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/eval/test/test_io.h>
 #include <unistd.h>
+#include <functional>
 
 #include "generate.h"
 
@@ -23,8 +25,11 @@ using namespace vespalib;
 using namespace vespalib::eval;
 using namespace vespalib::eval::test;
 using namespace vespalib::slime::convenience;
+using vespalib::slime::inject;
+using vespalib::slime::SlimeInserter;
 using slime::JsonFormat;
 using tensor::DefaultTensorEngine;
+using namespace std::placeholders;
 
 //-----------------------------------------------------------------------------
 
@@ -213,8 +218,84 @@ void verify(Input &in, Output &out) {
 
 //-----------------------------------------------------------------------------
 
+struct TestList {
+    std::vector<Slime> list;
+    void add_test(Slime &slime) {
+        list.emplace_back();
+        inject(slime.get(), SlimeInserter(list.back()));
+    }
+};
+
+struct TestSpec {
+    vespalib::string expression;
+    std::vector<TensorSpec> inputs;
+    TensorSpec result;
+    TestSpec() : expression(), inputs(), result("error") {}
+    ~TestSpec();
+    void decode(const Inspector &test) {
+        const auto &my_expression = test["expression"];
+        ASSERT_TRUE(my_expression.valid());
+        expression = my_expression.asString().make_string();
+        Function fun = Function::parse(expression);
+        ASSERT_TRUE(!fun.has_error());
+        ASSERT_EQUAL(fun.num_params(), test["inputs"].fields());
+        for (size_t i = 0; i < fun.num_params(); ++i) {
+            TEST_STATE(make_string("input #%zu", i).c_str());
+            const auto &my_input = test["inputs"][fun.param_name(i)];
+            ASSERT_TRUE(my_input.valid());
+            inputs.push_back(extract_value(my_input));
+        }
+        const auto &my_result = test["result"]["expect"];
+        ASSERT_TRUE(my_result.valid());
+        result = extract_value(my_result);
+    }
+};
+TestSpec::~TestSpec() = default;
+
+void compare_test(const Inspector &expect_in, const Inspector &actual_in) {
+    TestSpec expect;
+    TestSpec actual;
+    {
+        TEST_STATE("decoding expected test case");
+        expect.decode(expect_in);
+    }
+    {
+        TEST_STATE("decoding actual test case");
+        actual.decode(actual_in);
+    }
+    {
+        TEST_STATE("comparing test cases");
+        ASSERT_EQUAL(expect.expression, actual.expression);
+        ASSERT_EQUAL(expect.inputs.size(), actual.inputs.size());
+        for (size_t i = 0; i < expect.inputs.size(); ++i) {
+            TEST_STATE(make_string("input #%zu", i).c_str());
+            ASSERT_EQUAL(expect.inputs[i], actual.inputs[i]);
+        }
+        ASSERT_EQUAL(expect.result, actual.result);
+    }
+}
+
+void compare(Input &expect, Input &actual) {
+    TestList expect_tests;
+    TestList actual_tests;
+    for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &){});
+    for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &){});
+    ASSERT_TRUE(!expect_tests.list.empty());
+    ASSERT_TRUE(!actual_tests.list.empty());
+    ASSERT_EQUAL(expect_tests.list.size(), actual_tests.list.size());
+    size_t num_tests = expect_tests.list.size();
+    fprintf(stderr, "...found %zu test cases to compare...\n", num_tests);
+    for (size_t i = 0; i < num_tests; ++i) {
+        TEST_STATE(make_string("test case #%zu", i).c_str());
+        compare_test(expect_tests.list[i].get(), actual_tests.list[i].get());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
 int usage(const char *self) {
     fprintf(stderr, "usage: %s <mode>\n", self);
+    fprintf(stderr, "usage: %s compare <expect> <actual>\n", self);
     fprintf(stderr, "  <mode>: which mode to activate\n");
     fprintf(stderr, "    'generate': write test cases to stdout\n");
     fprintf(stderr, "    'evaluate': read test cases from stdin, annotate them with\n");
@@ -222,13 +303,15 @@ int usage(const char *self) {
     fprintf(stderr, "                them to stdout\n");
     fprintf(stderr, "    'verify': read annotated test cases from stdin and verify\n");
     fprintf(stderr, "              that all results are as expected\n");
+    fprintf(stderr, "    'compare': read test cases from two separate files and\n");
+    fprintf(stderr, "               compare them to verify equivalence\n");
     return 1;
 }
 
 int main(int argc, char **argv) {
     StdIn std_in;
     StdOut std_out;
-    if (argc != 2) {
+    if (argc < 2) {
         return usage(argv[0]);
     }
     vespalib::string mode = argv[1];
@@ -239,6 +322,23 @@ int main(int argc, char **argv) {
         evaluate(std_in, std_out);
     } else if (mode == "verify") {
         verify(std_in, std_out);
+    } else if (mode == "compare") {
+        if (argc == 4) {
+            MappedFileInput expect(argv[2]);
+            MappedFileInput actual(argv[3]);
+            if (expect.valid() && actual.valid()) {
+                compare(expect, actual);
+            } else {
+                if (!expect.valid()) {
+                    TEST_ERROR(make_string("could not read file: %s", argv[2]).c_str());
+                }
+                if (!actual.valid()) {
+                    TEST_ERROR(make_string("could not read file: %s", argv[3]).c_str());
+                }
+            }
+        } else {
+            TEST_ERROR("wrong number of parameters for 'compare'\n");
+        }
     } else {
         TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str());
     }
diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
index ec9fc396f0c..df415e39134 100644
--- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
+++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
@@ -32,7 +32,7 @@ TEST("require that cross-language tensor conformance test spec can be generated"
     vespalib::string spec = module_src_path + "src/apps/tensor_conformance/test_spec.json";
     vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance";
     EXPECT_EQUAL(system(make_string("%s generate > conformance_test_spec.json", binary.c_str()).c_str()), 0);
-    EXPECT_EQUAL(system(make_string("diff -u %s conformance_test_spec.json", spec.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(make_string("%s compare %s conformance_test_spec.json", binary.c_str(), spec.c_str()).c_str()), 0);
 }
 
 TEST("require that cross-language tensor conformance tests pass with production C++ expression evaluation") {