wire in spec-based tensor conformance testing

verify conformance spec generation
verify C++ default expression evaluation
bonus: verify binary format test spec generation

5 files changed, 135 insertions, 109 deletions

diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h
index d20d085f00c..0f74ce924b3 100644
--- a/eval/src/apps/tensor_conformance/generate.h
+++ b/eval/src/apps/tensor_conformance/generate.h
@@ -7,13 +7,13 @@
 
 struct TestBuilder {
     using TensorSpec = vespalib::eval::TensorSpec;
-    // add test with undefined expected result
-    virtual void add(const vespalib::string &expression,
-                     const std::map<vespalib::string,TensorSpec> &inputs) = 0;
     // add test with pre-defined expected result
     virtual void add(const vespalib::string &expression,
                      const std::map<vespalib::string,TensorSpec> &inputs,
                      const TensorSpec &expect) = 0;
+    // add test with undefined expected result
+    virtual void add(const vespalib::string &expression,
+                     const std::map<vespalib::string,TensorSpec> &inputs) = 0;
     virtual ~TestBuilder() {}
 };
 
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index 593e4439b0a..cfe9542ecda 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -25,23 +25,6 @@ using slime::JsonFormat;
 using tensor::DefaultTensorEngine;
 
 constexpr size_t CHUNK_SIZE = 16384;
-constexpr bool not_compact = false;
-
-//-----------------------------------------------------------------------------
-
-size_t num_tests = 0;
-std::map<vespalib::string,size_t> result_map;
-
-vespalib::string result_stats() {
-    vespalib::string stats;
-    for (const auto &entry: result_map) {        
-        if (!stats.empty()) {
-            stats += ", ";
-        }
-        stats += make_string("%s: %zu", entry.first.c_str(), entry.second);
-    }
-    return stats;
-}
 
 //-----------------------------------------------------------------------------
 
@@ -85,6 +68,16 @@ public:
     }
 };
 
+void write_compact(const Slime &slime, Output &out) {
+    JsonFormat::encode(slime, out, true);
+    out.reserve(1).data[0] = '\n';
+    out.commit(1);
+}
+
+void write_readable(const Slime &slime, Output &out) {
+    JsonFormat::encode(slime, out, false);
+}
+
 //-----------------------------------------------------------------------------
 
 uint8_t unhex(char c) {
@@ -193,93 +186,139 @@ std::vector<vespalib::string> extract_fields(const Inspector &object) {
     return std::move(extractor.result);
 };
 
-void dump_test(const Inspector &test) {
-    fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str());
-    for (const auto &input: extract_fields(test["inputs"])) {
-        auto value = extract_value(test["inputs"][input]);
-        fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str());
-    }
-}
-
 //-----------------------------------------------------------------------------
 
 class MyTestBuilder : public TestBuilder {
 private:
     Output &_out;
-    void build_test(Cursor &test, const vespalib::string &expression,
-                    const std::map<vespalib::string,TensorSpec> &input_map)
+    size_t  _num_tests;
+    void make_test(const vespalib::string &expression,
+                   const std::map<vespalib::string,TensorSpec> &input_map,
+                   const TensorSpec *expect = nullptr)
     {
+        Slime slime;
+        Cursor &test = slime.setObject();
         test.setString("expression", expression);
         Cursor &inputs = test.setObject("inputs");
         for (const auto &input: input_map) {
             insert_value(inputs, input.first, input.second);
         }
+        if (expect != nullptr) {
+            insert_value(test.setObject("result"), "expect", *expect);
+        } else {
+            insert_value(test.setObject("result"), "expect",
+                         eval_expr(slime.get(), SimpleTensorEngine::ref()));
+        }
+        write_compact(slime, _out);
+        ++_num_tests;
     }
 public:
-    MyTestBuilder(Output &out) : _out(out) {}
+    MyTestBuilder(Output &out) : _out(out), _num_tests(0) {}
     void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs) override
+             const std::map<vespalib::string,TensorSpec> &inputs,
+             const TensorSpec &expect) override
     {
-        Slime slime;
-        build_test(slime.setObject(), expression, inputs);
-        insert_value(slime.get().setObject("result"), "expect",
-                     eval_expr(slime.get(), SimpleTensorEngine::ref()));
-        JsonFormat::encode(slime, _out, not_compact);
-        ++num_tests;
+        make_test(expression, inputs, &expect);
     }
     void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs,
-             const TensorSpec &expect) override
+             const std::map<vespalib::string,TensorSpec> &inputs) override
     {
+        make_test(expression, inputs);
+    }
+    void make_summary() {
         Slime slime;
-        build_test(slime.setObject(), expression, inputs);
-        insert_value(slime.get().setObject("result"), "expect", expect);
-        if (!EXPECT_EQUAL(eval_expr(slime.get(), SimpleTensorEngine::ref()), expect)) {
-            dump_test(slime.get());
-        }
-        JsonFormat::encode(slime, _out, not_compact);
-        ++num_tests;
+        Cursor &summary = slime.setObject();
+        summary.setLong("num_tests", _num_tests);
+        write_compact(slime, _out);
     }
 };
 
 void generate(Output &out) {
     MyTestBuilder my_test_builder(out);
     Generator::generate(my_test_builder);
+    my_test_builder.make_summary();
 }
 
 //-----------------------------------------------------------------------------
 
-void evaluate(Input &in, Output &out) {
+void for_each_test(Input &in,
+                   const std::function<void(Slime&)> &handle_test,
+                   const std::function<void(Slime&)> &handle_summary)
+{
+    size_t num_tests = 0;
+    bool got_summary = false;
     while (in.obtain().size > 0) {
         Slime slime;
         if (JsonFormat::decode(in, slime)) {
-            ++num_tests;
-            insert_value(slime.get()["result"], "prod_cpp",
-                         eval_expr(slime.get(), DefaultTensorEngine::ref()));
-            JsonFormat::encode(slime, out, not_compact);
+            bool is_test = slime.get()["expression"].valid();
+            bool is_summary = slime.get()["num_tests"].valid();
+            ASSERT_TRUE(is_test != is_summary);
+            if (is_test) {
+                ++num_tests;
+                ASSERT_TRUE(!got_summary);
+                handle_test(slime);
+            } else {
+                got_summary = true;
+                ASSERT_EQUAL(slime.get()["num_tests"].asLong(), int64_t(num_tests));
+                handle_summary(slime);
+            }
+        } else {
+            ASSERT_EQUAL(in.obtain().size, 0u);
         }
     }
+    ASSERT_TRUE(got_summary);
 }
 
 //-----------------------------------------------------------------------------
 
-void verify(Input &in) {
-    while (in.obtain().size > 0) {
-        Slime slime;
-        if (JsonFormat::decode(in, slime)) {
-            ++num_tests;
-            TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref());
-            for (const auto &result: extract_fields(slime.get()["result"])) {
-                ++result_map[result];
-                TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str());
-                if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) {
-                    dump_test(slime.get());
-                }
-            }
-        }
+void evaluate(Input &in, Output &out) {
+    auto handle_test = [&out](Slime &slime)
+                       {
+                           insert_value(slime.get()["result"], "prod_cpp",
+                                   eval_expr(slime.get(), DefaultTensorEngine::ref()));
+                           write_compact(slime, out);
+                       };
+    auto handle_summary = [&out](Slime &slime)
+                          {
+                              write_compact(slime, out);
+                          };
+    for_each_test(in, handle_test, handle_summary);
+}
+
+//-----------------------------------------------------------------------------
+
+void dump_test(const Inspector &test) {
+    fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str());
+    for (const auto &input: extract_fields(test["inputs"])) {
+        auto value = extract_value(test["inputs"][input]);
+        fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str());
     }
 }
 
+void verify(Input &in, Output &out) {
+    std::map<vespalib::string,size_t> result_map;
+    auto handle_test = [&out,&result_map](Slime &slime)
+                       {
+                           TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref());
+                           for (const auto &result: extract_fields(slime.get()["result"])) {
+                               ++result_map[result];
+                               TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str());
+                               if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) {
+                                   dump_test(slime.get());
+                               }
+                           }
+                       };
+    auto handle_summary = [&out,&result_map](Slime &slime)
+                          {
+                              Cursor &stats = slime.get().setObject("stats");
+                              for (const auto &entry: result_map) {
+                                  stats.setLong(entry.first, entry.second);
+                              }
+                              write_readable(slime, out);
+                          };
+    for_each_test(in, handle_test, handle_summary);
+}
+
 //-----------------------------------------------------------------------------
 
 int usage(const char *self) {
@@ -304,13 +343,10 @@ int main(int argc, char **argv) {
     TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str());
     if (mode == "generate") {
         generate(std_out);
-        fprintf(stderr, "generated %zu test cases\n", num_tests);
     } else if (mode == "evaluate") {
         evaluate(std_in, std_out);
-        fprintf(stderr, "evaluated %zu test cases\n", num_tests);
     } else if (mode == "verify") {
-        verify(std_in);
-        fprintf(stderr, "verified %zu test cases (%s)\n", num_tests, result_stats().c_str());
+        verify(std_in, std_out);
     } else {
         TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str());
     }
diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json
index a7c906cfb85..c66931c2df8 100644
--- a/eval/src/apps/tensor_conformance/test_spec.json
+++ b/eval/src/apps/tensor_conformance/test_spec.json
@@ -1,39 +1,5 @@
-{
-    "expression": "a+a",
-    "inputs": {
-        "a": "0x02004000000000000000"
-    },
-    "result": {
-        "expect": "0x02004010000000000000"
-    }
-}
-{
-    "expression": "a*b",
-    "inputs": {
-        "a": "0x02004000000000000000",
-        "b": "0x02004008000000000000"
-    },
-    "result": {
-        "expect": "0x02004018000000000000"
-    }
-}
-{
-    "expression": "(a+b)*(a-b)",
-    "inputs": {
-        "a": "0x02004014000000000000",
-        "b": "0x02004000000000000000"
-    },
-    "result": {
-        "expect": "0x02004035000000000000"
-    }
-}
-{
-    "expression": "(a-b)/(a+b)",
-    "inputs": {
-        "a": "0x02004014000000000000",
-        "b": "0x02004000000000000000"
-    },
-    "result": {
-        "expect": "0x02003FDB6DB6DB6DB6DB"
-    }
-}
+{"expression":"a+a","inputs":{"a":"0x02004000000000000000"},"result":{"expect":"0x02004010000000000000"}}
+{"expression":"a*b","inputs":{"a":"0x02004000000000000000","b":"0x02004008000000000000"},"result":{"expect":"0x02004018000000000000"}}
+{"expression":"(a+b)*(a-b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02004035000000000000"}}
+{"expression":"(a-b)/(a+b)","inputs":{"a":"0x02004014000000000000","b":"0x02004000000000000000"},"result":{"expect":"0x02003FDB6DB6DB6DB6DB"}}
+{"num_tests":4}
diff --git a/eval/src/tests/tensor/tensor_conformance/.gitignore b/eval/src/tests/tensor/tensor_conformance/.gitignore
new file mode 100644
index 00000000000..60177365cf7
--- /dev/null
+++ b/eval/src/tests/tensor/tensor_conformance/.gitignore
@@ -0,0 +1,2 @@
+/binary_test_spec.json
+/conformance_test_spec.json
diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
index daf641143e8..ad21a50cf1d 100644
--- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
+++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
@@ -3,10 +3,12 @@
 #include <vespa/eval/eval/test/tensor_conformance.h>
 #include <vespa/eval/eval/simple_tensor_engine.h>
 #include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using vespalib::eval::SimpleTensorEngine;
 using vespalib::eval::test::TensorConformance;
 using vespalib::tensor::DefaultTensorEngine;
+using vespalib::make_string;
 
 vespalib::string module_path(TEST_PATH("../../../../"));
 
@@ -19,4 +21,24 @@ TEST("require that production tensor implementation passes all conformance tests
     TEST_DO(TensorConformance::run_tests(module_path, DefaultTensorEngine::ref()));
 }
 
+TEST("require that tensor serialization test spec can be generated") {
+    vespalib::string spec = module_path + "src/apps/make_tensor_binary_format_test_spec/test_spec.json";
+    vespalib::string binary = module_path + "src/apps/make_tensor_binary_format_test_spec/eval_make_tensor_binary_format_test_spec_app";
+    EXPECT_EQUAL(system(make_string("%s > binary_test_spec.json", binary.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(make_string("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0);
+}
+
+TEST("require that cross-language tensor conformance test spec can be generated") {
+    vespalib::string spec = module_path + "src/apps/tensor_conformance/test_spec.json";
+    vespalib::string binary = module_path + "src/apps/tensor_conformance/vespa-tensor-conformance";
+    EXPECT_EQUAL(system(make_string("%s generate > conformance_test_spec.json", binary.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(make_string("diff -u %s conformance_test_spec.json", spec.c_str()).c_str()), 0);
+}
+
+TEST("require that cross-language tensor conformance tests pass with production C++ expression evaluation") {
+    vespalib::string spec = module_path + "src/apps/tensor_conformance/test_spec.json";
+    vespalib::string binary = module_path + "src/apps/tensor_conformance/vespa-tensor-conformance";
+    EXPECT_EQUAL(system(make_string("cat %s | %s evaluate | %s verify", spec.c_str(), binary.c_str(), binary.c_str()).c_str()), 0);
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }