remove conformance test file (run tests directly instead)

2 files changed, 26 insertions, 7 deletions

diff --git a/eval/src/tests/tensor/tensor_conformance/.gitignore b/eval/src/tests/tensor/tensor_conformance/.gitignore
index 60177365cf7..bcc561de73c 100644
--- a/eval/src/tests/tensor/tensor_conformance/.gitignore
+++ b/eval/src/tests/tensor/tensor_conformance/.gitignore
@@ -1,2 +1,2 @@
 /binary_test_spec.json
-/conformance_test_spec.json
+/conformance_result.json
diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
index 765905c67fc..15aa7212a94 100644
--- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
+++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
@@ -5,12 +5,17 @@
 #include <vespa/eval/streamed/streamed_value_builder_factory.h>
 #include <vespa/eval/eval/fast_value.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/io/mapped_file_input.h>
+#include <vespa/vespalib/data/slime/slime.h>
 
 using vespalib::eval::SimpleValueBuilderFactory;
 using vespalib::eval::StreamedValueBuilderFactory;
 using vespalib::eval::FastValueBuilderFactory;
 using vespalib::eval::test::TensorConformance;
-using vespalib::make_string;
+using vespalib::make_string_short::fmt;
+using vespalib::Slime;
+using vespalib::slime::JsonFormat;
+using vespalib::MappedFileInput;
 
 vespalib::string module_src_path(TEST_PATH("../../../../"));
 vespalib::string module_build_path("../../../../");
@@ -30,14 +35,28 @@ TEST("require that FastValue implementation passes all conformance tests") {
 TEST("require that tensor serialization test spec can be generated") {
     vespalib::string spec = module_src_path + "src/apps/make_tensor_binary_format_test_spec/test_spec.json";
     vespalib::string binary = module_build_path + "src/apps/make_tensor_binary_format_test_spec/eval_make_tensor_binary_format_test_spec_app";
-    EXPECT_EQUAL(system(make_string("%s > binary_test_spec.json", binary.c_str()).c_str()), 0);
-    EXPECT_EQUAL(system(make_string("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(fmt("%s > binary_test_spec.json", binary.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(fmt("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0);
 }
 
-TEST("require that cross-language tensor conformance tests pass with production C++ expression evaluation") {
-    vespalib::string spec = module_src_path + "src/apps/tensor_conformance/test_spec.json";
+TEST("require that cross-language tensor conformance tests pass with C++ expression evaluation") {
+    vespalib::string result_file = "conformance_result.json";
     vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance";
-    EXPECT_EQUAL(system(make_string("cat %s | %s evaluate | %s verify", spec.c_str(), binary.c_str(), binary.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(fmt("%s generate | %s evaluate | %s verify > %s", binary.c_str(), binary.c_str(), binary.c_str(), result_file.c_str()).c_str()), 0);
+    Slime result;
+    MappedFileInput input(result_file);
+    JsonFormat::decode(input, result);
+    fprintf(stderr, "conformance summary: %s\n", result.toString().c_str());
+    int num_tests = result.get()["num_tests"].asLong();
+    int prod_tests = result.get()["stats"]["cpp_prod"].asLong();
+    int simple_tests = result.get()["stats"]["cpp_simple_value"].asLong();
+    int streamed_tests = result.get()["stats"]["cpp_streamed_value"].asLong();
+    int with_expect = result.get()["stats"]["expect"].asLong();
+    EXPECT_GREATER(num_tests, 1000);
+    EXPECT_EQUAL(prod_tests, num_tests);
+    EXPECT_EQUAL(simple_tests, num_tests);
+    EXPECT_EQUAL(streamed_tests, num_tests);
+    EXPECT_EQUAL(with_expect, num_tests);
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }