unify conformance test format, refactor code and update spec

7 files changed, 264 insertions, 468 deletions

diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp
index a7695408a85..6b10d2782f2 100644
--- a/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp
+++ b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp
@@ -5,6 +5,7 @@
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/eval/eval/tensor_spec.h>
 #include <vespa/eval/eval/value_type.h>
+#include <vespa/eval/eval/test/test_io.h>
 #include <iostream>
 
 using namespace vespalib;
@@ -297,27 +298,24 @@ void make_vector_map_test(Cursor &test,
 
 //-----------------------------------------------------------------------------
 
-void make_tests(Cursor &tests) {
-    make_number_test(tests.addObject(), 0.0);
-    make_number_test(tests.addObject(), 42.0);
-    make_vector_test(tests.addObject(), 3);
-    make_matrix_test(tests.addObject(), 2, 3);
-    make_map_test(tests.addObject(), {});
-    make_map_test(tests.addObject(), {"a", "b", "c"});
-    make_mesh_test(tests.addObject(), {}, "a");
-    make_mesh_test(tests.addObject(), {"foo", "bar"}, "a");
-    make_vector_map_test(tests.addObject(), "x", {}, "y", 10);
-    make_vector_map_test(tests.addObject(), "y", {}, "x", 10);
-    make_vector_map_test(tests.addObject(), "x", {"a", "b"}, "y", 3);
-    make_vector_map_test(tests.addObject(), "y", {"a", "b"}, "x", 3);
+void make_tests(test::TestWriter &writer) {
+    make_number_test(writer.create(), 0.0);
+    make_number_test(writer.create(), 42.0);
+    make_vector_test(writer.create(), 3);
+    make_matrix_test(writer.create(), 2, 3);
+    make_map_test(writer.create(), {});
+    make_map_test(writer.create(), {"a", "b", "c"});
+    make_mesh_test(writer.create(), {}, "a");
+    make_mesh_test(writer.create(), {"foo", "bar"}, "a");
+    make_vector_map_test(writer.create(), "x", {}, "y", 10);
+    make_vector_map_test(writer.create(), "y", {}, "x", 10);
+    make_vector_map_test(writer.create(), "x", {"a", "b"}, "y", 3);
+    make_vector_map_test(writer.create(), "y", {"a", "b"}, "x", 3);
 }
 
 int main(int, char **) {
-    Slime slime;
-    Cursor &top = slime.setObject();
-    Cursor &tests = top.setArray("tests");
-    make_tests(tests);
-    top.setLong("num_tests", tests.entries());
-    fprintf(stdout, "%s", slime.toString().c_str());
+    test::StdOut std_out;
+    test::TestWriter writer(std_out);
+    make_tests(writer);
     return 0;
 }
diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json b/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json
index 1b74b4b8838..701b829e5bc 100644
--- a/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json
+++ b/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json
@@ -1,336 +1,13 @@
-{
-    "tests": [
-        {
-            "tensor": {
-                "type": "double",
-                "cells": [
-                    {
-                        "address": {
-                        },
-                        "value": 0
-                    }
-                ]
-            },
-            "binary": [
-                "0x0100010000000000000000",
-                "0x02000000000000000000",
-                "0x0300000000000000000000",
-                "0x010000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "double",
-                "cells": [
-                    {
-                        "address": {
-                        },
-                        "value": 42
-                    }
-                ]
-            },
-            "binary": [
-                "0x0100014045000000000000",
-                "0x02004045000000000000",
-                "0x0300004045000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x[3])",
-                "cells": [
-                    {
-                        "address": {
-                            "x": 0
-                        },
-                        "value": 1
-                    },
-                    {
-                        "address": {
-                            "x": 1
-                        },
-                        "value": 2
-                    },
-                    {
-                        "address": {
-                            "x": 2
-                        },
-                        "value": 3
-                    }
-                ]
-            },
-            "binary": [
-                "0x02010178033FF000000000000040000000000000004008000000000000",
-                "0x0300010178033FF000000000000040000000000000004008000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x[2],y[3])",
-                "cells": [
-                    {
-                        "address": {
-                            "x": 0,
-                            "y": 0
-                        },
-                        "value": 11
-                    },
-                    {
-                        "address": {
-                            "x": 0,
-                            "y": 1
-                        },
-                        "value": 12
-                    },
-                    {
-                        "address": {
-                            "x": 0,
-                            "y": 2
-                        },
-                        "value": 13
-                    },
-                    {
-                        "address": {
-                            "x": 1,
-                            "y": 0
-                        },
-                        "value": 21
-                    },
-                    {
-                        "address": {
-                            "x": 1,
-                            "y": 1
-                        },
-                        "value": 22
-                    },
-                    {
-                        "address": {
-                            "x": 1,
-                            "y": 2
-                        },
-                        "value": 23
-                    }
-                ]
-            },
-            "binary": [
-                "0x020201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000",
-                "0x03000201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{})",
-                "cells": [
-                ]
-            },
-            "binary": [
-                "0x0101017800",
-                "0x030101780000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{})",
-                "cells": [
-                    {
-                        "address": {
-                            "x": "a"
-                        },
-                        "value": 1
-                    },
-                    {
-                        "address": {
-                            "x": "b"
-                        },
-                        "value": 2
-                    },
-                    {
-                        "address": {
-                            "x": "c"
-                        },
-                        "value": 3
-                    }
-                ]
-            },
-            "binary": [
-                "0x010101780301613FF00000000000000162400000000000000001634008000000000000",
-                "0x03010178000301613FF00000000000000162400000000000000001634008000000000000",
-                "0x010101780301613FF00000000000000163400800000000000001624000000000000000",
-                "0x03010178000301613FF00000000000000163400800000000000001624000000000000000",
-                "0x01010178030162400000000000000001613FF000000000000001634008000000000000",
-                "0x0301017800030162400000000000000001613FF000000000000001634008000000000000",
-                "0x0101017803016240000000000000000163400800000000000001613FF0000000000000",
-                "0x030101780003016240000000000000000163400800000000000001613FF0000000000000",
-                "0x01010178030163400800000000000001613FF000000000000001624000000000000000",
-                "0x0301017800030163400800000000000001613FF000000000000001624000000000000000",
-                "0x0101017803016340080000000000000162400000000000000001613FF0000000000000",
-                "0x030101780003016340080000000000000162400000000000000001613FF0000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{},y{})",
-                "cells": [
-                ]
-            },
-            "binary": [
-                "0x01020178017900",
-                "0x0302017801790000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{},y{})",
-                "cells": [
-                    {
-                        "address": {
-                            "x": "bar",
-                            "y": "a"
-                        },
-                        "value": 21
-                    },
-                    {
-                        "address": {
-                            "x": "foo",
-                            "y": "a"
-                        },
-                        "value": 11
-                    }
-                ]
-            },
-            "binary": [
-                "0x0102017801790203666F6F016140260000000000000362617201614035000000000000",
-                "0x030201780179000203666F6F016140260000000000000362617201614035000000000000",
-                "0x01020178017902036261720161403500000000000003666F6F01614026000000000000",
-                "0x0302017801790002036261720161403500000000000003666F6F01614026000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{},y[10])",
-                "cells": [
-                ]
-            },
-            "binary": [
-                "0x030101780101790A00"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x[10],y{})",
-                "cells": [
-                ]
-            },
-            "binary": [
-                "0x030101790101780A00"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x{},y[3])",
-                "cells": [
-                    {
-                        "address": {
-                            "x": "a",
-                            "y": 0
-                        },
-                        "value": 11
-                    },
-                    {
-                        "address": {
-                            "x": "a",
-                            "y": 1
-                        },
-                        "value": 12
-                    },
-                    {
-                        "address": {
-                            "x": "a",
-                            "y": 2
-                        },
-                        "value": 13
-                    },
-                    {
-                        "address": {
-                            "x": "b",
-                            "y": 0
-                        },
-                        "value": 21
-                    },
-                    {
-                        "address": {
-                            "x": "b",
-                            "y": 1
-                        },
-                        "value": 22
-                    },
-                    {
-                        "address": {
-                            "x": "b",
-                            "y": 2
-                        },
-                        "value": 23
-                    }
-                ]
-            },
-            "binary": [
-                "0x030101780101790302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000",
-                "0x0301017801017903020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000"
-            ]
-        },
-        {
-            "tensor": {
-                "type": "tensor(x[3],y{})",
-                "cells": [
-                    {
-                        "address": {
-                            "x": 0,
-                            "y": "a"
-                        },
-                        "value": 11
-                    },
-                    {
-                        "address": {
-                            "x": 0,
-                            "y": "b"
-                        },
-                        "value": 21
-                    },
-                    {
-                        "address": {
-                            "x": 1,
-                            "y": "a"
-                        },
-                        "value": 12
-                    },
-                    {
-                        "address": {
-                            "x": 1,
-                            "y": "b"
-                        },
-                        "value": 22
-                    },
-                    {
-                        "address": {
-                            "x": 2,
-                            "y": "a"
-                        },
-                        "value": 13
-                    },
-                    {
-                        "address": {
-                            "x": 2,
-                            "y": "b"
-                        },
-                        "value": 23
-                    }
-                ]
-            },
-            "binary": [
-                "0x030101790101780302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000",
-                "0x0301017901017803020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000"
-            ]
-        }
-    ],
-    "num_tests": 12
-}
+{"tensor":{"type":"double","cells":[{"address":{},"value":0}]},"binary":["0x0100010000000000000000","0x02000000000000000000","0x0300000000000000000000","0x010000"]}
+{"tensor":{"type":"double","cells":[{"address":{},"value":42}]},"binary":["0x0100014045000000000000","0x02004045000000000000","0x0300004045000000000000"]}
+{"tensor":{"type":"tensor(x[3])","cells":[{"address":{"x":0},"value":1},{"address":{"x":1},"value":2},{"address":{"x":2},"value":3}]},"binary":["0x02010178033FF000000000000040000000000000004008000000000000","0x0300010178033FF000000000000040000000000000004008000000000000"]}
+{"tensor":{"type":"tensor(x[2],y[3])","cells":[{"address":{"x":0,"y":0},"value":11},{"address":{"x":0,"y":1},"value":12},{"address":{"x":0,"y":2},"value":13},{"address":{"x":1,"y":0},"value":21},{"address":{"x":1,"y":1},"value":22},{"address":{"x":1,"y":2},"value":23}]},"binary":["0x020201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000","0x03000201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000"]}
+{"tensor":{"type":"tensor(x{})","cells":[]},"binary":["0x0101017800","0x030101780000"]}
+{"tensor":{"type":"tensor(x{})","cells":[{"address":{"x":"a"},"value":1},{"address":{"x":"b"},"value":2},{"address":{"x":"c"},"value":3}]},"binary":["0x010101780301613FF00000000000000162400000000000000001634008000000000000","0x03010178000301613FF00000000000000162400000000000000001634008000000000000","0x010101780301613FF00000000000000163400800000000000001624000000000000000","0x03010178000301613FF00000000000000163400800000000000001624000000000000000","0x01010178030162400000000000000001613FF000000000000001634008000000000000","0x0301017800030162400000000000000001613FF000000000000001634008000000000000","0x0101017803016240000000000000000163400800000000000001613FF0000000000000","0x030101780003016240000000000000000163400800000000000001613FF0000000000000","0x01010178030163400800000000000001613FF000000000000001624000000000000000","0x0301017800030163400800000000000001613FF000000000000001624000000000000000","0x0101017803016340080000000000000162400000000000000001613FF0000000000000","0x030101780003016340080000000000000162400000000000000001613FF0000000000000"]}
+{"tensor":{"type":"tensor(x{},y{})","cells":[]},"binary":["0x01020178017900","0x0302017801790000"]}
+{"tensor":{"type":"tensor(x{},y{})","cells":[{"address":{"x":"bar","y":"a"},"value":21},{"address":{"x":"foo","y":"a"},"value":11}]},"binary":["0x0102017801790203666F6F016140260000000000000362617201614035000000000000","0x030201780179000203666F6F016140260000000000000362617201614035000000000000","0x01020178017902036261720161403500000000000003666F6F01614026000000000000","0x0302017801790002036261720161403500000000000003666F6F01614026000000000000"]}
+{"tensor":{"type":"tensor(x{},y[10])","cells":[]},"binary":["0x030101780101790A00"]}
+{"tensor":{"type":"tensor(x[10],y{})","cells":[]},"binary":["0x030101790101780A00"]}
+{"tensor":{"type":"tensor(x{},y[3])","cells":[{"address":{"x":"a","y":0},"value":11},{"address":{"x":"a","y":1},"value":12},{"address":{"x":"a","y":2},"value":13},{"address":{"x":"b","y":0},"value":21},{"address":{"x":"b","y":1},"value":22},{"address":{"x":"b","y":2},"value":23}]},"binary":["0x030101780101790302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000","0x0301017801017903020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000"]}
+{"tensor":{"type":"tensor(x[3],y{})","cells":[{"address":{"x":0,"y":"a"},"value":11},{"address":{"x":0,"y":"b"},"value":21},{"address":{"x":1,"y":"a"},"value":12},{"address":{"x":1,"y":"b"},"value":22},{"address":{"x":2,"y":"a"},"value":13},{"address":{"x":2,"y":"b"},"value":23}]},"binary":["0x030101790101780302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000","0x0301017901017803020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000"]}
+{"num_tests":12}
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index 367dca33515..d1163fb579d 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -14,70 +14,18 @@
 #include <vespa/eval/tensor/default_tensor_engine.h>
 #include <vespa/eval/eval/value_type.h>
 #include <vespa/eval/eval/value.h>
+#include <vespa/eval/eval/test/test_io.h>
 #include <unistd.h>
 
 #include "generate.h"
 
 using namespace vespalib;
 using namespace vespalib::eval;
+using namespace vespalib::eval::test;
 using namespace vespalib::slime::convenience;
 using slime::JsonFormat;
 using tensor::DefaultTensorEngine;
 
-constexpr size_t CHUNK_SIZE = 16384;
-
-//-----------------------------------------------------------------------------
-
-class StdIn : public Input {
-private:
-    bool _eof = false;
-    SimpleBuffer _input;
-public:
-    ~StdIn() {}
-    Memory obtain() override {
-        if ((_input.get().size == 0) && !_eof) {
-            WritableMemory buf = _input.reserve(CHUNK_SIZE);
-            ssize_t res = read(STDIN_FILENO, buf.data, buf.size);
-            _eof = (res == 0);
-            assert(res >= 0); // fail on stdio read errors
-            _input.commit(res);
-        }
-        return _input.obtain();
-    }
-    Input &evict(size_t bytes) override {
-        _input.evict(bytes);
-        return *this;
-    }
-};
-
-class StdOut : public Output {
-private:
-    SimpleBuffer _output;
-public:
-    ~StdOut() {}
-    WritableMemory reserve(size_t bytes) override {
-        return _output.reserve(bytes);
-    }
-    Output &commit(size_t bytes) override {
-        _output.commit(bytes);
-        Memory buf = _output.obtain();
-        ssize_t res = write(STDOUT_FILENO, buf.data, buf.size);
-        assert(res == ssize_t(buf.size)); // fail on stdout write failures
-        _output.evict(res);
-        return *this;
-    }
-};
-
-void write_compact(const Slime &slime, Output &out) {
-    JsonFormat::encode(slime, out, true);
-    out.reserve(1).data[0] = '\n';
-    out.commit(1);
-}
-
-void write_readable(const Slime &slime, Output &out) {
-    JsonFormat::encode(slime, out, false);
-}
-
 //-----------------------------------------------------------------------------
 
 uint8_t unhex(char c) {
@@ -194,14 +142,12 @@ std::vector<vespalib::string> extract_fields(const Inspector &object) {
 
 class MyTestBuilder : public TestBuilder {
 private:
-    Output &_out;
-    size_t  _num_tests;
+    TestWriter _writer;
     void make_test(const vespalib::string &expression,
                    const std::map<vespalib::string,TensorSpec> &input_map,
                    const TensorSpec *expect = nullptr)
     {
-        Slime slime;
-        Cursor &test = slime.setObject();
+        Cursor &test = _writer.create();
         test.setString("expression", expression);
         Cursor &inputs = test.setObject("inputs");
         for (const auto &input: input_map) {
@@ -211,13 +157,11 @@ private:
             insert_value(test.setObject("result"), "expect", *expect);
         } else {
             insert_value(test.setObject("result"), "expect",
-                         eval_expr(slime.get(), SimpleTensorEngine::ref(), false));
+                         eval_expr(test, SimpleTensorEngine::ref(), false));
         }
-        write_compact(slime, _out);
-        ++_num_tests;
     }
 public:
-    MyTestBuilder(Output &out) : _out(out), _num_tests(0) {}
+    MyTestBuilder(Output &out) : _writer(out) {}
     void add(const vespalib::string &expression,
              const std::map<vespalib::string,TensorSpec> &inputs,
              const TensorSpec &expect) override
@@ -229,48 +173,11 @@ public:
     {
         make_test(expression, inputs);
     }
-    void make_summary() {
-        Slime slime;
-        Cursor &summary = slime.setObject();
-        summary.setLong("num_tests", _num_tests);
-        write_compact(slime, _out);
-    }
 };
 
 void generate(Output &out) {
     MyTestBuilder my_test_builder(out);
     Generator::generate(my_test_builder);
-    my_test_builder.make_summary();
-}
-
-//-----------------------------------------------------------------------------
-
-void for_each_test(Input &in,
-                   const std::function<void(Slime&)> &handle_test,
-                   const std::function<void(Slime&)> &handle_summary)
-{
-    size_t num_tests = 0;
-    bool got_summary = false;
-    while (in.obtain().size > 0) {
-        Slime slime;
-        if (JsonFormat::decode(in, slime)) {
-            bool is_test = slime["expression"].valid();
-            bool is_summary = slime["num_tests"].valid();
-            ASSERT_TRUE(is_test != is_summary);
-            if (is_test) {
-                ++num_tests;
-                ASSERT_TRUE(!got_summary);
-                handle_test(slime);
-            } else {
-                got_summary = true;
-                ASSERT_EQUAL(slime["num_tests"].asLong(), int64_t(num_tests));
-                handle_summary(slime);
-            }
-        } else {
-            ASSERT_EQUAL(in.obtain().size, 0u);
-        }
-    }
-    ASSERT_TRUE(got_summary);
 }
 
 //-----------------------------------------------------------------------------
@@ -322,7 +229,7 @@ void verify(Input &in, Output &out) {
                               for (const auto &entry: result_map) {
                                   stats.setLong(entry.first, entry.second);
                               }
-                              write_readable(slime, out);
+                              JsonFormat::encode(slime, out, false);
                           };
     for_each_test(in, handle_test, handle_summary);
 }
diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt
index 2b4edd31655..f27c689e8b6 100644
--- a/eval/src/vespa/eval/eval/test/CMakeLists.txt
+++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt
@@ -3,4 +3,5 @@ vespa_add_library(eval_eval_test OBJECT
     SOURCES
     eval_spec.cpp
     tensor_conformance.cpp
+    test_io.cpp
 )
diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
index 7d84fc3684e..617aa75c945 100644
--- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
+++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
@@ -13,6 +13,7 @@
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/io/mapped_file_input.h>
 #include "tensor_model.hpp"
+#include "test_io.h"
 
 namespace vespalib {
 namespace eval {
@@ -955,20 +956,20 @@ struct TestContext {
         vespalib::string path = module_path;
         path.append("src/apps/make_tensor_binary_format_test_spec/test_spec.json");
         MappedFileInput file(path);
-        Slime slime;
         EXPECT_TRUE(file.valid());
-        EXPECT_TRUE(JsonFormat::decode(file, slime) > 0);
-        int64_t num_tests = slime.get()["num_tests"].asLong();
-        Cursor &tests = slime.get()["tests"];
-        EXPECT_GREATER(num_tests, 0u);
-        EXPECT_EQUAL(size_t(num_tests), tests.entries());
-        for (size_t i = 0; i < tests.entries(); ++i) {
-            size_t fail_cnt = TEST_MASTER.getProgress().failCnt; 
-            TEST_DO(test_binary_format_spec(tests[i]));
-            if (TEST_MASTER.getProgress().failCnt > fail_cnt) {
-                fprintf(stderr, "failed:\n%s", tests[i].toString().c_str());
-            }
-        }
+        auto handle_test = [this](Slime &slime)
+                           {
+                               size_t fail_cnt = TEST_MASTER.getProgress().failCnt;
+                               TEST_DO(test_binary_format_spec(slime.get()));
+                               if (TEST_MASTER.getProgress().failCnt > fail_cnt) {
+                                   fprintf(stderr, "failed:\n%s", slime.get().toString().c_str());
+                               }
+                           };
+        auto handle_summary = [](Slime &slime)
+                              {
+                                  EXPECT_GREATER(slime["num_tests"].asLong(), 0);
+                              };
+        for_each_test(file, handle_test, handle_summary);
     }
 
     void test_binary_format() {
diff --git a/eval/src/vespa/eval/eval/test/test_io.cpp b/eval/src/vespa/eval/eval/test/test_io.cpp
new file mode 100644
index 00000000000..73843ff9aed
--- /dev/null
+++ b/eval/src/vespa/eval/eval/test/test_io.cpp
@@ -0,0 +1,137 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "test_io.h"
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/data/slime/json_format.h>
+#include <unistd.h>
+#include <assert.h>
+
+using vespalib::Memory;
+using vespalib::WritableMemory;
+using vespalib::slime::JsonFormat;
+using vespalib::slime::Cursor;
+
+namespace vespalib::eval::test {
+
+//-----------------------------------------------------------------------------
+
+constexpr size_t CHUNK_SIZE = 16384;
+const char *num_tests_str = "num_tests";
+
+//-----------------------------------------------------------------------------
+
+Memory
+StdIn::obtain()
+{
+    if ((_input.get().size == 0) && !_eof) {
+        WritableMemory buf = _input.reserve(CHUNK_SIZE);
+        ssize_t res = read(STDIN_FILENO, buf.data, buf.size);
+        _eof = (res == 0);
+        assert(res >= 0); // fail on stdio read errors
+        _input.commit(res);
+    }
+    return _input.obtain();
+}
+
+Input &
+StdIn::evict(size_t bytes)
+{
+    _input.evict(bytes);
+    return *this;
+}
+
+//-----------------------------------------------------------------------------
+
+WritableMemory
+StdOut::reserve(size_t bytes)
+{
+    return _output.reserve(bytes);
+}
+
+Output &
+StdOut::commit(size_t bytes)
+{
+    _output.commit(bytes);
+    Memory buf = _output.obtain();
+    ssize_t res = write(STDOUT_FILENO, buf.data, buf.size);
+    assert(res == ssize_t(buf.size)); // fail on stdout write failures
+    _output.evict(res);
+    return *this;
+}
+
+//-----------------------------------------------------------------------------
+
+void write_compact(const Slime &slime, Output &out) {
+    JsonFormat::encode(slime, out, true);
+    out.reserve(1).data[0] = '\n';
+    out.commit(1);
+}
+
+//-----------------------------------------------------------------------------
+
+void
+TestWriter::maybe_write_test()
+{
+    if (_test.get().type().getId() != slime::NIX::ID) {
+        ASSERT_GREATER(_test.get().fields(), 0u);
+        ASSERT_FALSE(_test[num_tests_str].valid());
+        write_compact(_test, _out);
+        ++_num_tests;
+    }
+}
+
+TestWriter::TestWriter(Output &output)
+    : _out(output),
+      _test(),
+      _num_tests(0)
+{
+}
+
+Cursor &
+TestWriter::create()
+{
+    maybe_write_test();
+    _test = Slime();
+    return _test.setObject();
+}
+
+TestWriter::~TestWriter()
+{
+    create().setLong(num_tests_str, _num_tests);
+    write_compact(_test, _out); // summary
+}
+
+//-----------------------------------------------------------------------------
+
+void for_each_test(Input &in,
+                   const std::function<void(Slime&)> &handle_test,
+                   const std::function<void(Slime&)> &handle_summary)
+{
+    size_t num_tests = 0;
+    bool got_summary = false;
+    while (in.obtain().size > 0) {
+        Slime slime;
+        if (JsonFormat::decode(in, slime)) {
+            bool is_summary = slime[num_tests_str].valid();
+            bool is_test = (!is_summary && (slime.get().fields() > 0));
+            ASSERT_TRUE(is_test != is_summary);
+            if (is_test) {
+                ++num_tests;
+                ASSERT_TRUE(!got_summary);
+                handle_test(slime);
+            } else {
+                got_summary = true;
+                ASSERT_EQUAL(slime[num_tests_str].asLong(), int64_t(num_tests));
+                handle_summary(slime);
+            }
+        } else {
+            ASSERT_EQUAL(in.obtain().size, 0u);
+        }
+    }
+    ASSERT_TRUE(got_summary);
+}
+
+//-----------------------------------------------------------------------------
+
+} // namespace vespalib::eval::test
diff --git a/eval/src/vespa/eval/eval/test/test_io.h b/eval/src/vespa/eval/eval/test/test_io.h
new file mode 100644
index 00000000000..e57fa7e68a2
--- /dev/null
+++ b/eval/src/vespa/eval/eval/test/test_io.h
@@ -0,0 +1,75 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/data/memory.h>
+#include <vespa/vespalib/data/writable_memory.h>
+#include <vespa/vespalib/data/input.h>
+#include <vespa/vespalib/data/output.h>
+#include <vespa/vespalib/data/simple_buffer.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <functional>
+
+namespace vespalib::eval::test {
+
+/**
+ * Simple adapter making stdin act as an Input.
+ **/
+class StdIn : public Input {
+private:
+    bool _eof = false;
+    SimpleBuffer _input;
+public:
+    ~StdIn() {}
+    Memory obtain() override;
+    Input &evict(size_t bytes) override;
+};
+
+/**
+ * Simple adapter making stdout act as an Output.
+ **/
+class StdOut : public Output {
+private:
+    SimpleBuffer _output;
+public:
+    ~StdOut() {}
+    WritableMemory reserve(size_t bytes) override;
+    Output &commit(size_t bytes) override;
+};
+
+/**
+ * Write a slime structure as compact json with a trailing newline.
+ **/
+void write_compact(const Slime &slime, Output &out);
+
+/**
+ * Write tests to the given output. Will write a minimal summary when
+ * destructed. The current test will be flushed to the output when a
+ * new test is created or right before writing the summary. The
+ * 'create' function will return an object. A test may be any object
+ * containing at least one field, but a test may not contain the
+ * 'num_tests' field (to avoid confusion with the trailing summary)
+ **/
+class TestWriter {
+private:
+    Output &_out;
+    Slime   _test;
+    size_t  _num_tests;
+    void maybe_write_test();
+public:
+    TestWriter(Output &output);
+    slime::Cursor &create();
+    ~TestWriter();
+};
+
+/**
+ * Reads all tests from 'in' as well as the trailing summary. The
+ * provided 'handle_test' function will be called for each test and
+ * the 'handle_summary' function will be called once at the end. This
+ * function also does some minor consistency checking.
+ **/
+void for_each_test(Input &in,
+                   const std::function<void(Slime&)> &handle_test,
+                   const std::function<void(Slime&)> &handle_summary);
+
+} // namespace vespalib::eval::test