Merge pull request #17426 from vespa-engine/havardpe/improve-tensor-conformance-testing

Havardpe/improve tensor conformance testing

5 files changed, 189 insertions, 319 deletions

diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp
index 5feb6b3b3e4..ebd9d61a985 100644
--- a/eval/src/apps/tensor_conformance/generate.cpp
+++ b/eval/src/apps/tensor_conformance/generate.cpp
@@ -2,6 +2,7 @@
 
 #include "generate.h"
 #include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/test/tensor_model.h>
 #include <vespa/eval/eval/operation.h>
 #include <vespa/eval/eval/aggr.h>
@@ -9,35 +10,102 @@
 
 using namespace vespalib::eval;
 using namespace vespalib::eval::test;
+using vespalib::make_string_short::fmt;
+
+//-----------------------------------------------------------------------------
+
+namespace {
+
+//-----------------------------------------------------------------------------
+
+const std::vector<vespalib::string> basic_layouts = {
+    "",
+    "a3", "a3c5", "a3c5e7",
+    "b2_1", "b2_1d3_1", "b2_1d3_1f4_1",
+    "a3b2_1c5d3_1", "b2_1c5d3_1e7"
+};
+
+const std::vector<std::pair<vespalib::string,vespalib::string>> join_layouts = {
+    {"", ""},
+    {"", "a3"},
+    {"", "b2_1"},
+    {"", "a3b2_1"},
+    {"a3c5e7", "a3c5e7"},
+    {"c5", "a3e7"},
+    {"a3c5", "c5e7"},
+    {"b4_1d6_1f8_1", "b2_2d3_2f4_2"},
+    {"d3_1", "b2_1f4_1"},
+    {"b2_1d6_1", "d3_2f4_2"},
+    {"a3b4_1c5d6_1", "a3b2_1c5d3_1"},
+    {"a3b2_1", "c5d3_1"},
+    {"a3b4_1c5", "b2_1c5d3_1"}
+};
+
+//-----------------------------------------------------------------------------
+
+const std::vector<CellType> just_double = {CellType::DOUBLE};
+const std::vector<CellType> just_float = {CellType::FLOAT};
+const std::vector<CellType> all_types = CellTypeUtils::list_types();
 
 const double my_nan = std::numeric_limits<double>::quiet_NaN();
 
+Sequence skew(const Sequence &seq) {
+    return [seq](size_t i) { return seq(i + 7); };
+}
+
+Sequence my_seq(double x0, double delta, size_t n) {
+    std::vector<double> values;
+    double x = x0;
+    for (size_t i = 0; i < n; ++i) {
+        values.push_back(x);
+        x += delta;
+    }
+    return Seq(values);
+}
+
+//-----------------------------------------------------------------------------
+
+void generate(const vespalib::string &expr, const GenSpec &a, TestBuilder &dst) {
+    auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float;
+    for (auto a_ct: a_cell_types) {
+        dst.add(expr, {{"a", a.cpy().cells(a_ct)}});
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, TestBuilder &dst) {
+    auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float;
+    auto b_cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float;
+    for (auto a_ct: a_cell_types) {
+        for (auto b_ct: b_cell_types) {
+            dst.add(expr, {{"a", a.cpy().cells(a_ct)},{"b", b.cpy().cells(b_ct)}});
+        }
+    }
+}
+
 //-----------------------------------------------------------------------------
 
 void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) {
-    std::vector<Layout> layouts = {
-        {x(3)},
-        {x(3),y(5)},
-        {x(3),y(5),z(7)},
-        float_cells({x(3),y(5),z(7)}),
-        {x({"a","b","c"})},
-        {x({"a","b","c"}),y({"foo","bar"})},
-        {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})},
-        float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}),
-        {x(3),y({"foo", "bar"}),z(7)},
-        {x({"a","b","c"}),y(5),z({"i","j","k","l"})},
-        float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})})
-    };
-    for (const Layout &layout: layouts) {
-        TensorSpec input = spec(layout, seq);
-        for (const Domain &domain: layout) {
-            vespalib::string expr = vespalib::make_string("reduce(a,%s,%s)",
-                    AggrNames::name_of(aggr)->c_str(), domain.name().c_str());
-            dst.add(expr, {{"a", input}});
+    for (const auto &layout: basic_layouts) {
+        GenSpec a = GenSpec::from_desc(layout).seq(seq);
+        for (const auto &dim: a.dims()) {
+            vespalib::string expr = fmt("reduce(a,%s,%s)",
+                                        AggrNames::name_of(aggr)->c_str(),
+                                        dim.name().c_str());
+            generate(expr, a, dst);
+        }
+        if (a.dims().size() > 1) {
+            vespalib::string expr = fmt("reduce(a,%s,%s,%s)",
+                                        AggrNames::name_of(aggr)->c_str(),
+                                        a.dims().back().name().c_str(),
+                                        a.dims().front().name().c_str());
+            generate(expr, a, dst);
         }
         {
-            vespalib::string expr = vespalib::make_string("reduce(a,%s)", AggrNames::name_of(aggr)->c_str());
-            dst.add(expr, {{"a", input}});
+            vespalib::string expr = fmt("reduce(a,%s)",
+                                        AggrNames::name_of(aggr)->c_str());
+            generate(expr, a, dst);
         }
     }
 }
@@ -48,110 +116,66 @@ void generate_tensor_reduce(TestBuilder &dst) {
     generate_reduce(Aggr::PROD, SigmoidF(N()), dst);
     generate_reduce(Aggr::SUM, N(), dst);
     generate_reduce(Aggr::MAX, N(), dst);
-    // add MEDIAN cases when supported in Java
-    // generate_reduce(Aggr::MEDIAN, N(), dst);
+    generate_reduce(Aggr::MEDIAN, N(), dst);
     generate_reduce(Aggr::MIN, N(), dst);
 }
 
 //-----------------------------------------------------------------------------
 
-void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) {
-    std::vector<Layout> layouts = {
-        {},
-        {x(3)},
-        {x(3),y(5)},
-        {x(3),y(5),z(7)},
-        float_cells({x(3),y(5),z(7)}),
-        {x({"a","b","c"})},
-        {x({"a","b","c"}),y({"foo","bar"})},
-        {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})},
-        float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}),
-        {x(3),y({"foo", "bar"}),z(7)},
-        {x({"a","b","c"}),y(5),z({"i","j","k","l"})},
-        float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})})
-    };
-    for (const Layout &layout: layouts) {
-        dst.add(expr, {{"a", spec(layout, seq)}}, spec(layout, OpSeq(seq, ref_op)));
+void generate_map_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) {
+    for (const auto &layout: basic_layouts) {
+        GenSpec a = GenSpec::from_desc(layout).seq(seq);
+        generate(expr, a, dst);
     }
 }
 
-void generate_op1_map(const vespalib::string &op1_expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) {
-    generate_map_expr(op1_expr, ref_op, seq, dst);
-    generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), ref_op, seq, dst);
+void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, TestBuilder &dst) {
+    generate_map_expr(op1_expr, seq, dst);
+    generate_map_expr(fmt("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst);
 }
 
 void generate_tensor_map(TestBuilder &dst) {
-    generate_op1_map("-a", operation::Neg::f, Sub2(Div16(N())), dst);
-    generate_op1_map("!a", operation::Not::f, Seq({0.0, 1.0, 1.0}), dst);
-    generate_op1_map("cos(a)", operation::Cos::f, Div16(N()), dst);
-    generate_op1_map("sin(a)", operation::Sin::f, Div16(N()), dst);
-    generate_op1_map("tan(a)", operation::Tan::f, Div16(N()), dst);
-    generate_op1_map("cosh(a)", operation::Cosh::f, Div16(N()), dst);
-    generate_op1_map("sinh(a)", operation::Sinh::f, Div16(N()), dst);
-    generate_op1_map("tanh(a)", operation::Tanh::f, Div16(N()), dst);
-    generate_op1_map("acos(a)", operation::Acos::f, SigmoidF(Div16(N())), dst);
-    generate_op1_map("asin(a)", operation::Asin::f, SigmoidF(Div16(N())), dst);
-    generate_op1_map("atan(a)", operation::Atan::f, Div16(N()), dst);
-    generate_op1_map("exp(a)", operation::Exp::f, Div16(N()), dst);
-    generate_op1_map("log10(a)", operation::Log10::f, Div16(N()), dst);
-    generate_op1_map("log(a)", operation::Log::f, Div16(N()), dst);
-    generate_op1_map("sqrt(a)", operation::Sqrt::f, Div16(N()), dst);
-    generate_op1_map("ceil(a)", operation::Ceil::f, Div16(N()), dst);
-    generate_op1_map("fabs(a)", operation::Fabs::f, Div16(N()), dst);
-    generate_op1_map("floor(a)", operation::Floor::f, Div16(N()), dst);
-    generate_op1_map("isNan(a)", operation::IsNan::f, Seq({my_nan, 1.0, 1.0}), dst);
-    generate_op1_map("relu(a)", operation::Relu::f, Sub2(Div16(N())), dst);
-    generate_op1_map("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())), dst);
-    generate_op1_map("elu(a)", operation::Elu::f, Sub2(Div16(N())), dst);
-    // TODO(havardpe): add erf when supported by Java
-    // generate_op1_map("erf(a)", operation::Erf::f, Sub2(Div16(N())), dst);
-    generate_op1_map("a in [1,5,7,13,42]", MyIn::f, N(), dst);
-    generate_map_expr("map(a,f(a)((a+1)*2))", MyOp::f, Div16(N()), dst);
+    generate_op1_map("-a", Sub2(Div16(N())), dst);
+    generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst);
+    generate_op1_map("cos(a)", Div16(N()), dst);
+    generate_op1_map("sin(a)", Div16(N()), dst);
+    generate_op1_map("tan(a)", Div16(N()), dst);
+    generate_op1_map("cosh(a)", Div16(N()), dst);
+    generate_op1_map("sinh(a)", Div16(N()), dst);
+    generate_op1_map("tanh(a)", Div16(N()), dst);
+    generate_op1_map("acos(a)", SigmoidF(Div16(N())), dst);
+    generate_op1_map("asin(a)", SigmoidF(Div16(N())), dst);
+    generate_op1_map("atan(a)", Div16(N()), dst);
+    generate_op1_map("exp(a)", Div16(N()), dst);
+    generate_op1_map("log10(a)", Div16(N()), dst);
+    generate_op1_map("log(a)", Div16(N()), dst);
+    generate_op1_map("sqrt(a)", Div16(N()), dst);
+    generate_op1_map("ceil(a)", Div16(N()), dst);
+    generate_op1_map("fabs(a)", Div16(N()), dst);
+    generate_op1_map("floor(a)", Div16(N()), dst);
+    generate_op1_map("isNan(a)", Seq({my_nan, 1.0, 1.0}), dst);
+    generate_op1_map("relu(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("sigmoid(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("elu(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("erf(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("a in [1,5,7,13,42]", N(), dst);
+    generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst);
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_join_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) {
-    std::vector<Layout> layouts = {
-        {},                                                 {},
-        {},                                                 {x(5)},
-        {x(5)},                                             {},
-        {},                                                 float_cells({x(5)}),
-        float_cells({x(5)}),                                {},
-        {x(5)},                                             {x(5)},
-        {x(5)},                                             {y(5)},
-        {x(5)},                                             {x(5),y(5)},
-        {y(3)},                                             {x(2),z(3)},
-        {x(3),y(5)},                                        {y(5),z(7)},
-        float_cells({x(3),y(5)}),                           {y(5),z(7)},
-        {x(3),y(5)},                                        float_cells({y(5),z(7)}),
-        float_cells({x(3),y(5)}),                           float_cells({y(5),z(7)}),
-        {x({"a","b","c"})},                                 {x({"a","b","c"})},
-        {x({"a","b","c"})},                                 {x({"a","b"})},
-        {x({"a","b","c"})},                                 {y({"foo","bar","baz"})},
-        {x({"a","b","c"})},                                 {x({"a","b","c"}),y({"foo","bar","baz"})},
-        {x({"a","b"}),y({"foo","bar","baz"})},              {x({"a","b","c"}),y({"foo","bar"})},
-        {x({"a","b"}),y({"foo","bar","baz"})},              {y({"foo","bar"}),z({"i","j","k","l"})},
-        float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})},
-        {x({"a","b"}),y({"foo","bar","baz"})},              float_cells({y({"foo","bar"}),z({"i","j","k","l"})}),
-        float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}),
-        {x(3),y({"foo", "bar"})},                           {y({"foo", "bar"}),z(7)},
-        {x({"a","b","c"}),y(5)},                            {y(5),z({"i","j","k","l"})},
-        float_cells({x({"a","b","c"}),y(5)}),               {y(5),z({"i","j","k","l"})},
-        {x({"a","b","c"}),y(5)},                            float_cells({y(5),z({"i","j","k","l"})}),
-        float_cells({x({"a","b","c"}),y(5)}),               float_cells({y(5),z({"i","j","k","l"})})
-    };
-    ASSERT_TRUE((layouts.size() % 2) == 0);
-    for (size_t i = 0; i < layouts.size(); i += 2) {
-        auto a = spec(layouts[i], seq);
-        auto b = spec(layouts[i + 1], seq);
-        dst.add(expr, {{"a", a}, {"b", b}});
+    for (const auto &layouts: join_layouts) {
+        GenSpec a = GenSpec::from_desc(layouts.first).seq(seq);
+        GenSpec b = GenSpec::from_desc(layouts.second).seq(skew(seq));
+        generate(expr, a, b, dst);
+        generate(expr, b, a, dst);
     }
 }
 
 void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) {
     generate_join_expr(op2_expr, seq, dst);
-    generate_join_expr(vespalib::make_string("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst);
+    generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst);
 }
 
 void generate_tensor_join(TestBuilder &dst) {
@@ -160,8 +184,8 @@ void generate_tensor_join(TestBuilder &dst) {
     generate_op2_join("a*b", Div16(N()), dst);
     generate_op2_join("a/b", Div16(N()), dst);
     generate_op2_join("a%b", Div16(N()), dst);
-    generate_op2_join("a^b", Div16(N()), dst);
-    generate_op2_join("pow(a,b)", Div16(N()), dst);
+    generate_op2_join("a^b", my_seq(1.0, 1.0, 5), dst);
+    generate_op2_join("pow(a,b)", my_seq(1.0, 1.0, 5), dst);
     generate_op2_join("a==b", Div16(N()), dst);
     generate_op2_join("a!=b", Div16(N()), dst);
     generate_op2_join("a~=b", Div16(N()), dst);
@@ -182,30 +206,26 @@ void generate_tensor_join(TestBuilder &dst) {
 //-----------------------------------------------------------------------------
 
 void generate_dot_product(TestBuilder &dst,
-                          double expect,
                           const Layout &lhs, const Sequence &lhs_seq,
                           const Layout &rhs, const Sequence &rhs_seq)
 {
-    dst.add("reduce(a*b,sum)",
-            { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} },
-            spec(expect));
+    dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} });
 }
 
 void generate_dot_product(TestBuilder &dst,
-                          double expect,
                           const Layout &layout,
                           const Sequence &lhs_seq,
                           const Sequence &rhs_seq)
 {
     auto fl_lay = float_cells(layout);
-    generate_dot_product(dst, expect, layout, lhs_seq, layout, rhs_seq);
-    generate_dot_product(dst, expect, fl_lay, lhs_seq, layout, rhs_seq);
-    generate_dot_product(dst, expect, layout, lhs_seq, fl_lay, rhs_seq);
-    generate_dot_product(dst, expect, fl_lay, lhs_seq, fl_lay, rhs_seq);
+    generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq);
+    generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq);
+    generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq);
+    generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq);
 }
 
 void generate_dot_product(TestBuilder &dst) {
-    generate_dot_product(dst, ((2 * 7) + (3 * 11) + (5 * 13)), {x(3)},
+    generate_dot_product(dst, {x(3)},
                          Seq({ 2, 3, 5 }),
                          Seq({ 7, 11, 13 }));
 }
@@ -215,97 +235,78 @@ void generate_dot_product(TestBuilder &dst) {
 void generate_xw_product(TestBuilder &dst) {
     auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 }));
     auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 }));
-    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)",
-            {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)",
-            {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}},
-            spec(float_cells({y(3)}), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}},
-            spec(x(2), Seq({(1*3+2*5+3*7),(1*11+2*13+3*17)})));
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}});
+    dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}});
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_concat(TestBuilder &dst) {
-    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}},
-            spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})));
-    dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}},
-            spec({x(2),y(4)}, Seq({1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0})));
-    dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}},
-            spec({x(4),y(2)}, Seq({1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 6.0, 6.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}},
-            spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}},
-            spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0})));
-    dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}},
-            spec({x(2), y(2)}, Seq({1.0, 3.0, 2.0, 4.0})));
+    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}});
+    dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}});
+    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}});
 
     dst.add("concat(a,b,x)",
-            {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }},
-            spec(float_cells({x(2)}), Seq({10.0, 20.0})));
+            {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }});
 
     dst.add("concat(a,b,x)",
-            {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}},
-            spec(float_cells({x(2)}), Seq({10.0, 20.0})));
+            {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}});
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) },
                 {"b", spec(x(2), Seq({4.0, 5.0})) }
-            },
-            spec(x(5), N()));
+            });
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))},
                 {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))}
-            },
-            spec(x(5), N()));
+            });
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) },
                 {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) }
-            },
-            spec(float_cells({x(5)}), N()));
+            });
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_rename(TestBuilder &dst) {
-    dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}, spec(y(5), N()));
-    dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({x(5),z(5)}, N()));
-    dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({y(5),x(5)}, N()));
-    dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({z(5),y(5)}, N()));
-    dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({x(5),z(5)}, N()));
-    dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}, spec(float_cells({x(5),z(5)}), N()));
-    dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}, spec({y(5),x(5)}, N()));
+    dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}});
+    dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}});
+    dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}});
+    dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}});
+    dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}});
+    dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}});
+    dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}});
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_lambda(TestBuilder &dst) {
-    dst.add("tensor(x[10])(x+1)", {{}}, spec(x(10), N()));
-    dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}, spec(float_cells({x(5),y(4)}), N()));
-    dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}, spec({x(5),y(4)}, N()));
-    dst.add("tensor(x[5],y[4])(x==y)", {{}}, spec({x(5),y(4)},
-                                                  Seq({           1.0, 0.0, 0.0, 0.0,
-                                                                  0.0, 1.0, 0.0, 0.0,
-                                                                  0.0, 0.0, 1.0, 0.0,
-                                                                  0.0, 0.0, 0.0, 1.0,
-                                                                  0.0, 0.0, 0.0, 0.0})));
+    dst.add("tensor(x[10])(x+1)", {{}});
+    dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}});
+    dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}});
+    dst.add("tensor(x[5],y[4])(x==y)", {{}});
 }
 
 //-----------------------------------------------------------------------------
 
+} // namespace <unnamed>
+
+//-----------------------------------------------------------------------------
+
 void
 Generator::generate(TestBuilder &dst)
 {
diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h
index 0f74ce924b3..a71531f7cf3 100644
--- a/eval/src/apps/tensor_conformance/generate.h
+++ b/eval/src/apps/tensor_conformance/generate.h
@@ -6,12 +6,9 @@
 #include <map>
 
 struct TestBuilder {
+    bool full;
+    TestBuilder(bool full_in) : full(full_in) {}
     using TensorSpec = vespalib::eval::TensorSpec;
-    // add test with pre-defined expected result
-    virtual void add(const vespalib::string &expression,
-                     const std::map<vespalib::string,TensorSpec> &inputs,
-                     const TensorSpec &expect) = 0;
-    // add test with undefined expected result
     virtual void add(const vespalib::string &expression,
                      const std::map<vespalib::string,TensorSpec> &inputs) = 0;
     virtual ~TestBuilder() {}
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index a62e775abd5..7f09f13a174 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -153,39 +153,23 @@ void print_test(const Inspector &test, OutputWriter &dst) {
 class MyTestBuilder : public TestBuilder {
 private:
     TestWriter _writer;
-    void make_test(const vespalib::string &expression,
-                   const std::map<vespalib::string,TensorSpec> &input_map,
-                   const TensorSpec *expect = nullptr)
+public:
+    MyTestBuilder(bool full_in, Output &out) : TestBuilder(full_in), _writer(out) {}
+    void add(const vespalib::string &expression,
+             const std::map<vespalib::string,TensorSpec> &inputs_in) override
     {
         Cursor &test = _writer.create();
         test.setString("expression", expression);
         Cursor &inputs = test.setObject("inputs");
-        for (const auto &input: input_map) {
-            insert_value(inputs, input.first, input.second);
+        for (const auto [name, spec]: inputs_in) {
+            insert_value(inputs, name, spec);
         }
-        if (expect != nullptr) {
-            insert_value(test.setObject("result"), "expect", *expect);
-        } else {
-            insert_value(test.setObject("result"), "expect", ref_eval(test));
-        }
-    }
-public:
-    MyTestBuilder(Output &out) : _writer(out) {}
-    void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs,
-             const TensorSpec &expect) override
-    {
-        make_test(expression, inputs, &expect);
-    }
-    void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs) override
-    {
-        make_test(expression, inputs);
+        insert_value(test.setObject("result"), "expect", ref_eval(test));
     }
 };
 
-void generate(Output &out) {
-    MyTestBuilder my_test_builder(out);
+void generate(Output &out, bool full) {
+    MyTestBuilder my_test_builder(full, out);
     Generator::generate(my_test_builder);
 }
 
@@ -245,81 +229,6 @@ void verify(Input &in, Output &out) {
 
 //-----------------------------------------------------------------------------
 
-struct TestList {
-    std::vector<Slime> list;
-    void add_test(Slime &slime) {
-        list.emplace_back();
-        inject(slime.get(), SlimeInserter(list.back()));
-    }
-};
-
-struct TestSpec {
-    vespalib::string expression;
-    std::vector<TensorSpec> inputs;
-    TensorSpec result;
-    TestSpec() : expression(), inputs(), result("error") {}
-    ~TestSpec();
-    void decode(const Inspector &test) {
-        const auto &my_expression = test["expression"];
-        ASSERT_TRUE(my_expression.valid());
-        expression = my_expression.asString().make_string();
-        auto fun = Function::parse(expression);
-        ASSERT_TRUE(!fun->has_error());
-        ASSERT_EQUAL(fun->num_params(), test["inputs"].fields());
-        for (size_t i = 0; i < fun->num_params(); ++i) {
-            TEST_STATE(make_string("input #%zu", i).c_str());
-            const auto &my_input = test["inputs"][fun->param_name(i)];
-            ASSERT_TRUE(my_input.valid());
-            inputs.push_back(extract_value(my_input));
-        }
-        const auto &my_result = test["result"]["expect"];
-        ASSERT_TRUE(my_result.valid());
-        result = extract_value(my_result);
-    }
-};
-TestSpec::~TestSpec() = default;
-
-void compare_test(const Inspector &expect_in, const Inspector &actual_in) {
-    TestSpec expect;
-    TestSpec actual;
-    {
-        TEST_STATE("decoding expected test case");
-        expect.decode(expect_in);
-    }
-    {
-        TEST_STATE("decoding actual test case");
-        actual.decode(actual_in);
-    }
-    {
-        TEST_STATE("comparing test cases");
-        ASSERT_EQUAL(expect.expression, actual.expression);
-        ASSERT_EQUAL(expect.inputs.size(), actual.inputs.size());
-        for (size_t i = 0; i < expect.inputs.size(); ++i) {
-            TEST_STATE(make_string("input #%zu", i).c_str());
-            ASSERT_EQUAL(expect.inputs[i], actual.inputs[i]);
-        }
-        ASSERT_EQUAL(expect.result, actual.result);
-    }
-}
-
-void compare(Input &expect, Input &actual) {
-    TestList expect_tests;
-    TestList actual_tests;
-    for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &) noexcept {});
-    for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &) noexcept {});
-    ASSERT_TRUE(!expect_tests.list.empty());
-    ASSERT_TRUE(!actual_tests.list.empty());
-    ASSERT_EQUAL(expect_tests.list.size(), actual_tests.list.size());
-    size_t num_tests = expect_tests.list.size();
-    fprintf(stderr, "...found %zu test cases to compare...\n", num_tests);
-    for (size_t i = 0; i < num_tests; ++i) {
-        TEST_STATE(make_string("test case #%zu", i).c_str());
-        compare_test(expect_tests.list[i].get(), actual_tests.list[i].get());
-    }
-}
-
-//-----------------------------------------------------------------------------
-
 void display(Input &in, Output &out) {
     size_t test_cnt = 0;
     auto handle_test = [&out,&test_cnt](Slime &slime)
@@ -340,7 +249,6 @@ void display(Input &in, Output &out) {
 
 int usage(const char *self) {
     fprintf(stderr, "usage: %s <mode>\n", self);
-    fprintf(stderr, "usage: %s compare <expect> <actual>\n", self);
     fprintf(stderr, "  <mode>: which mode to activate\n");
     fprintf(stderr, "    'generate': write test cases to stdout\n");
     fprintf(stderr, "    'evaluate': read test cases from stdin, annotate them with\n");
@@ -350,8 +258,7 @@ int usage(const char *self) {
     fprintf(stderr, "              that all results are as expected\n");
     fprintf(stderr, "    'display': read tests from stdin and print them to stdout\n");
     fprintf(stderr, "               in human-readable form\n");
-    fprintf(stderr, "    'compare': read test cases from two separate files and\n");
-    fprintf(stderr, "               compare them to verify equivalence\n");
+    fprintf(stderr, "    'generate-some': write some test cases to stdout\n");
     return 1;
 }
 
@@ -364,30 +271,15 @@ int main(int argc, char **argv) {
     vespalib::string mode = argv[1];
     TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str());
     if (mode == "generate") {
-        generate(std_out);
+        generate(std_out, true);
+    } else if (mode == "generate-some") {
+        generate(std_out, false);
     } else if (mode == "evaluate") {
         evaluate(std_in, std_out);
     } else if (mode == "verify") {
         verify(std_in, std_out);
     } else if (mode == "display") {
         display(std_in, std_out);
-    } else if (mode == "compare") {
-        if (argc == 4) {
-            MappedFileInput expect(argv[2]);
-            MappedFileInput actual(argv[3]);
-            if (expect.valid() && actual.valid()) {
-                compare(expect, actual);
-            } else {
-                if (!expect.valid()) {
-                    TEST_ERROR(make_string("could not read file: %s", argv[2]).c_str());
-                }
-                if (!actual.valid()) {
-                    TEST_ERROR(make_string("could not read file: %s", argv[3]).c_str());
-                }
-            }
-        } else {
-            TEST_ERROR("wrong number of parameters for 'compare'\n");
-        }
     } else {
         TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str());
     }
diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
index 3d21f9b4113..e33cc116fba 100644
--- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
+++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
@@ -1,41 +1,21 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/eval/eval/test/tensor_conformance.h>
-#include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/streamed/streamed_value_builder_factory.h>
-#include <vespa/eval/eval/fast_value.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/io/mapped_file_input.h>
 #include <vespa/vespalib/data/slime/slime.h>
 
-using vespalib::eval::SimpleValueBuilderFactory;
-using vespalib::eval::StreamedValueBuilderFactory;
-using vespalib::eval::FastValueBuilderFactory;
-using vespalib::eval::test::TensorConformance;
 using vespalib::make_string_short::fmt;
 using vespalib::Slime;
 using vespalib::slime::JsonFormat;
 using vespalib::MappedFileInput;
 
-vespalib::string module_src_path(TEST_PATH("../../../../"));
 vespalib::string module_build_path("../../../../");
 
-TEST("require that SimpleValue implementation passes all conformance tests") {
-    TEST_DO(TensorConformance::run_tests(module_src_path, SimpleValueBuilderFactory::get()));
-}
-
-TEST("require that StreamedValue implementation passes all conformance tests") {
-    TEST_DO(TensorConformance::run_tests(module_src_path, StreamedValueBuilderFactory::get()));
-}
-
-TEST("require that FastValue implementation passes all conformance tests") {
-    TEST_DO(TensorConformance::run_tests(module_src_path, FastValueBuilderFactory::get()));
-}
-
-TEST("require that cross-language tensor conformance tests pass with C++ expression evaluation") {
+TEST("require that (some) cross-language tensor conformance tests pass with C++ expression evaluation") {
     vespalib::string result_file = "conformance_result.json";
     vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance";
-    EXPECT_EQUAL(system(fmt("%s generate | %s evaluate | %s verify > %s", binary.c_str(), binary.c_str(), binary.c_str(), result_file.c_str()).c_str()), 0);
+    EXPECT_EQUAL(system(fmt("%s generate-some | %s evaluate | %s verify > %s", binary.c_str(), binary.c_str(), binary.c_str(), result_file.c_str()).c_str()), 0);
     Slime result;
     MappedFileInput input(result_file);
     JsonFormat::decode(input, result);
diff --git a/eval/src/vespa/eval/eval/test/gen_spec.h b/eval/src/vespa/eval/eval/test/gen_spec.h
index 3f7550ba644..631c62ecd40 100644
--- a/eval/src/vespa/eval/eval/test/gen_spec.h
+++ b/eval/src/vespa/eval/eval/test/gen_spec.h
@@ -116,7 +116,7 @@ public:
     GenSpec &operator=(GenSpec &&other);
     GenSpec &operator=(const GenSpec &other);
     ~GenSpec();
-    std::vector<DimSpec> dims() const { return _dims; }
+    const std::vector<DimSpec> &dims() const { return _dims; }
     CellType cells() const { return _cells; }
     const seq_t &seq() const { return _seq; }
     GenSpec cpy() const { return *this; }