always use reference eval as expected result

3 files changed, 72 insertions, 120 deletions

diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp
index 5feb6b3b3e4..8a81e53b9ec 100644
--- a/eval/src/apps/tensor_conformance/generate.cpp
+++ b/eval/src/apps/tensor_conformance/generate.cpp
@@ -55,7 +55,7 @@ void generate_tensor_reduce(TestBuilder &dst) {
 
 //-----------------------------------------------------------------------------
 
-void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) {
+void generate_map_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) {
     std::vector<Layout> layouts = {
         {},
         {x(3)},
@@ -71,42 +71,42 @@ void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Seq
         float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})})
     };
     for (const Layout &layout: layouts) {
-        dst.add(expr, {{"a", spec(layout, seq)}}, spec(layout, OpSeq(seq, ref_op)));
+        dst.add(expr, {{"a", spec(layout, seq)}});
     }
 }
 
-void generate_op1_map(const vespalib::string &op1_expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) {
-    generate_map_expr(op1_expr, ref_op, seq, dst);
-    generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), ref_op, seq, dst);
+void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, TestBuilder &dst) {
+    generate_map_expr(op1_expr, seq, dst);
+    generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst);
 }
 
 void generate_tensor_map(TestBuilder &dst) {
-    generate_op1_map("-a", operation::Neg::f, Sub2(Div16(N())), dst);
-    generate_op1_map("!a", operation::Not::f, Seq({0.0, 1.0, 1.0}), dst);
-    generate_op1_map("cos(a)", operation::Cos::f, Div16(N()), dst);
-    generate_op1_map("sin(a)", operation::Sin::f, Div16(N()), dst);
-    generate_op1_map("tan(a)", operation::Tan::f, Div16(N()), dst);
-    generate_op1_map("cosh(a)", operation::Cosh::f, Div16(N()), dst);
-    generate_op1_map("sinh(a)", operation::Sinh::f, Div16(N()), dst);
-    generate_op1_map("tanh(a)", operation::Tanh::f, Div16(N()), dst);
-    generate_op1_map("acos(a)", operation::Acos::f, SigmoidF(Div16(N())), dst);
-    generate_op1_map("asin(a)", operation::Asin::f, SigmoidF(Div16(N())), dst);
-    generate_op1_map("atan(a)", operation::Atan::f, Div16(N()), dst);
-    generate_op1_map("exp(a)", operation::Exp::f, Div16(N()), dst);
-    generate_op1_map("log10(a)", operation::Log10::f, Div16(N()), dst);
-    generate_op1_map("log(a)", operation::Log::f, Div16(N()), dst);
-    generate_op1_map("sqrt(a)", operation::Sqrt::f, Div16(N()), dst);
-    generate_op1_map("ceil(a)", operation::Ceil::f, Div16(N()), dst);
-    generate_op1_map("fabs(a)", operation::Fabs::f, Div16(N()), dst);
-    generate_op1_map("floor(a)", operation::Floor::f, Div16(N()), dst);
-    generate_op1_map("isNan(a)", operation::IsNan::f, Seq({my_nan, 1.0, 1.0}), dst);
-    generate_op1_map("relu(a)", operation::Relu::f, Sub2(Div16(N())), dst);
-    generate_op1_map("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())), dst);
-    generate_op1_map("elu(a)", operation::Elu::f, Sub2(Div16(N())), dst);
+    generate_op1_map("-a", Sub2(Div16(N())), dst);
+    generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst);
+    generate_op1_map("cos(a)", Div16(N()), dst);
+    generate_op1_map("sin(a)", Div16(N()), dst);
+    generate_op1_map("tan(a)", Div16(N()), dst);
+    generate_op1_map("cosh(a)", Div16(N()), dst);
+    generate_op1_map("sinh(a)", Div16(N()), dst);
+    generate_op1_map("tanh(a)", Div16(N()), dst);
+    generate_op1_map("acos(a)", SigmoidF(Div16(N())), dst);
+    generate_op1_map("asin(a)", SigmoidF(Div16(N())), dst);
+    generate_op1_map("atan(a)", Div16(N()), dst);
+    generate_op1_map("exp(a)", Div16(N()), dst);
+    generate_op1_map("log10(a)", Div16(N()), dst);
+    generate_op1_map("log(a)", Div16(N()), dst);
+    generate_op1_map("sqrt(a)", Div16(N()), dst);
+    generate_op1_map("ceil(a)", Div16(N()), dst);
+    generate_op1_map("fabs(a)", Div16(N()), dst);
+    generate_op1_map("floor(a)", Div16(N()), dst);
+    generate_op1_map("isNan(a)", Seq({my_nan, 1.0, 1.0}), dst);
+    generate_op1_map("relu(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("sigmoid(a)", Sub2(Div16(N())), dst);
+    generate_op1_map("elu(a)", Sub2(Div16(N())), dst);
     // TODO(havardpe): add erf when supported by Java
     // generate_op1_map("erf(a)", operation::Erf::f, Sub2(Div16(N())), dst);
-    generate_op1_map("a in [1,5,7,13,42]", MyIn::f, N(), dst);
-    generate_map_expr("map(a,f(a)((a+1)*2))", MyOp::f, Div16(N()), dst);
+    generate_op1_map("a in [1,5,7,13,42]", N(), dst);
+    generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst);
 }
 
 //-----------------------------------------------------------------------------
@@ -182,30 +182,26 @@ void generate_tensor_join(TestBuilder &dst) {
 //-----------------------------------------------------------------------------
 
 void generate_dot_product(TestBuilder &dst,
-                          double expect,
                           const Layout &lhs, const Sequence &lhs_seq,
                           const Layout &rhs, const Sequence &rhs_seq)
 {
-    dst.add("reduce(a*b,sum)",
-            { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} },
-            spec(expect));
+    dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} });
 }
 
 void generate_dot_product(TestBuilder &dst,
-                          double expect,
                           const Layout &layout,
                           const Sequence &lhs_seq,
                           const Sequence &rhs_seq)
 {
     auto fl_lay = float_cells(layout);
-    generate_dot_product(dst, expect, layout, lhs_seq, layout, rhs_seq);
-    generate_dot_product(dst, expect, fl_lay, lhs_seq, layout, rhs_seq);
-    generate_dot_product(dst, expect, layout, lhs_seq, fl_lay, rhs_seq);
-    generate_dot_product(dst, expect, fl_lay, lhs_seq, fl_lay, rhs_seq);
+    generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq);
+    generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq);
+    generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq);
+    generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq);
 }
 
 void generate_dot_product(TestBuilder &dst) {
-    generate_dot_product(dst, ((2 * 7) + (3 * 11) + (5 * 13)), {x(3)},
+    generate_dot_product(dst, {x(3)},
                          Seq({ 2, 3, 5 }),
                          Seq({ 7, 11, 13 }));
 }
@@ -215,93 +211,70 @@ void generate_dot_product(TestBuilder &dst) {
 void generate_xw_product(TestBuilder &dst) {
     auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 }));
     auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 }));
-    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)",
-            {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}},
-            spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,x)",
-            {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}},
-            spec(float_cells({y(3)}), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)})));
-    dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}},
-            spec(x(2), Seq({(1*3+2*5+3*7),(1*11+2*13+3*17)})));
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}});
+    dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}});
+    dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}});
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_concat(TestBuilder &dst) {
-    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}, spec(x(2), Seq({10.0, 20.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}},
-            spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})));
-    dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}},
-            spec({x(2),y(4)}, Seq({1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0})));
-    dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}},
-            spec({x(4),y(2)}, Seq({1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 6.0, 6.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}},
-            spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0})));
-    dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}},
-            spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0})));
-    dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}},
-            spec({x(2), y(2)}, Seq({1.0, 3.0, 2.0, 4.0})));
+    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}});
+    dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}});
+    dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}});
+    dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}});
 
     dst.add("concat(a,b,x)",
-            {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }},
-            spec(float_cells({x(2)}), Seq({10.0, 20.0})));
+            {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }});
 
     dst.add("concat(a,b,x)",
-            {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}},
-            spec(float_cells({x(2)}), Seq({10.0, 20.0})));
+            {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}});
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) },
                 {"b", spec(x(2), Seq({4.0, 5.0})) }
-            },
-            spec(x(5), N()));
+            });
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))},
                 {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))}
-            },
-            spec(x(5), N()));
+            });
 
     dst.add("concat(a,b,x)",
             {
                 {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) },
                 {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) }
-            },
-            spec(float_cells({x(5)}), N()));
+            });
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_rename(TestBuilder &dst) {
-    dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}, spec(y(5), N()));
-    dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({x(5),z(5)}, N()));
-    dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({y(5),x(5)}, N()));
-    dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({z(5),y(5)}, N()));
-    dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({x(5),z(5)}, N()));
-    dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}, spec(float_cells({x(5),z(5)}), N()));
-    dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}, spec({y(5),x(5)}, N()));
+    dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}});
+    dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}});
+    dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}});
+    dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}});
+    dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}});
+    dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}});
+    dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}});
 }
 
 //-----------------------------------------------------------------------------
 
 void generate_tensor_lambda(TestBuilder &dst) {
-    dst.add("tensor(x[10])(x+1)", {{}}, spec(x(10), N()));
-    dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}, spec(float_cells({x(5),y(4)}), N()));
-    dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}, spec({x(5),y(4)}, N()));
-    dst.add("tensor(x[5],y[4])(x==y)", {{}}, spec({x(5),y(4)},
-                                                  Seq({           1.0, 0.0, 0.0, 0.0,
-                                                                  0.0, 1.0, 0.0, 0.0,
-                                                                  0.0, 0.0, 1.0, 0.0,
-                                                                  0.0, 0.0, 0.0, 1.0,
-                                                                  0.0, 0.0, 0.0, 0.0})));
+    dst.add("tensor(x[10])(x+1)", {{}});
+    dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}});
+    dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}});
+    dst.add("tensor(x[5],y[4])(x==y)", {{}});
 }
 
 //-----------------------------------------------------------------------------
diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h
index 0f74ce924b3..90e218f1d3c 100644
--- a/eval/src/apps/tensor_conformance/generate.h
+++ b/eval/src/apps/tensor_conformance/generate.h
@@ -7,11 +7,6 @@
 
 struct TestBuilder {
     using TensorSpec = vespalib::eval::TensorSpec;
-    // add test with pre-defined expected result
-    virtual void add(const vespalib::string &expression,
-                     const std::map<vespalib::string,TensorSpec> &inputs,
-                     const TensorSpec &expect) = 0;
-    // add test with undefined expected result
     virtual void add(const vespalib::string &expression,
                      const std::map<vespalib::string,TensorSpec> &inputs) = 0;
     virtual ~TestBuilder() {}
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index d6c93975c9c..04d4c91b579 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -153,34 +153,18 @@ void print_test(const Inspector &test, OutputWriter &dst) {
 class MyTestBuilder : public TestBuilder {
 private:
     TestWriter _writer;
-    void make_test(const vespalib::string &expression,
-                   const std::map<vespalib::string,TensorSpec> &input_map,
-                   const TensorSpec *expect = nullptr)
+public:
+    MyTestBuilder(Output &out) : _writer(out) {}
+    void add(const vespalib::string &expression,
+             const std::map<vespalib::string,TensorSpec> &inputs_in) override
     {
         Cursor &test = _writer.create();
         test.setString("expression", expression);
         Cursor &inputs = test.setObject("inputs");
-        for (const auto &input: input_map) {
-            insert_value(inputs, input.first, input.second);
+        for (const auto [name, spec]: inputs_in) {
+            insert_value(inputs, name, spec);
         }
-        if (expect != nullptr) {
-            insert_value(test.setObject("result"), "expect", *expect);
-        } else {
-            insert_value(test.setObject("result"), "expect", ref_eval(test));
-        }
-    }
-public:
-    MyTestBuilder(Output &out) : _writer(out) {}
-    void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs,
-             const TensorSpec &expect) override
-    {
-        make_test(expression, inputs, &expect);
-    }
-    void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs) override
-    {
-        make_test(expression, inputs);
+        insert_value(test.setObject("result"), "expect", ref_eval(test));
     }
 };