From 806aae9a0c3c1e962539990d53792d7427902f30 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 12 Apr 2021 13:27:34 +0000 Subject: always use reference eval as expected result --- eval/src/apps/tensor_conformance/generate.cpp | 157 +++++++++------------ eval/src/apps/tensor_conformance/generate.h | 5 - .../apps/tensor_conformance/tensor_conformance.cpp | 30 +--- 3 files changed, 72 insertions(+), 120 deletions(-) (limited to 'eval') diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 5feb6b3b3e4..8a81e53b9ec 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -55,7 +55,7 @@ void generate_tensor_reduce(TestBuilder &dst) { //----------------------------------------------------------------------------- -void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) { +void generate_map_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { std::vector layouts = { {}, {x(3)}, @@ -71,42 +71,42 @@ void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Seq float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { - dst.add(expr, {{"a", spec(layout, seq)}}, spec(layout, OpSeq(seq, ref_op))); + dst.add(expr, {{"a", spec(layout, seq)}}); } } -void generate_op1_map(const vespalib::string &op1_expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) { - generate_map_expr(op1_expr, ref_op, seq, dst); - generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), ref_op, seq, dst); +void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, TestBuilder &dst) { + generate_map_expr(op1_expr, seq, dst); + generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); } void generate_tensor_map(TestBuilder &dst) { - generate_op1_map("-a", operation::Neg::f, Sub2(Div16(N())), dst); - generate_op1_map("!a", operation::Not::f, Seq({0.0, 1.0, 1.0}), dst); - generate_op1_map("cos(a)", operation::Cos::f, Div16(N()), dst); - generate_op1_map("sin(a)", operation::Sin::f, Div16(N()), dst); - generate_op1_map("tan(a)", operation::Tan::f, Div16(N()), dst); - generate_op1_map("cosh(a)", operation::Cosh::f, Div16(N()), dst); - generate_op1_map("sinh(a)", operation::Sinh::f, Div16(N()), dst); - generate_op1_map("tanh(a)", operation::Tanh::f, Div16(N()), dst); - generate_op1_map("acos(a)", operation::Acos::f, SigmoidF(Div16(N())), dst); - generate_op1_map("asin(a)", operation::Asin::f, SigmoidF(Div16(N())), dst); - generate_op1_map("atan(a)", operation::Atan::f, Div16(N()), dst); - generate_op1_map("exp(a)", operation::Exp::f, Div16(N()), dst); - generate_op1_map("log10(a)", operation::Log10::f, Div16(N()), dst); - generate_op1_map("log(a)", operation::Log::f, Div16(N()), dst); - generate_op1_map("sqrt(a)", operation::Sqrt::f, Div16(N()), dst); - generate_op1_map("ceil(a)", operation::Ceil::f, Div16(N()), dst); - generate_op1_map("fabs(a)", operation::Fabs::f, Div16(N()), dst); - generate_op1_map("floor(a)", operation::Floor::f, Div16(N()), dst); - generate_op1_map("isNan(a)", operation::IsNan::f, Seq({my_nan, 1.0, 1.0}), dst); - generate_op1_map("relu(a)", operation::Relu::f, Sub2(Div16(N())), dst); - generate_op1_map("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())), dst); - generate_op1_map("elu(a)", operation::Elu::f, Sub2(Div16(N())), dst); + generate_op1_map("-a", Sub2(Div16(N())), dst); + generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst); + generate_op1_map("cos(a)", Div16(N()), dst); + generate_op1_map("sin(a)", Div16(N()), dst); + generate_op1_map("tan(a)", Div16(N()), dst); + generate_op1_map("cosh(a)", Div16(N()), dst); + generate_op1_map("sinh(a)", Div16(N()), dst); + generate_op1_map("tanh(a)", Div16(N()), dst); + generate_op1_map("acos(a)", SigmoidF(Div16(N())), dst); + generate_op1_map("asin(a)", SigmoidF(Div16(N())), dst); + generate_op1_map("atan(a)", Div16(N()), dst); + generate_op1_map("exp(a)", Div16(N()), dst); + generate_op1_map("log10(a)", Div16(N()), dst); + generate_op1_map("log(a)", Div16(N()), dst); + generate_op1_map("sqrt(a)", Div16(N()), dst); + generate_op1_map("ceil(a)", Div16(N()), dst); + generate_op1_map("fabs(a)", Div16(N()), dst); + generate_op1_map("floor(a)", Div16(N()), dst); + generate_op1_map("isNan(a)", Seq({my_nan, 1.0, 1.0}), dst); + generate_op1_map("relu(a)", Sub2(Div16(N())), dst); + generate_op1_map("sigmoid(a)", Sub2(Div16(N())), dst); + generate_op1_map("elu(a)", Sub2(Div16(N())), dst); // TODO(havardpe): add erf when supported by Java // generate_op1_map("erf(a)", operation::Erf::f, Sub2(Div16(N())), dst); - generate_op1_map("a in [1,5,7,13,42]", MyIn::f, N(), dst); - generate_map_expr("map(a,f(a)((a+1)*2))", MyOp::f, Div16(N()), dst); + generate_op1_map("a in [1,5,7,13,42]", N(), dst); + generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst); } //----------------------------------------------------------------------------- @@ -182,30 +182,26 @@ void generate_tensor_join(TestBuilder &dst) { //----------------------------------------------------------------------------- void generate_dot_product(TestBuilder &dst, - double expect, const Layout &lhs, const Sequence &lhs_seq, const Layout &rhs, const Sequence &rhs_seq) { - dst.add("reduce(a*b,sum)", - { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }, - spec(expect)); + dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }); } void generate_dot_product(TestBuilder &dst, - double expect, const Layout &layout, const Sequence &lhs_seq, const Sequence &rhs_seq) { auto fl_lay = float_cells(layout); - generate_dot_product(dst, expect, layout, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, expect, fl_lay, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, expect, layout, lhs_seq, fl_lay, rhs_seq); - generate_dot_product(dst, expect, fl_lay, lhs_seq, fl_lay, rhs_seq); + generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq); + generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq); + generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq); + generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq); } void generate_dot_product(TestBuilder &dst) { - generate_dot_product(dst, ((2 * 7) + (3 * 11) + (5 * 13)), {x(3)}, + generate_dot_product(dst, {x(3)}, Seq({ 2, 3, 5 }), Seq({ 7, 11, 13 })); } @@ -215,93 +211,70 @@ void generate_dot_product(TestBuilder &dst) { void generate_xw_product(TestBuilder &dst) { auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 })); auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 })); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", - {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", - {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}, - spec(float_cells({y(3)}), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}, - spec(x(2), Seq({(1*3+2*5+3*7),(1*11+2*13+3*17)}))); + dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}); + dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}); } //----------------------------------------------------------------------------- void generate_tensor_concat(TestBuilder &dst) { - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}, - spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0}))); - dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}, - spec({x(2),y(4)}, Seq({1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0}))); - dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}, - spec({x(4),y(2)}, Seq({1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 6.0, 6.0}))); - dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}, - spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0}))); - dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}, - spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0}))); - dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}, - spec({x(2), y(2)}, Seq({1.0, 3.0, 2.0, 4.0}))); + dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}); + dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}); + dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}); + dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); + dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}); dst.add("concat(a,b,x)", - {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}, - spec(float_cells({x(2)}), Seq({10.0, 20.0}))); + {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}); dst.add("concat(a,b,x)", - {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}, - spec(float_cells({x(2)}), Seq({10.0, 20.0}))); + {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}); dst.add("concat(a,b,x)", { {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, {"b", spec(x(2), Seq({4.0, 5.0})) } - }, - spec(x(5), N())); + }); dst.add("concat(a,b,x)", { {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))} - }, - spec(x(5), N())); + }); dst.add("concat(a,b,x)", { {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) } - }, - spec(float_cells({x(5)}), N())); + }); } //----------------------------------------------------------------------------- void generate_tensor_rename(TestBuilder &dst) { - dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}, spec(y(5), N())); - dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({x(5),z(5)}, N())); - dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({y(5),x(5)}, N())); - dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({z(5),y(5)}, N())); - dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({x(5),z(5)}, N())); - dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}, spec(float_cells({x(5),z(5)}), N())); - dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}, spec({y(5),x(5)}, N())); + dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}); + dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}); + dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}); + dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}); + dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}); + dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}); + dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}); } //----------------------------------------------------------------------------- void generate_tensor_lambda(TestBuilder &dst) { - dst.add("tensor(x[10])(x+1)", {{}}, spec(x(10), N())); - dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}, spec(float_cells({x(5),y(4)}), N())); - dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}, spec({x(5),y(4)}, N())); - dst.add("tensor(x[5],y[4])(x==y)", {{}}, spec({x(5),y(4)}, - Seq({ 1.0, 0.0, 0.0, 0.0, - 0.0, 1.0, 0.0, 0.0, - 0.0, 0.0, 1.0, 0.0, - 0.0, 0.0, 0.0, 1.0, - 0.0, 0.0, 0.0, 0.0}))); + dst.add("tensor(x[10])(x+1)", {{}}); + dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}); + dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}); + dst.add("tensor(x[5],y[4])(x==y)", {{}}); } //----------------------------------------------------------------------------- diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index 0f74ce924b3..90e218f1d3c 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -7,11 +7,6 @@ struct TestBuilder { using TensorSpec = vespalib::eval::TensorSpec; - // add test with pre-defined expected result - virtual void add(const vespalib::string &expression, - const std::map &inputs, - const TensorSpec &expect) = 0; - // add test with undefined expected result virtual void add(const vespalib::string &expression, const std::map &inputs) = 0; virtual ~TestBuilder() {} diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index d6c93975c9c..04d4c91b579 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -153,34 +153,18 @@ void print_test(const Inspector &test, OutputWriter &dst) { class MyTestBuilder : public TestBuilder { private: TestWriter _writer; - void make_test(const vespalib::string &expression, - const std::map &input_map, - const TensorSpec *expect = nullptr) +public: + MyTestBuilder(Output &out) : _writer(out) {} + void add(const vespalib::string &expression, + const std::map &inputs_in) override { Cursor &test = _writer.create(); test.setString("expression", expression); Cursor &inputs = test.setObject("inputs"); - for (const auto &input: input_map) { - insert_value(inputs, input.first, input.second); + for (const auto [name, spec]: inputs_in) { + insert_value(inputs, name, spec); } - if (expect != nullptr) { - insert_value(test.setObject("result"), "expect", *expect); - } else { - insert_value(test.setObject("result"), "expect", ref_eval(test)); - } - } -public: - MyTestBuilder(Output &out) : _writer(out) {} - void add(const vespalib::string &expression, - const std::map &inputs, - const TensorSpec &expect) override - { - make_test(expression, inputs, &expect); - } - void add(const vespalib::string &expression, - const std::map &inputs) override - { - make_test(expression, inputs); + insert_value(test.setObject("result"), "expect", ref_eval(test)); } }; -- cgit v1.2.3