diff options
Diffstat (limited to 'eval/src/apps')
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.cpp | 339 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.h | 7 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 134 |
3 files changed, 185 insertions, 295 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 5feb6b3b3e4..ebd9d61a985 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -2,6 +2,7 @@ #include "generate.h" #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/tensor_model.h> #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/aggr.h> @@ -9,35 +10,102 @@ using namespace vespalib::eval; using namespace vespalib::eval::test; +using vespalib::make_string_short::fmt; + +//----------------------------------------------------------------------------- + +namespace { + +//----------------------------------------------------------------------------- + +const std::vector<vespalib::string> basic_layouts = { + "", + "a3", "a3c5", "a3c5e7", + "b2_1", "b2_1d3_1", "b2_1d3_1f4_1", + "a3b2_1c5d3_1", "b2_1c5d3_1e7" +}; + +const std::vector<std::pair<vespalib::string,vespalib::string>> join_layouts = { + {"", ""}, + {"", "a3"}, + {"", "b2_1"}, + {"", "a3b2_1"}, + {"a3c5e7", "a3c5e7"}, + {"c5", "a3e7"}, + {"a3c5", "c5e7"}, + {"b4_1d6_1f8_1", "b2_2d3_2f4_2"}, + {"d3_1", "b2_1f4_1"}, + {"b2_1d6_1", "d3_2f4_2"}, + {"a3b4_1c5d6_1", "a3b2_1c5d3_1"}, + {"a3b2_1", "c5d3_1"}, + {"a3b4_1c5", "b2_1c5d3_1"} +}; + +//----------------------------------------------------------------------------- + +const std::vector<CellType> just_double = {CellType::DOUBLE}; +const std::vector<CellType> just_float = {CellType::FLOAT}; +const std::vector<CellType> all_types = CellTypeUtils::list_types(); const double my_nan = std::numeric_limits<double>::quiet_NaN(); +Sequence skew(const Sequence &seq) { + return [seq](size_t i) { return seq(i + 7); }; +} + +Sequence my_seq(double x0, double delta, size_t n) { + std::vector<double> values; + double x = x0; + for (size_t i = 0; i < n; ++i) { + values.push_back(x); + x += delta; + } + return Seq(values); +} + +//----------------------------------------------------------------------------- + +void generate(const vespalib::string &expr, const GenSpec &a, TestBuilder &dst) { + auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto a_ct: a_cell_types) { + dst.add(expr, {{"a", a.cpy().cells(a_ct)}}); + } +} + +//----------------------------------------------------------------------------- + +void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, TestBuilder &dst) { + auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + auto b_cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto a_ct: a_cell_types) { + for (auto b_ct: b_cell_types) { + dst.add(expr, {{"a", a.cpy().cells(a_ct)},{"b", b.cpy().cells(b_ct)}}); + } + } +} + //----------------------------------------------------------------------------- void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - TensorSpec input = spec(layout, seq); - for (const Domain &domain: layout) { - vespalib::string expr = vespalib::make_string("reduce(a,%s,%s)", - AggrNames::name_of(aggr)->c_str(), domain.name().c_str()); - dst.add(expr, {{"a", input}}); + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(seq); + for (const auto &dim: a.dims()) { + vespalib::string expr = fmt("reduce(a,%s,%s)", + AggrNames::name_of(aggr)->c_str(), + dim.name().c_str()); + generate(expr, a, dst); + } + if (a.dims().size() > 1) { + vespalib::string expr = fmt("reduce(a,%s,%s,%s)", + AggrNames::name_of(aggr)->c_str(), + a.dims().back().name().c_str(), + a.dims().front().name().c_str()); + generate(expr, a, dst); } { - vespalib::string expr = vespalib::make_string("reduce(a,%s)", AggrNames::name_of(aggr)->c_str()); - dst.add(expr, {{"a", input}}); + vespalib::string expr = fmt("reduce(a,%s)", + AggrNames::name_of(aggr)->c_str()); + generate(expr, a, dst); } } } @@ -48,110 +116,66 @@ void generate_tensor_reduce(TestBuilder &dst) { generate_reduce(Aggr::PROD, SigmoidF(N()), dst); generate_reduce(Aggr::SUM, N(), dst); generate_reduce(Aggr::MAX, N(), dst); - // add MEDIAN cases when supported in Java - // generate_reduce(Aggr::MEDIAN, N(), dst); + generate_reduce(Aggr::MEDIAN, N(), dst); generate_reduce(Aggr::MIN, N(), dst); } //----------------------------------------------------------------------------- -void generate_map_expr(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - dst.add(expr, {{"a", spec(layout, seq)}}, spec(layout, OpSeq(seq, ref_op))); +void generate_map_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(seq); + generate(expr, a, dst); } } -void generate_op1_map(const vespalib::string &op1_expr, map_fun_t ref_op, const Sequence &seq, TestBuilder &dst) { - generate_map_expr(op1_expr, ref_op, seq, dst); - generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), ref_op, seq, dst); +void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, TestBuilder &dst) { + generate_map_expr(op1_expr, seq, dst); + generate_map_expr(fmt("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); } void generate_tensor_map(TestBuilder &dst) { - generate_op1_map("-a", operation::Neg::f, Sub2(Div16(N())), dst); - generate_op1_map("!a", operation::Not::f, Seq({0.0, 1.0, 1.0}), dst); - generate_op1_map("cos(a)", operation::Cos::f, Div16(N()), dst); - generate_op1_map("sin(a)", operation::Sin::f, Div16(N()), dst); - generate_op1_map("tan(a)", operation::Tan::f, Div16(N()), dst); - generate_op1_map("cosh(a)", operation::Cosh::f, Div16(N()), dst); - generate_op1_map("sinh(a)", operation::Sinh::f, Div16(N()), dst); - generate_op1_map("tanh(a)", operation::Tanh::f, Div16(N()), dst); - generate_op1_map("acos(a)", operation::Acos::f, SigmoidF(Div16(N())), dst); - generate_op1_map("asin(a)", operation::Asin::f, SigmoidF(Div16(N())), dst); - generate_op1_map("atan(a)", operation::Atan::f, Div16(N()), dst); - generate_op1_map("exp(a)", operation::Exp::f, Div16(N()), dst); - generate_op1_map("log10(a)", operation::Log10::f, Div16(N()), dst); - generate_op1_map("log(a)", operation::Log::f, Div16(N()), dst); - generate_op1_map("sqrt(a)", operation::Sqrt::f, Div16(N()), dst); - generate_op1_map("ceil(a)", operation::Ceil::f, Div16(N()), dst); - generate_op1_map("fabs(a)", operation::Fabs::f, Div16(N()), dst); - generate_op1_map("floor(a)", operation::Floor::f, Div16(N()), dst); - generate_op1_map("isNan(a)", operation::IsNan::f, Seq({my_nan, 1.0, 1.0}), dst); - generate_op1_map("relu(a)", operation::Relu::f, Sub2(Div16(N())), dst); - generate_op1_map("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())), dst); - generate_op1_map("elu(a)", operation::Elu::f, Sub2(Div16(N())), dst); - // TODO(havardpe): add erf when supported by Java - // generate_op1_map("erf(a)", operation::Erf::f, Sub2(Div16(N())), dst); - generate_op1_map("a in [1,5,7,13,42]", MyIn::f, N(), dst); - generate_map_expr("map(a,f(a)((a+1)*2))", MyOp::f, Div16(N()), dst); + generate_op1_map("-a", Sub2(Div16(N())), dst); + generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst); + generate_op1_map("cos(a)", Div16(N()), dst); + generate_op1_map("sin(a)", Div16(N()), dst); + generate_op1_map("tan(a)", Div16(N()), dst); + generate_op1_map("cosh(a)", Div16(N()), dst); + generate_op1_map("sinh(a)", Div16(N()), dst); + generate_op1_map("tanh(a)", Div16(N()), dst); + generate_op1_map("acos(a)", SigmoidF(Div16(N())), dst); + generate_op1_map("asin(a)", SigmoidF(Div16(N())), dst); + generate_op1_map("atan(a)", Div16(N()), dst); + generate_op1_map("exp(a)", Div16(N()), dst); + generate_op1_map("log10(a)", Div16(N()), dst); + generate_op1_map("log(a)", Div16(N()), dst); + generate_op1_map("sqrt(a)", Div16(N()), dst); + generate_op1_map("ceil(a)", Div16(N()), dst); + generate_op1_map("fabs(a)", Div16(N()), dst); + generate_op1_map("floor(a)", Div16(N()), dst); + generate_op1_map("isNan(a)", Seq({my_nan, 1.0, 1.0}), dst); + generate_op1_map("relu(a)", Sub2(Div16(N())), dst); + generate_op1_map("sigmoid(a)", Sub2(Div16(N())), dst); + generate_op1_map("elu(a)", Sub2(Div16(N())), dst); + generate_op1_map("erf(a)", Sub2(Div16(N())), dst); + generate_op1_map("a in [1,5,7,13,42]", N(), dst); + generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst); } //----------------------------------------------------------------------------- void generate_join_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {}, {}, - {}, {x(5)}, - {x(5)}, {}, - {}, float_cells({x(5)}), - float_cells({x(5)}), {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) - }; - ASSERT_TRUE((layouts.size() % 2) == 0); - for (size_t i = 0; i < layouts.size(); i += 2) { - auto a = spec(layouts[i], seq); - auto b = spec(layouts[i + 1], seq); - dst.add(expr, {{"a", a}, {"b", b}}); + for (const auto &layouts: join_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq); + GenSpec b = GenSpec::from_desc(layouts.second).seq(skew(seq)); + generate(expr, a, b, dst); + generate(expr, b, a, dst); } } void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { generate_join_expr(op2_expr, seq, dst); - generate_join_expr(vespalib::make_string("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); + generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } void generate_tensor_join(TestBuilder &dst) { @@ -160,8 +184,8 @@ void generate_tensor_join(TestBuilder &dst) { generate_op2_join("a*b", Div16(N()), dst); generate_op2_join("a/b", Div16(N()), dst); generate_op2_join("a%b", Div16(N()), dst); - generate_op2_join("a^b", Div16(N()), dst); - generate_op2_join("pow(a,b)", Div16(N()), dst); + generate_op2_join("a^b", my_seq(1.0, 1.0, 5), dst); + generate_op2_join("pow(a,b)", my_seq(1.0, 1.0, 5), dst); generate_op2_join("a==b", Div16(N()), dst); generate_op2_join("a!=b", Div16(N()), dst); generate_op2_join("a~=b", Div16(N()), dst); @@ -182,30 +206,26 @@ void generate_tensor_join(TestBuilder &dst) { //----------------------------------------------------------------------------- void generate_dot_product(TestBuilder &dst, - double expect, const Layout &lhs, const Sequence &lhs_seq, const Layout &rhs, const Sequence &rhs_seq) { - dst.add("reduce(a*b,sum)", - { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }, - spec(expect)); + dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }); } void generate_dot_product(TestBuilder &dst, - double expect, const Layout &layout, const Sequence &lhs_seq, const Sequence &rhs_seq) { auto fl_lay = float_cells(layout); - generate_dot_product(dst, expect, layout, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, expect, fl_lay, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, expect, layout, lhs_seq, fl_lay, rhs_seq); - generate_dot_product(dst, expect, fl_lay, lhs_seq, fl_lay, rhs_seq); + generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq); + generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq); + generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq); + generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq); } void generate_dot_product(TestBuilder &dst) { - generate_dot_product(dst, ((2 * 7) + (3 * 11) + (5 * 13)), {x(3)}, + generate_dot_product(dst, {x(3)}, Seq({ 2, 3, 5 }), Seq({ 7, 11, 13 })); } @@ -215,97 +235,78 @@ void generate_dot_product(TestBuilder &dst) { void generate_xw_product(TestBuilder &dst) { auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 })); auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 })); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", - {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}, - spec(y(3), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,x)", - {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}, - spec(float_cells({y(3)}), Seq({(1*3+2*11),(1*5+2*13),(1*7+2*17)}))); - dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}, - spec(x(2), Seq({(1*3+2*5+3*7),(1*11+2*13+3*17)}))); + dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}); + dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}); + dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}); } //----------------------------------------------------------------------------- void generate_tensor_concat(TestBuilder &dst) { - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}, spec(x(2), Seq({10.0, 20.0}))); - dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}, - spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0}))); - dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}, - spec({x(2),y(4)}, Seq({1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0}))); - dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}, - spec({x(4),y(2)}, Seq({1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 6.0, 6.0}))); - dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}, - spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0}))); - dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}, - spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0}))); - dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}, - spec({x(2), y(2)}, Seq({1.0, 3.0, 2.0, 4.0}))); + dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}); + dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}); + dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}); + dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); + dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); + dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}); dst.add("concat(a,b,x)", - {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}, - spec(float_cells({x(2)}), Seq({10.0, 20.0}))); + {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}); dst.add("concat(a,b,x)", - {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}, - spec(float_cells({x(2)}), Seq({10.0, 20.0}))); + {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}); dst.add("concat(a,b,x)", { {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, {"b", spec(x(2), Seq({4.0, 5.0})) } - }, - spec(x(5), N())); + }); dst.add("concat(a,b,x)", { {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))} - }, - spec(x(5), N())); + }); dst.add("concat(a,b,x)", { {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) } - }, - spec(float_cells({x(5)}), N())); + }); } //----------------------------------------------------------------------------- void generate_tensor_rename(TestBuilder &dst) { - dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}, spec(y(5), N())); - dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({x(5),z(5)}, N())); - dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}, spec({y(5),x(5)}, N())); - dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({z(5),y(5)}, N())); - dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}, spec({x(5),z(5)}, N())); - dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}, spec(float_cells({x(5),z(5)}), N())); - dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}, spec({y(5),x(5)}, N())); + dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}); + dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}); + dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}); + dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}); + dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}); + dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}); + dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}); } //----------------------------------------------------------------------------- void generate_tensor_lambda(TestBuilder &dst) { - dst.add("tensor(x[10])(x+1)", {{}}, spec(x(10), N())); - dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}, spec(float_cells({x(5),y(4)}), N())); - dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}, spec({x(5),y(4)}, N())); - dst.add("tensor(x[5],y[4])(x==y)", {{}}, spec({x(5),y(4)}, - Seq({ 1.0, 0.0, 0.0, 0.0, - 0.0, 1.0, 0.0, 0.0, - 0.0, 0.0, 1.0, 0.0, - 0.0, 0.0, 0.0, 1.0, - 0.0, 0.0, 0.0, 0.0}))); + dst.add("tensor(x[10])(x+1)", {{}}); + dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}); + dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}); + dst.add("tensor(x[5],y[4])(x==y)", {{}}); } //----------------------------------------------------------------------------- +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + void Generator::generate(TestBuilder &dst) { diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index 0f74ce924b3..a71531f7cf3 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -6,12 +6,9 @@ #include <map> struct TestBuilder { + bool full; + TestBuilder(bool full_in) : full(full_in) {} using TensorSpec = vespalib::eval::TensorSpec; - // add test with pre-defined expected result - virtual void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs, - const TensorSpec &expect) = 0; - // add test with undefined expected result virtual void add(const vespalib::string &expression, const std::map<vespalib::string,TensorSpec> &inputs) = 0; virtual ~TestBuilder() {} diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index a62e775abd5..7f09f13a174 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -153,39 +153,23 @@ void print_test(const Inspector &test, OutputWriter &dst) { class MyTestBuilder : public TestBuilder { private: TestWriter _writer; - void make_test(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &input_map, - const TensorSpec *expect = nullptr) +public: + MyTestBuilder(bool full_in, Output &out) : TestBuilder(full_in), _writer(out) {} + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs_in) override { Cursor &test = _writer.create(); test.setString("expression", expression); Cursor &inputs = test.setObject("inputs"); - for (const auto &input: input_map) { - insert_value(inputs, input.first, input.second); + for (const auto [name, spec]: inputs_in) { + insert_value(inputs, name, spec); } - if (expect != nullptr) { - insert_value(test.setObject("result"), "expect", *expect); - } else { - insert_value(test.setObject("result"), "expect", ref_eval(test)); - } - } -public: - MyTestBuilder(Output &out) : _writer(out) {} - void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs, - const TensorSpec &expect) override - { - make_test(expression, inputs, &expect); - } - void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) override - { - make_test(expression, inputs); + insert_value(test.setObject("result"), "expect", ref_eval(test)); } }; -void generate(Output &out) { - MyTestBuilder my_test_builder(out); +void generate(Output &out, bool full) { + MyTestBuilder my_test_builder(full, out); Generator::generate(my_test_builder); } @@ -245,81 +229,6 @@ void verify(Input &in, Output &out) { //----------------------------------------------------------------------------- -struct TestList { - std::vector<Slime> list; - void add_test(Slime &slime) { - list.emplace_back(); - inject(slime.get(), SlimeInserter(list.back())); - } -}; - -struct TestSpec { - vespalib::string expression; - std::vector<TensorSpec> inputs; - TensorSpec result; - TestSpec() : expression(), inputs(), result("error") {} - ~TestSpec(); - void decode(const Inspector &test) { - const auto &my_expression = test["expression"]; - ASSERT_TRUE(my_expression.valid()); - expression = my_expression.asString().make_string(); - auto fun = Function::parse(expression); - ASSERT_TRUE(!fun->has_error()); - ASSERT_EQUAL(fun->num_params(), test["inputs"].fields()); - for (size_t i = 0; i < fun->num_params(); ++i) { - TEST_STATE(make_string("input #%zu", i).c_str()); - const auto &my_input = test["inputs"][fun->param_name(i)]; - ASSERT_TRUE(my_input.valid()); - inputs.push_back(extract_value(my_input)); - } - const auto &my_result = test["result"]["expect"]; - ASSERT_TRUE(my_result.valid()); - result = extract_value(my_result); - } -}; -TestSpec::~TestSpec() = default; - -void compare_test(const Inspector &expect_in, const Inspector &actual_in) { - TestSpec expect; - TestSpec actual; - { - TEST_STATE("decoding expected test case"); - expect.decode(expect_in); - } - { - TEST_STATE("decoding actual test case"); - actual.decode(actual_in); - } - { - TEST_STATE("comparing test cases"); - ASSERT_EQUAL(expect.expression, actual.expression); - ASSERT_EQUAL(expect.inputs.size(), actual.inputs.size()); - for (size_t i = 0; i < expect.inputs.size(); ++i) { - TEST_STATE(make_string("input #%zu", i).c_str()); - ASSERT_EQUAL(expect.inputs[i], actual.inputs[i]); - } - ASSERT_EQUAL(expect.result, actual.result); - } -} - -void compare(Input &expect, Input &actual) { - TestList expect_tests; - TestList actual_tests; - for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &) noexcept {}); - for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &) noexcept {}); - ASSERT_TRUE(!expect_tests.list.empty()); - ASSERT_TRUE(!actual_tests.list.empty()); - ASSERT_EQUAL(expect_tests.list.size(), actual_tests.list.size()); - size_t num_tests = expect_tests.list.size(); - fprintf(stderr, "...found %zu test cases to compare...\n", num_tests); - for (size_t i = 0; i < num_tests; ++i) { - TEST_STATE(make_string("test case #%zu", i).c_str()); - compare_test(expect_tests.list[i].get(), actual_tests.list[i].get()); - } -} - -//----------------------------------------------------------------------------- - void display(Input &in, Output &out) { size_t test_cnt = 0; auto handle_test = [&out,&test_cnt](Slime &slime) @@ -340,7 +249,6 @@ void display(Input &in, Output &out) { int usage(const char *self) { fprintf(stderr, "usage: %s <mode>\n", self); - fprintf(stderr, "usage: %s compare <expect> <actual>\n", self); fprintf(stderr, " <mode>: which mode to activate\n"); fprintf(stderr, " 'generate': write test cases to stdout\n"); fprintf(stderr, " 'evaluate': read test cases from stdin, annotate them with\n"); @@ -350,8 +258,7 @@ int usage(const char *self) { fprintf(stderr, " that all results are as expected\n"); fprintf(stderr, " 'display': read tests from stdin and print them to stdout\n"); fprintf(stderr, " in human-readable form\n"); - fprintf(stderr, " 'compare': read test cases from two separate files and\n"); - fprintf(stderr, " compare them to verify equivalence\n"); + fprintf(stderr, " 'generate-some': write some test cases to stdout\n"); return 1; } @@ -364,30 +271,15 @@ int main(int argc, char **argv) { vespalib::string mode = argv[1]; TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str()); if (mode == "generate") { - generate(std_out); + generate(std_out, true); + } else if (mode == "generate-some") { + generate(std_out, false); } else if (mode == "evaluate") { evaluate(std_in, std_out); } else if (mode == "verify") { verify(std_in, std_out); } else if (mode == "display") { display(std_in, std_out); - } else if (mode == "compare") { - if (argc == 4) { - MappedFileInput expect(argv[2]); - MappedFileInput actual(argv[3]); - if (expect.valid() && actual.valid()) { - compare(expect, actual); - } else { - if (!expect.valid()) { - TEST_ERROR(make_string("could not read file: %s", argv[2]).c_str()); - } - if (!actual.valid()) { - TEST_ERROR(make_string("could not read file: %s", argv[3]).c_str()); - } - } - } else { - TEST_ERROR("wrong number of parameters for 'compare'\n"); - } } else { TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str()); } |