diff options
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.cpp | 300 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 54 |
2 files changed, 243 insertions, 111 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index ebd9d61a985..7c577ef0086 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/tensor_model.h> -#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/value_type_spec.h> #include <vespa/eval/eval/aggr.h> #include <vespa/vespalib/util/stringfmt.h> @@ -41,6 +41,22 @@ const std::vector<std::pair<vespalib::string,vespalib::string>> join_layouts = { {"a3b4_1c5", "b2_1c5d3_1"} }; +const std::vector<std::pair<vespalib::string,vespalib::string>> merge_layouts = { + {"", ""}, + {"a3c5e7", "a3c5e7"}, + {"b15_2", "b10_3"}, + {"b6_2d4_3f6_2", "b4_3d6_2f4_3"}, + {"a3b6_2c1d4_3e2f6_2", "a3b4_3c1d6_2e2f4_3"}, +}; + +const std::vector<vespalib::string> concat_c_layouts_a = { + "", "c3", "a3", "b6_2", "a3b6_2", "a3b6_2c3" +}; + +const std::vector<vespalib::string> concat_c_layouts_b = { + "", "c5", "a3", "b4_3", "a3b4_3", "a3b4_3c5" +}; + //----------------------------------------------------------------------------- const std::vector<CellType> just_double = {CellType::DOUBLE}; @@ -72,8 +88,6 @@ void generate(const vespalib::string &expr, const GenSpec &a, TestBuilder &dst) } } -//----------------------------------------------------------------------------- - void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, TestBuilder &dst) { auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; auto b_cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; @@ -84,6 +98,52 @@ void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, } } +void generate_with_cell_type(const char *expr_fmt, TestBuilder &dst) { + auto cell_types = dst.full ? all_types : just_float; + for (auto ct: cell_types) { + auto name = value_type::cell_type_to_name(ct); + dst.add(fmt(expr_fmt, name.c_str()), {}); + } +} + +void generate_with_cell_type(const char *expr_fmt, double a, double b, double c, TestBuilder &dst) { + auto cell_types = dst.full ? all_types : just_float; + for (auto ct: cell_types) { + auto name = value_type::cell_type_to_name(ct); + dst.add(fmt(expr_fmt, name.c_str()), {{"a", GenSpec(a)},{"b", GenSpec(b)},{"c", GenSpec(c)}}); + } +} + +//----------------------------------------------------------------------------- + +void generate_const(TestBuilder &dst) { + dst.add("1.25", {}); + dst.add("2.75", {}); + dst.add("\"this is a string that will be hashed\"", {}); + dst.add("\"foo bar baz\"", {}); + // constant tensor lambda + generate_with_cell_type("tensor<%s>(x[10])(x+1)", dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(x*4+(y+1))", dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(x==y)", dst); + // constant verbose tensor create + generate_with_cell_type("tensor<%s>(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dst); + generate_with_cell_type("tensor<%s>(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}", dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}", dst); + // constant convenient tensor create + generate_with_cell_type("tensor<%s>(x[3]):[1,2,3]", dst); + generate_with_cell_type("tensor<%s>(x{}):{a:1,b:2,c:3}", dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{a:[1,2]}", dst); +} + +//----------------------------------------------------------------------------- + +void generate_inject(TestBuilder &dst) { + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(N()); + generate("a", a, dst); + } +} + //----------------------------------------------------------------------------- void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { @@ -110,7 +170,7 @@ void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { } } -void generate_tensor_reduce(TestBuilder &dst) { +void generate_reduce(TestBuilder &dst) { generate_reduce(Aggr::AVG, N(), dst); generate_reduce(Aggr::COUNT, N(), dst); generate_reduce(Aggr::PROD, SigmoidF(N()), dst); @@ -134,7 +194,7 @@ void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, Tes generate_map_expr(fmt("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); } -void generate_tensor_map(TestBuilder &dst) { +void generate_map(TestBuilder &dst) { generate_op1_map("-a", Sub2(Div16(N())), dst); generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst); generate_op1_map("cos(a)", Div16(N()), dst); @@ -159,6 +219,7 @@ void generate_tensor_map(TestBuilder &dst) { generate_op1_map("elu(a)", Sub2(Div16(N())), dst); generate_op1_map("erf(a)", Sub2(Div16(N())), dst); generate_op1_map("a in [1,5,7,13,42]", N(), dst); + // custom lambda generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst); } @@ -178,7 +239,7 @@ void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, Te generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } -void generate_tensor_join(TestBuilder &dst) { +void generate_join(TestBuilder &dst) { generate_op2_join("a+b", Div16(N()), dst); generate_op2_join("a-b", Div16(N()), dst); generate_op2_join("a*b", Div16(N()), dst); @@ -200,105 +261,161 @@ void generate_tensor_join(TestBuilder &dst) { generate_op2_join("fmod(a,b)", Div16(N()), dst); generate_op2_join("min(a,b)", Div16(N()), dst); generate_op2_join("max(a,b)", Div16(N()), dst); + // inverted lambda + generate_join_expr("join(a,b,f(a,b)(b-a))", Div16(N()), dst); + // custom lambda generate_join_expr("join(a,b,f(a,b)((a+b)/(a*b)))", Div16(N()), dst); } //----------------------------------------------------------------------------- -void generate_dot_product(TestBuilder &dst, - const Layout &lhs, const Sequence &lhs_seq, - const Layout &rhs, const Sequence &rhs_seq) -{ - dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }); +void generate_merge_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { + for (const auto &layouts: merge_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq); + GenSpec b = GenSpec::from_desc(layouts.second).seq(skew(seq)); + generate(expr, a, b, dst); + generate(expr, b, a, dst); + } } -void generate_dot_product(TestBuilder &dst, - const Layout &layout, - const Sequence &lhs_seq, - const Sequence &rhs_seq) -{ - auto fl_lay = float_cells(layout); - generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq); - generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq); +void generate_op2_merge(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { + generate_merge_expr(op2_expr, seq, dst); + generate_merge_expr(fmt("merge(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } -void generate_dot_product(TestBuilder &dst) { - generate_dot_product(dst, {x(3)}, - Seq({ 2, 3, 5 }), - Seq({ 7, 11, 13 })); +void generate_merge(TestBuilder &dst) { + generate_op2_merge("a+b", Div16(N()), dst); + generate_op2_merge("a-b", Div16(N()), dst); + generate_op2_merge("a*b", Div16(N()), dst); + generate_op2_merge("a/b", Div16(N()), dst); + generate_op2_merge("a%b", Div16(N()), dst); + generate_op2_merge("a^b", my_seq(1.0, 1.0, 5), dst); + generate_op2_merge("pow(a,b)", my_seq(1.0, 1.0, 5), dst); + generate_op2_merge("a==b", Div16(N()), dst); + generate_op2_merge("a!=b", Div16(N()), dst); + generate_op2_merge("a~=b", Div16(N()), dst); + generate_op2_merge("a<b", Div16(N()), dst); + generate_op2_merge("a<=b", Div16(N()), dst); + generate_op2_merge("a>b", Div16(N()), dst); + generate_op2_merge("a>=b", Div16(N()), dst); + generate_op2_merge("a&&b", Seq({0.0, 1.0, 1.0}), dst); + generate_op2_merge("a||b", Seq({0.0, 1.0, 1.0}), dst); + generate_op2_merge("atan2(a,b)", Div16(N()), dst); + generate_op2_merge("ldexp(a,b)", Div16(N()), dst); + generate_op2_merge("fmod(a,b)", Div16(N()), dst); + generate_op2_merge("min(a,b)", Div16(N()), dst); + generate_op2_merge("max(a,b)", Div16(N()), dst); + // inverted lambda + generate_merge_expr("merge(a,b,f(a,b)(b-a))", Div16(N()), dst); + // custom lambda + generate_merge_expr("merge(a,b,f(a,b)((a+b)/(a*b)))", Div16(N()), dst); } //----------------------------------------------------------------------------- -void generate_xw_product(TestBuilder &dst) { - auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 })); - auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 })); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}); - dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}); +void generate_concat(TestBuilder &dst) { + for (const auto &layout_a: concat_c_layouts_a) { + for (const auto &layout_b: concat_c_layouts_b) { + GenSpec a = GenSpec::from_desc(layout_a).seq(N()); + GenSpec b = GenSpec::from_desc(layout_b).seq(skew(N())); + generate("concat(a, b, c)", a, b, dst); + generate("concat(a, b, c)", b, a, dst); + } + } } //----------------------------------------------------------------------------- -void generate_tensor_concat(TestBuilder &dst) { - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}); - dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}); - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}); - dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); - dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}); - - dst.add("concat(a,b,x)", - {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}); - - dst.add("concat(a,b,x)", - {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}); - - dst.add("concat(a,b,x)", - { - {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, - {"b", spec(x(2), Seq({4.0, 5.0})) } - }); - - dst.add("concat(a,b,x)", - { - {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, - {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))} - }); - - dst.add("concat(a,b,x)", - { - {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, - {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) } - }); +void generate_create(TestBuilder &dst) { + generate_with_cell_type("tensor<%s>(x[3]):[a,b,c]", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x{}):{a:a,b:b,c:c}", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{a:[a,b+c]}", 1, 2, 3, dst); } //----------------------------------------------------------------------------- -void generate_tensor_rename(TestBuilder &dst) { - dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}); - dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}); - dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}); - dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}); - dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}); - dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}); - dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}); +void generate_lambda(TestBuilder &dst) { + generate_with_cell_type("tensor<%s>(x[10])(a+b+c+x+1)", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(a+b+c+x*4+(y+1))", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(a+b+c+(x==y))", 1, 2, 3, dst); +} + +//----------------------------------------------------------------------------- + +void generate_cell_cast(TestBuilder &dst) { + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(N()); + auto from_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + auto to_cell_types = a.dims().empty() ? just_double : all_types; + for (auto a_ct: from_cell_types) { + for (auto to_ct: to_cell_types) { + auto name = value_type::cell_type_to_name(to_ct); + dst.add(fmt("cell_cast(a,%s)", name.c_str()), {{"a", a.cpy().cells(a_ct)}}); + } + } + } +} + +//----------------------------------------------------------------------------- + +void generate_peek(TestBuilder &dst) { + GenSpec num(2); + GenSpec dense = GenSpec::from_desc("x3y5z7").seq(N()); + GenSpec sparse = GenSpec::from_desc("x3_1y5_1z7_1").seq(N()); + GenSpec mixed = GenSpec::from_desc("x3_1y5z7").seq(N()); + for (const auto &spec: {dense, sparse, mixed}) { + generate("a{x:1,y:2,z:4}", spec, dst); + generate("a{y:2,z:5}", spec, dst); + generate("a{x:2}", spec, dst); + generate("a{x:1,y:(b),z:(b+2)}", spec, num, dst); + generate("a{y:(b),z:5}", spec, num, dst); + generate("a{x:(b)}", spec, num, dst); + } +} + +//----------------------------------------------------------------------------- + +void generate_rename(TestBuilder &dst) { + GenSpec dense = GenSpec::from_desc("x3y5z7").seq(N()); + GenSpec sparse = GenSpec::from_desc("x3_1y5_1z7_1").seq(N()); + GenSpec mixed = GenSpec::from_desc("x3_1y5z7").seq(N()); + for (const auto &spec: {dense, sparse, mixed}) { + generate("rename(a,x,d)", spec, dst); + generate("rename(a,y,d)", spec, dst); + generate("rename(a,z,d)", spec, dst); + generate("rename(a,(x,z),(z,x))", spec, dst); + } +} + +//----------------------------------------------------------------------------- + +void generate_if(TestBuilder &dst) { + vespalib::string expr = "if(a,b,c)"; + for (const auto &layout: basic_layouts) { + GenSpec b = GenSpec::from_desc(layout).seq(N()); + GenSpec c = GenSpec::from_desc(layout).seq(skew(N())); + auto cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto ct: cell_types) { + dst.add(expr, {{"a", GenSpec(0.0)},{"b", b.cpy().cells(ct)},{"c", c.cpy().cells(ct)}}); + dst.add(expr, {{"a", GenSpec(1.0)},{"b", b.cpy().cells(ct)},{"c", c.cpy().cells(ct)}}); + } + } } //----------------------------------------------------------------------------- -void generate_tensor_lambda(TestBuilder &dst) { - dst.add("tensor(x[10])(x+1)", {{}}); - dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}); - dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}); - dst.add("tensor(x[5],y[4])(x==y)", {{}}); +void generate_products(TestBuilder &dst) { + auto z1 = GenSpec(1).from_desc("z7"); + auto z2 = GenSpec(7).from_desc("z7"); + auto xz = GenSpec(1).from_desc("x3z7"); + auto yz = GenSpec(3).from_desc("y5z7"); + // dot product + generate("reduce(a*b,sum,z)", z1, z2, dst); + // xw product + generate("reduce(a*b,sum,z)", z1, xz, dst); + generate("reduce(a*b,sum,z)", xz, z2, dst); + // matmul + generate("reduce(a*b,sum,z)", xz, yz, dst); } //----------------------------------------------------------------------------- @@ -310,12 +427,19 @@ void generate_tensor_lambda(TestBuilder &dst) { void Generator::generate(TestBuilder &dst) { - generate_tensor_reduce(dst); - generate_tensor_map(dst); - generate_tensor_join(dst); - generate_dot_product(dst); - generate_xw_product(dst); - generate_tensor_concat(dst); - generate_tensor_rename(dst); - generate_tensor_lambda(dst); + generate_const(dst); + generate_inject(dst); + generate_reduce(dst); + generate_map(dst); + generate_join(dst); + generate_merge(dst); + generate_concat(dst); + generate_create(dst); + generate_lambda(dst); + generate_cell_cast(dst); + generate_peek(dst); + generate_rename(dst); + generate_if(dst); + //-------------------- + generate_products(dst); } diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 7f09f13a174..d305e57ffc5 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/function.h> @@ -87,13 +88,41 @@ TensorSpec extract_value(const Inspector &inspector) { //----------------------------------------------------------------------------- +std::vector<vespalib::string> extract_fields(const Inspector &object) { + struct FieldExtractor : slime::ObjectTraverser { + std::vector<vespalib::string> result; + void field(const Memory &symbol, const Inspector &) override { + result.push_back(symbol.make_string()); + } + } extractor; + object.traverse(extractor); + return std::move(extractor.result); +}; + +//----------------------------------------------------------------------------- + +void dump_test(const Inspector &test) { + fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); + for (const auto &input: extract_fields(test["inputs"])) { + auto value = extract_value(test["inputs"][input]); + fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); + } +} + +//----------------------------------------------------------------------------- + TensorSpec ref_eval(const Inspector &test) { auto fun = Function::parse(test["expression"].asString().make_string()); std::vector<TensorSpec> params; for (size_t i = 0; i < fun->num_params(); ++i) { params.push_back(extract_value(test["inputs"][fun->param_name(i)])); } - return ReferenceEvaluation::eval(*fun, params); + auto result = ReferenceEvaluation::eval(*fun, params); + if (result.type() == "error") { + dump_test(test); + REQUIRE(((void)"reference evaluation failed!", false)); + } + return result; } //----------------------------------------------------------------------------- @@ -119,25 +148,12 @@ TensorSpec eval_expr(const Inspector &test, const ValueBuilderFactory &factory) InterpretedFunction::Context ctx(ifun); SimpleObjectParams params(param_refs); const Value &result = ifun.eval(ctx, params); - ASSERT_EQUAL(result.type(), types.get_type(fun->root())); + REQUIRE_EQ(result.type(), types.get_type(fun->root())); return spec_from_value(result); } //----------------------------------------------------------------------------- -std::vector<vespalib::string> extract_fields(const Inspector &object) { - struct FieldExtractor : slime::ObjectTraverser { - std::vector<vespalib::string> result; - void field(const Memory &symbol, const Inspector &) override { - result.push_back(symbol.make_string()); - } - } extractor; - object.traverse(extractor); - return std::move(extractor.result); -}; - -//----------------------------------------------------------------------------- - void print_test(const Inspector &test, OutputWriter &dst) { dst.printf("expression: '%s'\n", test["expression"].asString().make_string().c_str()); for (const auto &input: extract_fields(test["inputs"])) { @@ -195,14 +211,6 @@ void evaluate(Input &in, Output &out) { //----------------------------------------------------------------------------- -void dump_test(const Inspector &test) { - fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); - for (const auto &input: extract_fields(test["inputs"])) { - auto value = extract_value(test["inputs"][input]); - fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); - } -} - void verify(Input &in, Output &out) { std::map<vespalib::string,size_t> result_map; auto handle_test = [&result_map](Slime &slime) |