diff options
author | HÃ¥vard Pettersen <3535158+havardpe@users.noreply.github.com> | 2021-04-16 12:17:02 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-04-16 12:17:02 +0200 |
commit | 6cf4ddd370e4e52005077e59706e0314cfb8f876 (patch) | |
tree | 8a380c015376a6475dd9659a7030c671cca66751 /eval | |
parent | 7a5ee32b088a37bd4df99c34bc705001037bd015 (diff) | |
parent | 29376f5d74c7fd1bafc6ef6cb64eec2a9ebdfe94 (diff) |
Merge pull request #17452 from vespa-engine/havardpe/extend-tensor-conformance-test
Havardpe/extend tensor conformance test
Diffstat (limited to 'eval')
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.cpp | 300 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 54 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/test/CMakeLists.txt | 1 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/test/tensor_conformance.cpp | 681 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/test/tensor_conformance.h | 18 |
5 files changed, 243 insertions, 811 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index ebd9d61a985..7c577ef0086 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/tensor_model.h> -#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/value_type_spec.h> #include <vespa/eval/eval/aggr.h> #include <vespa/vespalib/util/stringfmt.h> @@ -41,6 +41,22 @@ const std::vector<std::pair<vespalib::string,vespalib::string>> join_layouts = { {"a3b4_1c5", "b2_1c5d3_1"} }; +const std::vector<std::pair<vespalib::string,vespalib::string>> merge_layouts = { + {"", ""}, + {"a3c5e7", "a3c5e7"}, + {"b15_2", "b10_3"}, + {"b6_2d4_3f6_2", "b4_3d6_2f4_3"}, + {"a3b6_2c1d4_3e2f6_2", "a3b4_3c1d6_2e2f4_3"}, +}; + +const std::vector<vespalib::string> concat_c_layouts_a = { + "", "c3", "a3", "b6_2", "a3b6_2", "a3b6_2c3" +}; + +const std::vector<vespalib::string> concat_c_layouts_b = { + "", "c5", "a3", "b4_3", "a3b4_3", "a3b4_3c5" +}; + //----------------------------------------------------------------------------- const std::vector<CellType> just_double = {CellType::DOUBLE}; @@ -72,8 +88,6 @@ void generate(const vespalib::string &expr, const GenSpec &a, TestBuilder &dst) } } -//----------------------------------------------------------------------------- - void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, TestBuilder &dst) { auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; auto b_cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; @@ -84,6 +98,52 @@ void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, } } +void generate_with_cell_type(const char *expr_fmt, TestBuilder &dst) { + auto cell_types = dst.full ? all_types : just_float; + for (auto ct: cell_types) { + auto name = value_type::cell_type_to_name(ct); + dst.add(fmt(expr_fmt, name.c_str()), {}); + } +} + +void generate_with_cell_type(const char *expr_fmt, double a, double b, double c, TestBuilder &dst) { + auto cell_types = dst.full ? all_types : just_float; + for (auto ct: cell_types) { + auto name = value_type::cell_type_to_name(ct); + dst.add(fmt(expr_fmt, name.c_str()), {{"a", GenSpec(a)},{"b", GenSpec(b)},{"c", GenSpec(c)}}); + } +} + +//----------------------------------------------------------------------------- + +void generate_const(TestBuilder &dst) { + dst.add("1.25", {}); + dst.add("2.75", {}); + dst.add("\"this is a string that will be hashed\"", {}); + dst.add("\"foo bar baz\"", {}); + // constant tensor lambda + generate_with_cell_type("tensor<%s>(x[10])(x+1)", dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(x*4+(y+1))", dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(x==y)", dst); + // constant verbose tensor create + generate_with_cell_type("tensor<%s>(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dst); + generate_with_cell_type("tensor<%s>(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}", dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}", dst); + // constant convenient tensor create + generate_with_cell_type("tensor<%s>(x[3]):[1,2,3]", dst); + generate_with_cell_type("tensor<%s>(x{}):{a:1,b:2,c:3}", dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{a:[1,2]}", dst); +} + +//----------------------------------------------------------------------------- + +void generate_inject(TestBuilder &dst) { + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(N()); + generate("a", a, dst); + } +} + //----------------------------------------------------------------------------- void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { @@ -110,7 +170,7 @@ void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { } } -void generate_tensor_reduce(TestBuilder &dst) { +void generate_reduce(TestBuilder &dst) { generate_reduce(Aggr::AVG, N(), dst); generate_reduce(Aggr::COUNT, N(), dst); generate_reduce(Aggr::PROD, SigmoidF(N()), dst); @@ -134,7 +194,7 @@ void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, Tes generate_map_expr(fmt("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); } -void generate_tensor_map(TestBuilder &dst) { +void generate_map(TestBuilder &dst) { generate_op1_map("-a", Sub2(Div16(N())), dst); generate_op1_map("!a", Seq({0.0, 1.0, 1.0}), dst); generate_op1_map("cos(a)", Div16(N()), dst); @@ -159,6 +219,7 @@ void generate_tensor_map(TestBuilder &dst) { generate_op1_map("elu(a)", Sub2(Div16(N())), dst); generate_op1_map("erf(a)", Sub2(Div16(N())), dst); generate_op1_map("a in [1,5,7,13,42]", N(), dst); + // custom lambda generate_map_expr("map(a,f(a)((a+1)*2))", Div16(N()), dst); } @@ -178,7 +239,7 @@ void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, Te generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } -void generate_tensor_join(TestBuilder &dst) { +void generate_join(TestBuilder &dst) { generate_op2_join("a+b", Div16(N()), dst); generate_op2_join("a-b", Div16(N()), dst); generate_op2_join("a*b", Div16(N()), dst); @@ -200,105 +261,161 @@ void generate_tensor_join(TestBuilder &dst) { generate_op2_join("fmod(a,b)", Div16(N()), dst); generate_op2_join("min(a,b)", Div16(N()), dst); generate_op2_join("max(a,b)", Div16(N()), dst); + // inverted lambda + generate_join_expr("join(a,b,f(a,b)(b-a))", Div16(N()), dst); + // custom lambda generate_join_expr("join(a,b,f(a,b)((a+b)/(a*b)))", Div16(N()), dst); } //----------------------------------------------------------------------------- -void generate_dot_product(TestBuilder &dst, - const Layout &lhs, const Sequence &lhs_seq, - const Layout &rhs, const Sequence &rhs_seq) -{ - dst.add("reduce(a*b,sum)", { {"a", spec(lhs, lhs_seq)},{"b", spec(rhs, rhs_seq)} }); +void generate_merge_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { + for (const auto &layouts: merge_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq); + GenSpec b = GenSpec::from_desc(layouts.second).seq(skew(seq)); + generate(expr, a, b, dst); + generate(expr, b, a, dst); + } } -void generate_dot_product(TestBuilder &dst, - const Layout &layout, - const Sequence &lhs_seq, - const Sequence &rhs_seq) -{ - auto fl_lay = float_cells(layout); - generate_dot_product(dst, layout, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, fl_lay, lhs_seq, layout, rhs_seq); - generate_dot_product(dst, layout, lhs_seq, fl_lay, rhs_seq); - generate_dot_product(dst, fl_lay, lhs_seq, fl_lay, rhs_seq); +void generate_op2_merge(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { + generate_merge_expr(op2_expr, seq, dst); + generate_merge_expr(fmt("merge(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } -void generate_dot_product(TestBuilder &dst) { - generate_dot_product(dst, {x(3)}, - Seq({ 2, 3, 5 }), - Seq({ 7, 11, 13 })); +void generate_merge(TestBuilder &dst) { + generate_op2_merge("a+b", Div16(N()), dst); + generate_op2_merge("a-b", Div16(N()), dst); + generate_op2_merge("a*b", Div16(N()), dst); + generate_op2_merge("a/b", Div16(N()), dst); + generate_op2_merge("a%b", Div16(N()), dst); + generate_op2_merge("a^b", my_seq(1.0, 1.0, 5), dst); + generate_op2_merge("pow(a,b)", my_seq(1.0, 1.0, 5), dst); + generate_op2_merge("a==b", Div16(N()), dst); + generate_op2_merge("a!=b", Div16(N()), dst); + generate_op2_merge("a~=b", Div16(N()), dst); + generate_op2_merge("a<b", Div16(N()), dst); + generate_op2_merge("a<=b", Div16(N()), dst); + generate_op2_merge("a>b", Div16(N()), dst); + generate_op2_merge("a>=b", Div16(N()), dst); + generate_op2_merge("a&&b", Seq({0.0, 1.0, 1.0}), dst); + generate_op2_merge("a||b", Seq({0.0, 1.0, 1.0}), dst); + generate_op2_merge("atan2(a,b)", Div16(N()), dst); + generate_op2_merge("ldexp(a,b)", Div16(N()), dst); + generate_op2_merge("fmod(a,b)", Div16(N()), dst); + generate_op2_merge("min(a,b)", Div16(N()), dst); + generate_op2_merge("max(a,b)", Div16(N()), dst); + // inverted lambda + generate_merge_expr("merge(a,b,f(a,b)(b-a))", Div16(N()), dst); + // custom lambda + generate_merge_expr("merge(a,b,f(a,b)((a+b)/(a*b)))", Div16(N()), dst); } //----------------------------------------------------------------------------- -void generate_xw_product(TestBuilder &dst) { - auto matrix = spec({x(2),y(3)}, Seq({ 3, 5, 7, 11, 13, 17 })); - auto fmatrix = spec(float_cells({x(2),y(3)}), Seq({ 3, 5, 7, 11, 13, 17 })); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", matrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", matrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(x(2), Seq({ 1, 2 }))}, {"b", fmatrix}}); - dst.add("reduce(a*b,sum,x)", {{"a", spec(float_cells({x(2)}), Seq({ 1, 2 }))}, {"b", fmatrix}}); - dst.add("reduce(a*b,sum,y)", {{"a", spec(y(3), Seq({ 1, 2, 3 }))}, {"b", matrix}}); +void generate_concat(TestBuilder &dst) { + for (const auto &layout_a: concat_c_layouts_a) { + for (const auto &layout_b: concat_c_layouts_b) { + GenSpec a = GenSpec::from_desc(layout_a).seq(N()); + GenSpec b = GenSpec::from_desc(layout_b).seq(skew(N())); + generate("concat(a, b, c)", a, b, dst); + generate("concat(a, b, c)", b, a, dst); + } + } } //----------------------------------------------------------------------------- -void generate_tensor_concat(TestBuilder &dst) { - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(20.0)}}); - dst.add("concat(a,b,x)", {{"a", spec(x(1), Seq({10.0}))}, {"b", spec(20.0)}}); - dst.add("concat(a,b,x)", {{"a", spec(10.0)}, {"b", spec(x(1), Seq({20.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(x(2), Seq({4.0, 5.0}))}}); - dst.add("concat(a,b,y)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(y(2), Seq({5.0, 6.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}))}, {"b", spec(x(2), Seq({5.0, 6.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(z(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); - dst.add("concat(a,b,x)", {{"a", spec(y(2), Seq({1.0, 2.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}); - dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}); - - dst.add("concat(a,b,x)", - {{"a", spec(float_cells({x(1)}), Seq({10.0}))}, {"b", spec(20.0) }}); - - dst.add("concat(a,b,x)", - {{"a", spec(10.0)}, {"b", spec(float_cells({x(1)}), Seq({20.0}))}}); - - dst.add("concat(a,b,x)", - { - {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, - {"b", spec(x(2), Seq({4.0, 5.0})) } - }); - - dst.add("concat(a,b,x)", - { - {"a", spec(x(3), Seq({1.0, 2.0, 3.0}))}, - {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0}))} - }); - - dst.add("concat(a,b,x)", - { - {"a", spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})) }, - {"b", spec(float_cells({x(2)}), Seq({4.0, 5.0})) } - }); +void generate_create(TestBuilder &dst) { + generate_with_cell_type("tensor<%s>(x[3]):[a,b,c]", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x{}):{a:a,b:b,c:c}", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x{},y[2]):{a:[a,b+c]}", 1, 2, 3, dst); } //----------------------------------------------------------------------------- -void generate_tensor_rename(TestBuilder &dst) { - dst.add("rename(a,x,y)", {{"a", spec(x(5), N())}}); - dst.add("rename(a,y,x)", {{"a", spec({y(5),z(5)}, N())}}); - dst.add("rename(a,z,x)", {{"a", spec({y(5),z(5)}, N())}}); - dst.add("rename(a,x,z)", {{"a", spec({x(5),y(5)}, N())}}); - dst.add("rename(a,y,z)", {{"a", spec({x(5),y(5)}, N())}}); - dst.add("rename(a,y,z)", {{"a", spec(float_cells({x(5),y(5)}), N())}}); - dst.add("rename(a,(x,y),(y,x))", {{"a", spec({x(5),y(5)}, N())}}); +void generate_lambda(TestBuilder &dst) { + generate_with_cell_type("tensor<%s>(x[10])(a+b+c+x+1)", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(a+b+c+x*4+(y+1))", 1, 2, 3, dst); + generate_with_cell_type("tensor<%s>(x[5],y[4])(a+b+c+(x==y))", 1, 2, 3, dst); +} + +//----------------------------------------------------------------------------- + +void generate_cell_cast(TestBuilder &dst) { + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(N()); + auto from_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + auto to_cell_types = a.dims().empty() ? just_double : all_types; + for (auto a_ct: from_cell_types) { + for (auto to_ct: to_cell_types) { + auto name = value_type::cell_type_to_name(to_ct); + dst.add(fmt("cell_cast(a,%s)", name.c_str()), {{"a", a.cpy().cells(a_ct)}}); + } + } + } +} + +//----------------------------------------------------------------------------- + +void generate_peek(TestBuilder &dst) { + GenSpec num(2); + GenSpec dense = GenSpec::from_desc("x3y5z7").seq(N()); + GenSpec sparse = GenSpec::from_desc("x3_1y5_1z7_1").seq(N()); + GenSpec mixed = GenSpec::from_desc("x3_1y5z7").seq(N()); + for (const auto &spec: {dense, sparse, mixed}) { + generate("a{x:1,y:2,z:4}", spec, dst); + generate("a{y:2,z:5}", spec, dst); + generate("a{x:2}", spec, dst); + generate("a{x:1,y:(b),z:(b+2)}", spec, num, dst); + generate("a{y:(b),z:5}", spec, num, dst); + generate("a{x:(b)}", spec, num, dst); + } +} + +//----------------------------------------------------------------------------- + +void generate_rename(TestBuilder &dst) { + GenSpec dense = GenSpec::from_desc("x3y5z7").seq(N()); + GenSpec sparse = GenSpec::from_desc("x3_1y5_1z7_1").seq(N()); + GenSpec mixed = GenSpec::from_desc("x3_1y5z7").seq(N()); + for (const auto &spec: {dense, sparse, mixed}) { + generate("rename(a,x,d)", spec, dst); + generate("rename(a,y,d)", spec, dst); + generate("rename(a,z,d)", spec, dst); + generate("rename(a,(x,z),(z,x))", spec, dst); + } +} + +//----------------------------------------------------------------------------- + +void generate_if(TestBuilder &dst) { + vespalib::string expr = "if(a,b,c)"; + for (const auto &layout: basic_layouts) { + GenSpec b = GenSpec::from_desc(layout).seq(N()); + GenSpec c = GenSpec::from_desc(layout).seq(skew(N())); + auto cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto ct: cell_types) { + dst.add(expr, {{"a", GenSpec(0.0)},{"b", b.cpy().cells(ct)},{"c", c.cpy().cells(ct)}}); + dst.add(expr, {{"a", GenSpec(1.0)},{"b", b.cpy().cells(ct)},{"c", c.cpy().cells(ct)}}); + } + } } //----------------------------------------------------------------------------- -void generate_tensor_lambda(TestBuilder &dst) { - dst.add("tensor(x[10])(x+1)", {{}}); - dst.add("tensor<float>(x[5],y[4])(x*4+(y+1))", {{}}); - dst.add("tensor(x[5],y[4])(x*4+(y+1))", {{}}); - dst.add("tensor(x[5],y[4])(x==y)", {{}}); +void generate_products(TestBuilder &dst) { + auto z1 = GenSpec(1).from_desc("z7"); + auto z2 = GenSpec(7).from_desc("z7"); + auto xz = GenSpec(1).from_desc("x3z7"); + auto yz = GenSpec(3).from_desc("y5z7"); + // dot product + generate("reduce(a*b,sum,z)", z1, z2, dst); + // xw product + generate("reduce(a*b,sum,z)", z1, xz, dst); + generate("reduce(a*b,sum,z)", xz, z2, dst); + // matmul + generate("reduce(a*b,sum,z)", xz, yz, dst); } //----------------------------------------------------------------------------- @@ -310,12 +427,19 @@ void generate_tensor_lambda(TestBuilder &dst) { void Generator::generate(TestBuilder &dst) { - generate_tensor_reduce(dst); - generate_tensor_map(dst); - generate_tensor_join(dst); - generate_dot_product(dst); - generate_xw_product(dst); - generate_tensor_concat(dst); - generate_tensor_rename(dst); - generate_tensor_lambda(dst); + generate_const(dst); + generate_inject(dst); + generate_reduce(dst); + generate_map(dst); + generate_join(dst); + generate_merge(dst); + generate_concat(dst); + generate_create(dst); + generate_lambda(dst); + generate_cell_cast(dst); + generate_peek(dst); + generate_rename(dst); + generate_if(dst); + //-------------------- + generate_products(dst); } diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 2f01f9a7e4d..b5b2e2792a2 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/function.h> @@ -87,13 +88,41 @@ TensorSpec extract_value(const Inspector &inspector) { //----------------------------------------------------------------------------- +std::vector<vespalib::string> extract_fields(const Inspector &object) { + struct FieldExtractor : slime::ObjectTraverser { + std::vector<vespalib::string> result; + void field(const Memory &symbol, const Inspector &) override { + result.push_back(symbol.make_string()); + } + } extractor; + object.traverse(extractor); + return std::move(extractor.result); +}; + +//----------------------------------------------------------------------------- + +void dump_test(const Inspector &test) { + fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); + for (const auto &input: extract_fields(test["inputs"])) { + auto value = extract_value(test["inputs"][input]); + fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); + } +} + +//----------------------------------------------------------------------------- + TensorSpec ref_eval(const Inspector &test) { auto fun = Function::parse(test["expression"].asString().make_string()); std::vector<TensorSpec> params; for (size_t i = 0; i < fun->num_params(); ++i) { params.push_back(extract_value(test["inputs"][fun->param_name(i)])); } - return ReferenceEvaluation::eval(*fun, params); + auto result = ReferenceEvaluation::eval(*fun, params); + if (result.type() == "error") { + dump_test(test); + REQUIRE(((void)"reference evaluation failed!", false)); + } + return result; } //----------------------------------------------------------------------------- @@ -119,25 +148,12 @@ TensorSpec eval_expr(const Inspector &test, const ValueBuilderFactory &factory) InterpretedFunction::Context ctx(ifun); SimpleObjectParams params(param_refs); const Value &result = ifun.eval(ctx, params); - ASSERT_EQUAL(result.type(), types.get_type(fun->root())); + REQUIRE_EQ(result.type(), types.get_type(fun->root())); return spec_from_value(result); } //----------------------------------------------------------------------------- -std::vector<vespalib::string> extract_fields(const Inspector &object) { - struct FieldExtractor : slime::ObjectTraverser { - std::vector<vespalib::string> result; - void field(const Memory &symbol, const Inspector &) override { - result.push_back(symbol.make_string()); - } - } extractor; - object.traverse(extractor); - return std::move(extractor.result); -}; - -//----------------------------------------------------------------------------- - void print_test(const Inspector &test, OutputWriter &dst) { dst.printf("expression: '%s'\n", test["expression"].asString().make_string().c_str()); for (const auto &input: extract_fields(test["inputs"])) { @@ -195,14 +211,6 @@ void evaluate(Input &in, Output &out) { //----------------------------------------------------------------------------- -void dump_test(const Inspector &test) { - fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); - for (const auto &input: extract_fields(test["inputs"])) { - auto value = extract_value(test["inputs"][input]); - fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); - } -} - void verify(Input &in, Output &out) { std::map<vespalib::string,size_t> result_map; auto handle_test = [&result_map](Slime &slime) diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index dc0576f07a2..de628244523 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_library(eval_eval_test OBJECT gen_spec.cpp reference_evaluation.cpp reference_operations.cpp - tensor_conformance.cpp tensor_model.cpp test_io.cpp value_compare.cpp diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp deleted file mode 100644 index c58f8312cbf..00000000000 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ /dev/null @@ -1,681 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor_conformance.h" -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/function.h> -#include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/aggr.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/value_type_spec.h> -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/require.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/io/mapped_file_input.h> -#include "tensor_model.h" -#include "reference_evaluation.h" - -using vespalib::make_string_short::fmt; - -namespace vespalib::eval::test { - -namespace { - -using slime::Cursor; -using slime::Inspector; -using slime::JsonFormat; - -//----------------------------------------------------------------------------- - -TensorSpec ref_eval(const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { - TensorSpec result = ReferenceEvaluation::eval(*Function::parse(expr), params); - EXPECT_FALSE(ValueType::from_spec(result.type()).is_error()); - return result; -} - -TensorSpec eval(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { - auto fun = Function::parse(expr); - std::vector<ValueType> param_types; - std::vector<Value::UP> param_values; - std::vector<Value::CREF> param_refs; - for (const auto ¶m: params) { - param_types.push_back(ValueType::from_spec(param.type())); - param_values.push_back(value_from_spec(param, factory)); - param_refs.emplace_back(*param_values.back()); - } - NodeTypes types(*fun, param_types); - const auto &expect_type = types.get_type(fun->root()); - REQUIRE(!expect_type.is_error()); - InterpretedFunction ifun(factory, *fun, types); - InterpretedFunction::Context ctx(ifun); - const Value &result = ifun.eval(ctx, SimpleObjectParams{param_refs}); - EXPECT_EQUAL(result.type(), expect_type); - return spec_from_value(result); -} - -void verify_result(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms, const TensorSpec &expect) { - auto actual = eval(factory, expr, params); - EXPECT_EQUAL(actual, expect); -} - -void verify_result(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { - TEST_DO(verify_result(factory, expr, params, ref_eval(expr, params))); -} - -//----------------------------------------------------------------------------- - -// NaN value -const double my_nan = std::numeric_limits<double>::quiet_NaN(); - -// Test wrapper to avoid passing global test parameters around -struct TestContext { - - vespalib::string module_path; - const ValueBuilderFactory &factory; - - TestContext(const vespalib::string &module_path_in, const ValueBuilderFactory &factory_in) - : module_path(module_path_in), factory(factory_in) {} - - //------------------------------------------------------------------------- - - void verify_create_type(const vespalib::string &type_spec) { - Value::UP value = value_from_spec(TensorSpec(type_spec), factory); - EXPECT_EQUAL(type_spec, value->type().to_spec()); - } - - void test_tensor_create_type() { - TEST_DO(verify_create_type("double")); - TEST_DO(verify_create_type("tensor(x{})")); - TEST_DO(verify_create_type("tensor(x{},y{})")); - TEST_DO(verify_create_type("tensor<float>(x{},y{})")); - TEST_DO(verify_create_type("tensor(x[5])")); - TEST_DO(verify_create_type("tensor(x[5],y[10])")); - TEST_DO(verify_create_type("tensor<float>(x[5],y[10])")); - TEST_DO(verify_create_type("tensor(x{},y[10])")); - TEST_DO(verify_create_type("tensor(x[5],y{})")); - TEST_DO(verify_create_type("tensor<float>(x[5],y{})")); - } - - //------------------------------------------------------------------------- - - void test_reduce_op(Aggr aggr, const Sequence &seq) { - std::vector<Layout> layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - TensorSpec input = spec(layout, seq); - for (const Domain &domain: layout) { - TEST_STATE(fmt("shape: %s, reduce dimension: %s", - infer_type(layout).c_str(), domain.name().c_str()).c_str()); - vespalib::string expr = fmt("reduce(a,%s,%s)", - AggrNames::name_of(aggr)->c_str(), domain.name().c_str()); - TEST_DO(verify_result(factory, expr, {input})); - } - { - TEST_STATE(fmt("shape: %s, reduce all dimensions", - infer_type(layout).c_str()).c_str()); - vespalib::string expr = fmt("reduce(a,%s)", AggrNames::name_of(aggr)->c_str()); - TEST_DO(verify_result(factory, expr, {input})); - } - } - } - - void test_tensor_reduce() { - TEST_DO(test_reduce_op(Aggr::AVG, N())); - TEST_DO(test_reduce_op(Aggr::COUNT, N())); - TEST_DO(test_reduce_op(Aggr::PROD, SigmoidF(N()))); - TEST_DO(test_reduce_op(Aggr::SUM, N())); - TEST_DO(test_reduce_op(Aggr::MAX, N())); - TEST_DO(test_reduce_op(Aggr::MEDIAN, N())); - TEST_DO(test_reduce_op(Aggr::MIN, N())); - } - - //------------------------------------------------------------------------- - - void test_map_op_inner(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq) { - std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - TEST_DO(verify_result(factory, expr, {spec(layout, seq)}, spec(layout, OpSeq(seq, ref_op)))); - } - } - - void test_map_op(const vespalib::string &expr, map_fun_t op, const Sequence &seq) { - TEST_DO(test_map_op_inner(expr, op, seq)); - TEST_DO(test_map_op_inner(fmt("map(x,f(a)(%s))", expr.c_str()), op, seq)); - } - - void test_tensor_map() { - TEST_DO(test_map_op("-a", operation::Neg::f, Sub2(Div16(N())))); - TEST_DO(test_map_op("!a", operation::Not::f, Seq({0.0, 1.0, 1.0}))); - TEST_DO(test_map_op("cos(a)", operation::Cos::f, Div16(N()))); - TEST_DO(test_map_op("sin(a)", operation::Sin::f, Div16(N()))); - TEST_DO(test_map_op("tan(a)", operation::Tan::f, Div16(N()))); - TEST_DO(test_map_op("cosh(a)", operation::Cosh::f, Div16(N()))); - TEST_DO(test_map_op("sinh(a)", operation::Sinh::f, Div16(N()))); - TEST_DO(test_map_op("tanh(a)", operation::Tanh::f, Div16(N()))); - TEST_DO(test_map_op("acos(a)", operation::Acos::f, SigmoidF(Div16(N())))); - TEST_DO(test_map_op("asin(a)", operation::Asin::f, SigmoidF(Div16(N())))); - TEST_DO(test_map_op("atan(a)", operation::Atan::f, Div16(N()))); - TEST_DO(test_map_op("exp(a)", operation::Exp::f, Div16(N()))); - TEST_DO(test_map_op("log10(a)", operation::Log10::f, Div16(N()))); - TEST_DO(test_map_op("log(a)", operation::Log::f, Div16(N()))); - TEST_DO(test_map_op("sqrt(a)", operation::Sqrt::f, Div16(N()))); - TEST_DO(test_map_op("ceil(a)", operation::Ceil::f, Div16(N()))); - TEST_DO(test_map_op("fabs(a)", operation::Fabs::f, Div16(N()))); - TEST_DO(test_map_op("floor(a)", operation::Floor::f, Div16(N()))); - TEST_DO(test_map_op("isNan(a)", operation::IsNan::f, Seq({my_nan, 1.0, 1.0}))); - TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div16(N())))); - TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())))); - TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div16(N())))); - TEST_DO(test_map_op("erf(a)", operation::Erf::f, Sub2(Div16(N())))); - TEST_DO(test_map_op("a in [1,5,7,13,42]", MyIn::f, N())); - TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div16(N()))); - } - - //------------------------------------------------------------------------- - - void test_apply_op(const vespalib::string &expr, - const TensorSpec &expect, - const TensorSpec &lhs, - const TensorSpec &rhs) { - TEST_DO(verify_result(factory, expr, {lhs, rhs}, expect)); - } - - void test_fixed_sparse_cases_apply_op(const vespalib::string &expr, - join_fun_t op) - { - TEST_DO(test_apply_op(expr, - spec("x{}", {}), - spec("x{}", { { {{"x","1"}}, 3 } }), - spec("x{}", { { {{"x","2"}}, 5 } }))); - TEST_DO(test_apply_op(expr, - spec("x{}", { { {{"x","1"}}, op(3,5) } }), - spec("x{}", { { {{"x","1"}}, 3 } }), - spec("x{}", { { {{"x","1"}}, 5 } }))); - TEST_DO(test_apply_op(expr, - spec("x{}", { { {{"x","1"}}, op(3,-5) } }), - spec("x{}", { { {{"x","1"}}, 3 } }), - spec("x{}", { { {{"x","1"}}, -5 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","-"},{"y","2"},{"z","-"}}, - op(5,7) }, - { {{"x","1"},{"y","-"},{"z","3"}}, - op(3,11) } }), - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, 5 }, - { {{"x","1"},{"y","-"}}, 3 } }), - spec("y{},z{}", - { { {{"y","-"},{"z","3"}}, 11 }, - { {{"y","2"},{"z","-"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","-"},{"y","2"},{"z","-"}}, - op(7,5) }, - { {{"x","1"},{"y","-"},{"z","3"}}, - op(11,3) } }), - spec("y{},z{}", - { { {{"y","-"},{"z","3"}}, 11 }, - { {{"y","2"},{"z","-"}}, 7 } }), - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, 5 }, - { {{"x","1"},{"y","-"}}, 3 } }))); - TEST_DO(test_apply_op(expr, - spec("y{},z{}", - { { {{"y","2"},{"z","-"}}, - op(5,7) } }), - spec("y{}", { { {{"y","2"}}, 5 } }), - spec("y{},z{}", - { { {{"y","-"},{"z","3"}}, 11 }, - { {{"y","2"},{"z","-"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("y{},z{}", - { { {{"y","2"},{"z","-"}}, - op(7,5) } }), - spec("y{},z{}", - { { {{"y","-"},{"z","3"}}, 11 }, - { {{"y","2"},{"z","-"}}, 7 } }), - spec("y{}", { { {{"y","2"}}, 5 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, - op(5,7) } }), - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, 5 }, - { {{"x","1"},{"y","-"}}, 3 } }), - spec("y{}", { { {{"y","2"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, - op(7,5) } }), - spec("y{}", { { {{"y","2"}}, 7 } }), - spec("x{},y{}", - { { {{"x","-"},{"y","2"}}, 5 }, - { {{"x","1"},{"y","-"}}, 3 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},z{}", - { { {{"x","1"},{"z","3"}}, - op(3,11) } }), - spec("x{}", { { {{"x","1"}}, 3 } }), - spec("z{}", { { {{"z","3"}}, 11 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},z{}", - { { {{"x","1"},{"z","3"}}, - op(11,3) } }), - spec("z{}",{ { {{"z","3"}}, 11 } }), - spec("x{}",{ { {{"x","1"}}, 3 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{}", - { { {{"x","1"},{"y","1"}}, - op(3,5) }, - { {{"x","2"},{"y","1"}}, - op(7,5) } }), - spec("x{}", - { { {{"x","1"}}, 3 }, - { {{"x","2"}}, 7 } }), - spec("y{}", - { { {{"y","1"}}, 5 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","1"},{"y","1"},{"z","1"}}, - op(1,7) }, - { {{"x","1"},{"y","1"},{"z","2"}}, - op(1,13) }, - { {{"x","1"},{"y","2"},{"z","1"}}, - op(5,11) }, - { {{"x","2"},{"y","1"},{"z","1"}}, - op(3,7) }, - { {{"x","2"},{"y","1"},{"z","2"}}, - op(3,13) } }), - spec("x{},y{}", - { { {{"x","1"},{"y","1"}}, 1 }, - { {{"x","1"},{"y","2"}}, 5 }, - { {{"x","2"},{"y","1"}}, 3 } }), - spec("y{},z{}", - { { {{"y","1"},{"z","1"}}, 7 }, - { {{"y","1"},{"z","2"}}, 13 }, - { {{"y","2"},{"z","1"}}, 11 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","1"},{"y","1"},{"z","1"}}, - op(1,7) } }), - spec("x{},y{}", - { { {{"x","1"},{"y","-"}}, 5 }, - { {{"x","1"},{"y","1"}}, 1 } }), - spec("y{},z{}", - { { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","1"},{"y","-"},{"z","1"}}, - op(5,11) }, - { {{"x","1"},{"y","1"},{"z","1"}}, - op(1,7) } }), - spec("x{},y{}", - { { {{"x","1"},{"y","-"}}, 5 }, - { {{"x","1"},{"y","1"}}, 1 } }), - spec("y{},z{}", - { { {{"y","-"},{"z","1"}}, 11 }, - { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","1"},{"y","1"},{"z","1"}}, - op(1,7) } }), - spec("x{},y{}", - { { {{"x","-"},{"y","-"}}, 5 }, - { {{"x","1"},{"y","1"}}, 1 } }), - spec("y{},z{}", - { { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(expr, - spec("x{},y{},z{}", - { { {{"x","-"},{"y","-"},{"z", "-"}}, - op(5,11) }, - { {{"x","1"},{"y","1"},{"z","1"}}, - op(1,7) } }), - spec("x{},y{}", - { { {{"x","-"},{"y","-"}}, 5 }, - { {{"x","1"},{"y","1"}}, 1 } }), - spec("y{},z{}", - { { {{"y","-"},{"z","-"}}, 11 }, - { {{"y","1"},{"z","1"}}, 7 } }))); - } - - void test_fixed_dense_cases_apply_op(const vespalib::string &expr, - join_fun_t op) - { - TEST_DO(test_apply_op(expr, - spec(op(0.1,0.2)), spec(0.1), spec(0.2))); - TEST_DO(test_apply_op(expr, - spec(x(1), Seq({ op(3,5) })), - spec(x(1), Seq({ 3 })), - spec(x(1), Seq({ 5 })))); - TEST_DO(test_apply_op(expr, - spec(x(1), Seq({ op(3,-5) })), - spec(x(1), Seq({ 3 })), - spec(x(1), Seq({ -5 })))); - TEST_DO(test_apply_op(expr, - spec(x(2), Seq({ op(3,7), op(5,11) })), - spec(x(2), Seq({ 3, 5 })), - spec(x(2), Seq({ 7, 11 })))); - TEST_DO(test_apply_op(expr, - spec({x(1),y(1)}, Seq({ op(3,5) })), - spec({x(1),y(1)}, Seq({ 3 })), - spec({x(1),y(1)}, Seq({ 5 })))); - TEST_DO(test_apply_op(expr, - spec({x(2),y(2),z(2)}, - Seq({ op(1, 7), op(1, 11), - op(2, 13), op(2, 17), - op(3, 7), op(3, 11), - op(5, 13), op(5, 17) - })), - spec({x(2),y(2)}, - Seq({ 1, 2, - 3, 5 })), - spec({y(2),z(2)}, - Seq({ 7, 11, - 13, 17 })))); - } - - void test_apply_op_inner(const vespalib::string &expr, join_fun_t op, const Sequence &seq) { - std::vector<Layout> layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) - }; - REQUIRE((layouts.size() % 2) == 0); - for (size_t i = 0; i < layouts.size(); i += 2) { - TensorSpec lhs_input = spec(layouts[i], seq); - TensorSpec rhs_input = spec(layouts[i + 1], seq); - TEST_STATE(fmt("lhs shape: %s, rhs shape: %s", - lhs_input.type().c_str(), - rhs_input.type().c_str()).c_str()); - TEST_DO(verify_result(factory, expr, {lhs_input, rhs_input})); - } - TEST_DO(test_fixed_sparse_cases_apply_op(expr, op)); - TEST_DO(test_fixed_dense_cases_apply_op(expr, op)); - } - - void test_apply_op(const vespalib::string &expr, join_fun_t op, const Sequence &seq) { - TEST_DO(test_apply_op_inner(expr, op, seq)); - TEST_DO(test_apply_op_inner(fmt("join(x,y,f(a,b)(%s))", expr.c_str()), op, seq)); - } - - void test_tensor_apply() { - TEST_DO(test_apply_op("a+b", operation::Add::f, Div16(N()))); - TEST_DO(test_apply_op("a-b", operation::Sub::f, Div16(N()))); - TEST_DO(test_apply_op("a*b", operation::Mul::f, Div16(N()))); - TEST_DO(test_apply_op("a/b", operation::Div::f, Div16(N()))); - TEST_DO(test_apply_op("a%b", operation::Mod::f, Div16(N()))); - TEST_DO(test_apply_op("a^b", operation::Pow::f, Div16(N()))); - TEST_DO(test_apply_op("pow(a,b)", operation::Pow::f, Div16(N()))); - TEST_DO(test_apply_op("a==b", operation::Equal::f, Div16(N()))); - TEST_DO(test_apply_op("a!=b", operation::NotEqual::f, Div16(N()))); - TEST_DO(test_apply_op("a~=b", operation::Approx::f, Div16(N()))); - TEST_DO(test_apply_op("a<b", operation::Less::f, Div16(N()))); - TEST_DO(test_apply_op("a<=b", operation::LessEqual::f, Div16(N()))); - TEST_DO(test_apply_op("a>b", operation::Greater::f, Div16(N()))); - TEST_DO(test_apply_op("a>=b", operation::GreaterEqual::f, Div16(N()))); - TEST_DO(test_apply_op("a&&b", operation::And::f, Seq({0.0, 1.0, 1.0}))); - TEST_DO(test_apply_op("a||b", operation::Or::f, Seq({0.0, 1.0, 1.0}))); - TEST_DO(test_apply_op("atan2(a,b)", operation::Atan2::f, Div16(N()))); - TEST_DO(test_apply_op("ldexp(a,b)", operation::Ldexp::f, Div16(N()))); - TEST_DO(test_apply_op("fmod(a,b)", operation::Mod::f, Div16(N()))); - TEST_DO(test_apply_op("min(a,b)", operation::Min::f, Div16(N()))); - TEST_DO(test_apply_op("max(a,b)", operation::Max::f, Div16(N()))); - } - - //------------------------------------------------------------------------- - - void test_dot_product(double expect, - const TensorSpec &lhs, - const TensorSpec &rhs) - { - vespalib::string expr("reduce(a*b,sum)"); - TEST_DO(verify_result(factory, expr, {lhs, rhs}, spec(expect))); - } - - void test_dot_product(double expect, - const Layout &lhs, const Sequence &lhs_seq, - const Layout &rhs, const Sequence &rhs_seq) - { - TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(rhs, rhs_seq))); - TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(rhs, rhs_seq))); - TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(float_cells(rhs), rhs_seq))); - TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(float_cells(rhs), rhs_seq))); - } - - void test_dot_product() { - TEST_DO(test_dot_product(((2 * 7) + (3 * 11) + (5 * 13)), - {x(3)}, Seq({ 2, 3, 5 }), - {x(3)}, Seq({ 7, 11, 13 }))); - } - - //------------------------------------------------------------------------- - - void test_concat(const TensorSpec &a, - const TensorSpec &b, - const vespalib::string &dimension, - const TensorSpec &expect) - { - vespalib::string expr = fmt("concat(a,b,%s)", dimension.c_str()); - TEST_DO(verify_result(factory, expr, {a, b}, expect)); - } - - void test_concat() { - TEST_DO(test_concat(spec(10.0), spec(20.0), "x", spec(x(2), Seq({10.0, 20.0})))); - TEST_DO(test_concat(spec(x(1), Seq({10.0})), spec(20.0), "x", spec(x(2), Seq({10.0, 20.0})))); - TEST_DO(test_concat(spec(10.0), spec(x(1), Seq({20.0})), "x", spec(x(2), Seq({10.0, 20.0})))); - TEST_DO(test_concat(spec(x(3), Seq({1.0, 2.0, 3.0})), spec(x(2), Seq({4.0, 5.0})), "x", - spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); - TEST_DO(test_concat(spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0})), spec(y(2), Seq({5.0, 6.0})), "y", - spec({x(2),y(4)}, Seq({1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0})))); - TEST_DO(test_concat(spec({x(2),y(2)}, Seq({1.0, 2.0, 3.0, 4.0})), spec(x(2), Seq({5.0, 6.0})), "x", - spec({x(4),y(2)}, Seq({1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 6.0, 6.0})))); - TEST_DO(test_concat(spec(z(3), Seq({1.0, 2.0, 3.0})), spec(y(2), Seq({4.0, 5.0})), "x", - spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0})))); - TEST_DO(test_concat(spec(y(2), Seq({1.0, 2.0})), spec(y(2), Seq({4.0, 5.0})), "x", - spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0})))); - - TEST_DO(test_concat(spec(float_cells({x(1)}), Seq({10.0})), spec(20.0), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); - TEST_DO(test_concat(spec(10.0), spec(float_cells({x(1)}), Seq({20.0})), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); - - TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(x(2), Seq({4.0, 5.0})), "x", - spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); - TEST_DO(test_concat(spec(x(3), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", - spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); - TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", - spec(float_cells({x(5)}), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); - } - - //------------------------------------------------------------------------- - - void test_cell_cast(const GenSpec &a) { - for (CellType cell_type: CellTypeUtils::list_types()) { - auto expect = a.cpy().cells(cell_type); - if (expect.bad_scalar()) continue; - vespalib::string expr = fmt("cell_cast(a,%s)", value_type::cell_type_to_name(cell_type).c_str()); - TEST_DO(verify_result(factory, expr, {a}, expect)); - } - } - - void test_cell_cast() { - std::vector<GenSpec> gen_list; - for (CellType cell_type: CellTypeUtils::list_types()) { - gen_list.push_back(GenSpec(-3).cells(cell_type)); - } - TEST_DO(test_cell_cast(GenSpec(42))); - for (const auto &gen: gen_list) { - TEST_DO(test_cell_cast(gen.cpy().idx("x", 10))); - TEST_DO(test_cell_cast(gen.cpy().map("x", 10, 1))); - TEST_DO(test_cell_cast(gen.cpy().map("x", 4, 1).idx("y", 4))); - } - } - - //------------------------------------------------------------------------- - - void test_rename(const vespalib::string &expr, - const TensorSpec &input, - const TensorSpec &expect) - { - TEST_DO(verify_result(factory, expr, {input}, expect)); - } - - void test_rename() { - TEST_DO(test_rename("rename(a,x,y)", spec(x(5), N()), spec(y(5), N()))); - TEST_DO(test_rename("rename(a,y,x)", spec({y(5),z(5)}, N()), spec({x(5),z(5)}, N()))); - TEST_DO(test_rename("rename(a,y,x)", spec(float_cells({y(5),z(5)}), N()), spec(float_cells({x(5),z(5)}), N()))); - TEST_DO(test_rename("rename(a,z,x)", spec({y(5),z(5)}, N()), spec({y(5),x(5)}, N()))); - TEST_DO(test_rename("rename(a,x,z)", spec({x(5),y(5)}, N()), spec({z(5),y(5)}, N()))); - TEST_DO(test_rename("rename(a,y,z)", spec({x(5),y(5)}, N()), spec({x(5),z(5)}, N()))); - TEST_DO(test_rename("rename(a,(x,y),(y,x))", spec({x(5),y(5)}, N()), spec({y(5),x(5)}, N()))); - } - - //------------------------------------------------------------------------- - - void test_tensor_lambda(const vespalib::string &expr, const TensorSpec &expect) { - TEST_DO(verify_result(factory, expr, {}, expect)); - } - - void test_tensor_lambda() { - TEST_DO(test_tensor_lambda("tensor(x[10])(x+1)", spec(x(10), N()))); - TEST_DO(test_tensor_lambda("tensor<float>(x[10])(x+1)", spec(float_cells({x(10)}), N()))); - TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x*4+(y+1))", spec({x(5),y(4)}, N()))); - TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x==y)", spec({x(5),y(4)}, - Seq({ 1.0, 0.0, 0.0, 0.0, - 0.0, 1.0, 0.0, 0.0, - 0.0, 0.0, 1.0, 0.0, - 0.0, 0.0, 0.0, 1.0, - 0.0, 0.0, 0.0, 0.0})))); - } - - //------------------------------------------------------------------------- - - void test_tensor_create(const vespalib::string &expr, double a, double b, const TensorSpec &expect) { - TEST_DO(verify_result(factory, expr, {spec(a), spec(b)}, expect)); - } - - void test_tensor_create() { - TEST_DO(test_tensor_create("tensor(x[3]):{{x:0}:a,{x:1}:b,{x:2}:3}", 1, 2, spec(x(3), N()))); - TEST_DO(test_tensor_create("tensor<float>(x[3]):{{x:0}:a,{x:1}:b,{x:2}:3}", 1, 2, spec(float_cells({x(3)}), N()))); - TEST_DO(test_tensor_create("tensor(x{}):{{x:a}:a,{x:b}:b,{x:c}:3}", 1, 2, spec(x({"a", "b", "c"}), N()))); - TEST_DO(test_tensor_create("tensor(x{},y[2]):{{x:a,y:0}:a,{x:a,y:1}:b}", 1, 2, spec({x({"a"}),y(2)}, N()))); - } - - //------------------------------------------------------------------------- - - void test_tensor_peek(const vespalib::string &expr, const TensorSpec ¶m, const TensorSpec &expect) { - TEST_DO(verify_result(factory, expr, {param, spec(1.0)}, expect)); - } - - void test_tensor_peek() { - auto param_double = spec({x({"0", "1"}),y(2)}, Seq({1.0, 2.0, 3.0, 4.0})); - auto param_float = spec(float_cells({x({"0", "1"}),y(2)}), Seq({1.0, 2.0, 3.0, 4.0})); - TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_double, spec(x(2), Seq({4.0, 1.0})))); - TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_float, spec(x(2), Seq({4.0, 1.0})))); - TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_double, spec(float_cells({x(2)}), Seq({4.0, 1.0})))); - TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_float, spec(float_cells({x(2)}), Seq({4.0, 1.0})))); - TEST_DO(test_tensor_peek("a{x:(b)}", param_double, spec(y(2), Seq({3.0, 4.0})))); - TEST_DO(test_tensor_peek("a{x:(b)}", param_float, spec(float_cells({y(2)}), Seq({3.0, 4.0})))); - TEST_DO(test_tensor_peek("a{y:(b)}", param_double, spec(x({"0", "1"}), Seq({2.0, 4.0})))); - TEST_DO(test_tensor_peek("a{y:(b)}", param_float, spec(float_cells({x({"0", "1"})}), Seq({2.0, 4.0})))); - } - - //------------------------------------------------------------------------- - - void test_tensor_merge(const vespalib::string &type_base, const vespalib::string &a_str, - const vespalib::string &b_str, const vespalib::string &expect_str) - { - vespalib::string expr = "merge(a,b,f(x,y)(2*x+y))"; - for (bool a_float: {false, true}) { - for (bool b_float: {false, true}) { - bool both_float = a_float && b_float; - vespalib::string a_expr = fmt("tensor%s(%s):%s", a_float ? "<float>" : "", type_base.c_str(), a_str.c_str()); - vespalib::string b_expr = fmt("tensor%s(%s):%s", b_float ? "<float>" : "", type_base.c_str(), b_str.c_str()); - vespalib::string expect_expr = fmt("tensor%s(%s):%s", both_float ? "<float>" : "", type_base.c_str(), expect_str.c_str()); - TensorSpec a = spec(a_expr); - TensorSpec b = spec(b_expr); - TensorSpec expect = spec(expect_expr); - TEST_DO(verify_result(factory, expr, {a, b}, expect)); - } - } - } - - void test_tensor_merge() { - TEST_DO(test_tensor_merge("x[3]", "[1,2,3]", "[4,5,6]", "[6,9,12]")); - TEST_DO(test_tensor_merge("x{}", "{a:1,b:2,c:3}", "{b:4,c:5,d:6}", "{a:1,b:8,c:11,d:6}")); - TEST_DO(test_tensor_merge("x{},y[2]", "{a:[1,2],b:[3,4]}", "{b:[5,6],c:[6,7]}", "{a:[1,2],b:[11,14],c:[6,7]}")); - } - - //------------------------------------------------------------------------- - - void run_tests() { - TEST_DO(test_tensor_create_type()); - TEST_DO(test_tensor_reduce()); - TEST_DO(test_tensor_map()); - TEST_DO(test_tensor_apply()); - TEST_DO(test_dot_product()); - TEST_DO(test_concat()); - TEST_DO(test_cell_cast()); - TEST_DO(test_rename()); - TEST_DO(test_tensor_lambda()); - TEST_DO(test_tensor_create()); - TEST_DO(test_tensor_peek()); - TEST_DO(test_tensor_merge()); - } -}; - -} // <unnamed> - -void -TensorConformance::run_tests(const vespalib::string &module_path, const ValueBuilderFactory &factory) -{ - TestContext ctx(module_path, factory); - fprintf(stderr, "module path: '%s'\n", ctx.module_path.c_str()); - ctx.run_tests(); -} - -} // namespace diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.h b/eval/src/vespa/eval/eval/test/tensor_conformance.h deleted file mode 100644 index ad30b60e24c..00000000000 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/value.h> -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib::eval::test { - -/** - * A collection of tensor-related tests that can be run for various - * implementations. - **/ -struct TensorConformance { - static void run_tests(const vespalib::string &module_path, const ValueBuilderFactory &factory); -}; - -} // namespace |