diff options
author | Håvard Pettersen <havardpe@oath.com> | 2021-04-14 12:37:02 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2021-04-14 12:42:29 +0000 |
commit | b922143541bbd26bb534d1ce35fb8fb0de6f1cff (patch) | |
tree | af7f2cb770739845ff249ca4bc7992646fa9f40d /eval/src/apps/tensor_conformance | |
parent | fd3d1ebd87ee376e0995681f713808eb67a0142a (diff) |
use GenSpec to generate more tests for map/reduce/join
- limit the number of cases run in unit test
- make pow tests use very limited input values to avoid rounding
issues.
Diffstat (limited to 'eval/src/apps/tensor_conformance')
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.cpp | 180 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/generate.h | 2 | ||||
-rw-r--r-- | eval/src/apps/tensor_conformance/tensor_conformance.cpp | 11 |
3 files changed, 114 insertions, 79 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index ce7cdb239b2..ebd9d61a985 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -2,6 +2,7 @@ #include "generate.h" #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/tensor_model.h> #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/aggr.h> @@ -9,35 +10,102 @@ using namespace vespalib::eval; using namespace vespalib::eval::test; +using vespalib::make_string_short::fmt; + +//----------------------------------------------------------------------------- + +namespace { + +//----------------------------------------------------------------------------- + +const std::vector<vespalib::string> basic_layouts = { + "", + "a3", "a3c5", "a3c5e7", + "b2_1", "b2_1d3_1", "b2_1d3_1f4_1", + "a3b2_1c5d3_1", "b2_1c5d3_1e7" +}; + +const std::vector<std::pair<vespalib::string,vespalib::string>> join_layouts = { + {"", ""}, + {"", "a3"}, + {"", "b2_1"}, + {"", "a3b2_1"}, + {"a3c5e7", "a3c5e7"}, + {"c5", "a3e7"}, + {"a3c5", "c5e7"}, + {"b4_1d6_1f8_1", "b2_2d3_2f4_2"}, + {"d3_1", "b2_1f4_1"}, + {"b2_1d6_1", "d3_2f4_2"}, + {"a3b4_1c5d6_1", "a3b2_1c5d3_1"}, + {"a3b2_1", "c5d3_1"}, + {"a3b4_1c5", "b2_1c5d3_1"} +}; + +//----------------------------------------------------------------------------- + +const std::vector<CellType> just_double = {CellType::DOUBLE}; +const std::vector<CellType> just_float = {CellType::FLOAT}; +const std::vector<CellType> all_types = CellTypeUtils::list_types(); const double my_nan = std::numeric_limits<double>::quiet_NaN(); +Sequence skew(const Sequence &seq) { + return [seq](size_t i) { return seq(i + 7); }; +} + +Sequence my_seq(double x0, double delta, size_t n) { + std::vector<double> values; + double x = x0; + for (size_t i = 0; i < n; ++i) { + values.push_back(x); + x += delta; + } + return Seq(values); +} + +//----------------------------------------------------------------------------- + +void generate(const vespalib::string &expr, const GenSpec &a, TestBuilder &dst) { + auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto a_ct: a_cell_types) { + dst.add(expr, {{"a", a.cpy().cells(a_ct)}}); + } +} + +//----------------------------------------------------------------------------- + +void generate(const vespalib::string &expr, const GenSpec &a, const GenSpec &b, TestBuilder &dst) { + auto a_cell_types = a.dims().empty() ? just_double : dst.full ? all_types : just_float; + auto b_cell_types = b.dims().empty() ? just_double : dst.full ? all_types : just_float; + for (auto a_ct: a_cell_types) { + for (auto b_ct: b_cell_types) { + dst.add(expr, {{"a", a.cpy().cells(a_ct)},{"b", b.cpy().cells(b_ct)}}); + } + } +} + //----------------------------------------------------------------------------- void generate_reduce(Aggr aggr, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - TensorSpec input = spec(layout, seq); - for (const Domain &domain: layout) { - vespalib::string expr = vespalib::make_string("reduce(a,%s,%s)", - AggrNames::name_of(aggr)->c_str(), domain.name().c_str()); - dst.add(expr, {{"a", input}}); + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(seq); + for (const auto &dim: a.dims()) { + vespalib::string expr = fmt("reduce(a,%s,%s)", + AggrNames::name_of(aggr)->c_str(), + dim.name().c_str()); + generate(expr, a, dst); + } + if (a.dims().size() > 1) { + vespalib::string expr = fmt("reduce(a,%s,%s,%s)", + AggrNames::name_of(aggr)->c_str(), + a.dims().back().name().c_str(), + a.dims().front().name().c_str()); + generate(expr, a, dst); } { - vespalib::string expr = vespalib::make_string("reduce(a,%s)", AggrNames::name_of(aggr)->c_str()); - dst.add(expr, {{"a", input}}); + vespalib::string expr = fmt("reduce(a,%s)", + AggrNames::name_of(aggr)->c_str()); + generate(expr, a, dst); } } } @@ -55,28 +123,15 @@ void generate_tensor_reduce(TestBuilder &dst) { //----------------------------------------------------------------------------- void generate_map_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) - }; - for (const Layout &layout: layouts) { - dst.add(expr, {{"a", spec(layout, seq)}}); + for (const auto &layout: basic_layouts) { + GenSpec a = GenSpec::from_desc(layout).seq(seq); + generate(expr, a, dst); } } void generate_op1_map(const vespalib::string &op1_expr, const Sequence &seq, TestBuilder &dst) { generate_map_expr(op1_expr, seq, dst); - generate_map_expr(vespalib::make_string("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); + generate_map_expr(fmt("map(a,f(a)(%s))", op1_expr.c_str()), seq, dst); } void generate_tensor_map(TestBuilder &dst) { @@ -110,46 +165,17 @@ void generate_tensor_map(TestBuilder &dst) { //----------------------------------------------------------------------------- void generate_join_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { - std::vector<Layout> layouts = { - {}, {}, - {}, {x(5)}, - {x(5)}, {}, - {}, float_cells({x(5)}), - float_cells({x(5)}), {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) - }; - ASSERT_TRUE((layouts.size() % 2) == 0); - for (size_t i = 0; i < layouts.size(); i += 2) { - auto a = spec(layouts[i], seq); - auto b = spec(layouts[i + 1], seq); - dst.add(expr, {{"a", a}, {"b", b}}); + for (const auto &layouts: join_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq); + GenSpec b = GenSpec::from_desc(layouts.second).seq(skew(seq)); + generate(expr, a, b, dst); + generate(expr, b, a, dst); } } void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { generate_join_expr(op2_expr, seq, dst); - generate_join_expr(vespalib::make_string("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); + generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } void generate_tensor_join(TestBuilder &dst) { @@ -158,8 +184,8 @@ void generate_tensor_join(TestBuilder &dst) { generate_op2_join("a*b", Div16(N()), dst); generate_op2_join("a/b", Div16(N()), dst); generate_op2_join("a%b", Div16(N()), dst); - generate_op2_join("a^b", Div16(N()), dst); - generate_op2_join("pow(a,b)", Div16(N()), dst); + generate_op2_join("a^b", my_seq(1.0, 1.0, 5), dst); + generate_op2_join("pow(a,b)", my_seq(1.0, 1.0, 5), dst); generate_op2_join("a==b", Div16(N()), dst); generate_op2_join("a!=b", Div16(N()), dst); generate_op2_join("a~=b", Div16(N()), dst); @@ -277,6 +303,10 @@ void generate_tensor_lambda(TestBuilder &dst) { //----------------------------------------------------------------------------- +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + void Generator::generate(TestBuilder &dst) { diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index 90e218f1d3c..a71531f7cf3 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -6,6 +6,8 @@ #include <map> struct TestBuilder { + bool full; + TestBuilder(bool full_in) : full(full_in) {} using TensorSpec = vespalib::eval::TensorSpec; virtual void add(const vespalib::string &expression, const std::map<vespalib::string,TensorSpec> &inputs) = 0; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 04d4c91b579..7f09f13a174 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -154,7 +154,7 @@ class MyTestBuilder : public TestBuilder { private: TestWriter _writer; public: - MyTestBuilder(Output &out) : _writer(out) {} + MyTestBuilder(bool full_in, Output &out) : TestBuilder(full_in), _writer(out) {} void add(const vespalib::string &expression, const std::map<vespalib::string,TensorSpec> &inputs_in) override { @@ -168,8 +168,8 @@ public: } }; -void generate(Output &out) { - MyTestBuilder my_test_builder(out); +void generate(Output &out, bool full) { + MyTestBuilder my_test_builder(full, out); Generator::generate(my_test_builder); } @@ -258,6 +258,7 @@ int usage(const char *self) { fprintf(stderr, " that all results are as expected\n"); fprintf(stderr, " 'display': read tests from stdin and print them to stdout\n"); fprintf(stderr, " in human-readable form\n"); + fprintf(stderr, " 'generate-some': write some test cases to stdout\n"); return 1; } @@ -270,7 +271,9 @@ int main(int argc, char **argv) { vespalib::string mode = argv[1]; TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str()); if (mode == "generate") { - generate(std_out); + generate(std_out, true); + } else if (mode == "generate-some") { + generate(std_out, false); } else if (mode == "evaluate") { evaluate(std_in, std_out); } else if (mode == "verify") { |