diff options
Diffstat (limited to 'eval')
4 files changed, 40 insertions, 14 deletions
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 9ae33c1234f..3c2a9b12272 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -436,9 +436,9 @@ void generate_converting_lambda(TestBuilder &dst) { auto sparse = GenSpec::from_desc("y5_2"); auto mixed = GenSpec::from_desc("x3y5_2"); // change cell type and dimension types - dst.add("tensor<bfloat16>(x[5])(a{x:(x)})", {{"a", dense}}); - dst.add("tensor<bfloat16>(y[10])(a{y:(y)})", {{"a", sparse}}); - dst.add("tensor<bfloat16>(x[5],y[10])(a{x:(x),y:(y)})", {{"a", mixed}}); + dst.add_ignore_java("tensor<bfloat16>(x[5])(a{x:(x)})", {{"a", dense}}); + dst.add_ignore_java("tensor<bfloat16>(y[10])(a{y:(y)})", {{"a", sparse}}); + dst.add_ignore_java("tensor<bfloat16>(x[5],y[10])(a{x:(x),y:(y)})", {{"a", mixed}}); } //----------------------------------------------------------------------------- @@ -492,12 +492,12 @@ void generate_nan_existence(TestBuilder &dst) { const vespalib::string inner_expr = "f(x,y)(if(isNan(x),11,x)+if(isNan(y),22,y))"; vespalib::string merge_expr = fmt("merge(a,b,%s)", inner_expr.c_str()); vespalib::string join_expr = fmt("join(a,b,%s)", inner_expr.c_str()); - dst.add(merge_expr, {{"a", sparse1}, {"b", sparse2}}); - dst.add(merge_expr, {{"a", mixed1}, {"b", mixed2}}); - dst.add(join_expr, {{"a", sparse1}, {"b", sparse2}}); - dst.add(join_expr, {{"a", mixed1}, {"b", mixed2}}); - dst.add(join_expr, {{"a", sparse1}, {"b", mixed2}}); - dst.add(join_expr, {{"a", mixed1}, {"b", sparse2}}); + dst.add_ignore_java(merge_expr, {{"a", sparse1}, {"b", sparse2}}); + dst.add_ignore_java(merge_expr, {{"a", mixed1}, {"b", mixed2}}); + dst.add_ignore_java(join_expr, {{"a", sparse1}, {"b", sparse2}}); + dst.add_ignore_java(join_expr, {{"a", mixed1}, {"b", mixed2}}); + dst.add_ignore_java(join_expr, {{"a", sparse1}, {"b", mixed2}}); + dst.add_ignore_java(join_expr, {{"a", mixed1}, {"b", sparse2}}); } //----------------------------------------------------------------------------- diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index a71531f7cf3..e9482b9015c 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -4,13 +4,25 @@ #include <vespa/eval/eval/tensor_spec.h> #include <map> +#include <set> struct TestBuilder { bool full; TestBuilder(bool full_in) : full(full_in) {} using TensorSpec = vespalib::eval::TensorSpec; virtual void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) = 0; + const std::map<vespalib::string,TensorSpec> &inputs, + const std::set<vespalib::string> &ignore) = 0; + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) + { + add(expression, inputs, {}); + } + void add_ignore_java(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) + { + add(expression, inputs, {"vespajlib"}); + } virtual ~TestBuilder() {} }; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 47089e1a8ff..e6bbb1f8a41 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -37,6 +37,7 @@ using namespace std::placeholders; //----------------------------------------------------------------------------- size_t fail_cnt = 0; +size_t ignore_cnt = 0; //----------------------------------------------------------------------------- @@ -176,7 +177,8 @@ private: public: MyTestBuilder(bool full_in, Output &out) : TestBuilder(full_in), _writer(out) {} void add(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs_in) override + const std::map<vespalib::string,TensorSpec> &inputs_in, + const std::set<vespalib::string> &ignore_in) override { Cursor &test = _writer.create(); test.setString("expression", expression); @@ -185,6 +187,12 @@ public: insert_value(inputs, name, spec); } test.setObject("result"); + if (!ignore_in.empty()) { + Cursor &ignore = test.setObject("ignore"); + for (const auto &impl: ignore_in) { + ignore.setBool(impl, true); + } + } } void add_failing_test(bool ignore_fail) { Cursor &test = _writer.create(); @@ -192,7 +200,7 @@ public: insert_value(test.setObject("inputs"), "a", GenSpec(1).idx("x", 3)); insert_value(test.setObject("result"), "dummy", GenSpec(2).idx("x", 3)); if (ignore_fail) { - test.setBool("ignore_fail", true); + test.setObject("ignore").setBool("dummy", true); } } }; @@ -232,8 +240,10 @@ void verify(Input &in, Output &out) { ++result_map[result]; auto actual_result = extract_value(slime["result"][result]); if (!require_impl::eq(actual_result, reference_result)) { - bool ignore_fail = slime["ignore_fail"].asBool(); - if (!ignore_fail) { + bool ignore_fail = slime["ignore"][result].asBool(); + if (ignore_fail) { + ++ignore_cnt; + } else { ++fail_cnt; } fprintf(stderr, "%sexpression failed('%s'): '%s'\n", ignore_fail ? "IGNORED: " : "", @@ -249,7 +259,9 @@ void verify(Input &in, Output &out) { stats.setLong(entry.first, entry.second); } REQUIRE(!slime["fail_cnt"].valid()); + REQUIRE(!slime["ignore_cnt"].valid()); slime.get().setLong("fail_cnt", fail_cnt); + slime.get().setLong("ignore_cnt", ignore_cnt); JsonFormat::encode(slime, out, false); }; for_each_test(in, handle_test, handle_summary); diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index 3f582779202..8271e8b57ee 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -26,6 +26,8 @@ TEST("require that (some) cross-language tensor conformance tests pass with C++ int streamed_tests = result["stats"]["cpp_streamed_value"].asLong(); EXPECT_TRUE(result["fail_cnt"].valid()); EXPECT_EQUAL(result["fail_cnt"].asLong(), 0); + EXPECT_TRUE(result["ignore_cnt"].valid()); + EXPECT_EQUAL(result["ignore_cnt"].asLong(), 0); EXPECT_GREATER(num_tests, 1000); EXPECT_EQUAL(prod_tests, num_tests); EXPECT_EQUAL(simple_tests, num_tests); |