From 5e62a5290c218dc45736c9221724060eb50fdce9 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Tue, 16 Mar 2021 12:48:12 +0000 Subject: common code for multi-cell-type-testing --- eval/CMakeLists.txt | 1 + eval/src/tests/eval/cell_type_space/CMakeLists.txt | 9 +++ .../eval/cell_type_space/cell_type_space_test.cpp | 76 ++++++++++++++++++++++ eval/src/tests/eval/gen_spec/gen_spec_test.cpp | 7 ++ .../dense_single_reduce_function_test.cpp | 43 ++++-------- .../dense_xw_product_function_test.cpp | 46 +++---------- eval/src/vespa/eval/eval/test/CMakeLists.txt | 1 + eval/src/vespa/eval/eval/test/cell_type_space.cpp | 36 ++++++++++ eval/src/vespa/eval/eval/test/cell_type_space.h | 69 ++++++++++++++++++++ eval/src/vespa/eval/eval/test/eval_fixture.cpp | 23 ++++++- eval/src/vespa/eval/eval/test/eval_fixture.h | 59 +++++++++++++++++ 11 files changed, 299 insertions(+), 71 deletions(-) create mode 100644 eval/src/tests/eval/cell_type_space/CMakeLists.txt create mode 100644 eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp create mode 100644 eval/src/vespa/eval/eval/test/cell_type_space.cpp create mode 100644 eval/src/vespa/eval/eval/test/cell_type_space.h (limited to 'eval') diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 51b8b850747..9820163725d 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_define_module( src/tests/ann src/tests/eval/aggr src/tests/eval/array_array_map + src/tests/eval/cell_type_space src/tests/eval/compile_cache src/tests/eval/compiled_function src/tests/eval/fast_value diff --git a/eval/src/tests/eval/cell_type_space/CMakeLists.txt b/eval/src/tests/eval/cell_type_space/CMakeLists.txt new file mode 100644 index 00000000000..dea89bc6937 --- /dev/null +++ b/eval/src/tests/eval/cell_type_space/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_cell_type_space_test_app TEST + SOURCES + cell_type_space_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_cell_type_space_test_app COMMAND eval_cell_type_space_test_app) diff --git a/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp b/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp new file mode 100644 index 00000000000..25747cc0304 --- /dev/null +++ b/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp @@ -0,0 +1,76 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +//----------------------------------------------------------------------------- + +auto all_types = CellTypeUtils::list_types(); + +//----------------------------------------------------------------------------- + +TEST(CellTypeSpaceTest, n_1) { + auto space = CellTypeSpace(all_types, 1); + for (auto t0: all_types) { + ASSERT_TRUE(space.valid()); + auto ts = space.get(); + ASSERT_EQ(ts.size(), 1); + EXPECT_EQ(t0, ts[0]); + space.next(); + } + EXPECT_FALSE(space.valid()); +} + +TEST(CellTypeSpaceTest, n_2) { + auto space = CellTypeSpace(all_types, 2); + for (auto t0: all_types) { + for (auto t1: all_types) { + ASSERT_TRUE(space.valid()); + auto ts = space.get(); + ASSERT_EQ(ts.size(), 2); + EXPECT_EQ(t0, ts[0]); + EXPECT_EQ(t1, ts[1]); + space.next(); + } + } + EXPECT_FALSE(space.valid()); +} + +TEST(CellTypeSpaceTest, n_2_same) { + auto space = CellTypeSpace(all_types, 2).same(); + for (auto t0: all_types) { + for (auto t1: all_types) { + if (t0 != t1) continue; + ASSERT_TRUE(space.valid()); + auto ts = space.get(); + ASSERT_EQ(ts.size(), 2); + EXPECT_EQ(t0, ts[0]); + EXPECT_EQ(t1, ts[1]); + space.next(); + } + } + EXPECT_FALSE(space.valid()); +} + +TEST(CellTypeSpaceTest, n_2_different) { + auto space = CellTypeSpace(all_types, 2).different(); + for (auto t0: all_types) { + for (auto t1: all_types) { + if (t0 == t1) continue; + ASSERT_TRUE(space.valid()); + auto ts = space.get(); + ASSERT_EQ(ts.size(), 2); + EXPECT_EQ(t0, ts[0]); + EXPECT_EQ(t1, ts[1]); + space.next(); + } + } + EXPECT_FALSE(space.valid()); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/gen_spec/gen_spec_test.cpp b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp index 0d620873b07..d89bf5b8e79 100644 --- a/eval/src/tests/eval/gen_spec/gen_spec_test.cpp +++ b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp @@ -263,6 +263,13 @@ TEST(GenSpecFromDescTest, capital_letter_allowed) { EXPECT_EQ(gen_desc, expect); } +TEST(GenSpecFromDescTest, scalar_can_be_created) { + // '' + auto expect = GenSpec().gen(); + auto gen_desc = GenSpec::from_desc("").gen(); + EXPECT_EQ(gen_desc, expect); +} + //----------------------------------------------------------------------------- GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp b/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp index c26f46f5e11..6e476f7e094 100644 --- a/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp +++ b/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp @@ -16,58 +16,37 @@ using namespace vespalib::eval::test; using namespace vespalib::eval::tensor_function; using vespalib::make_string_short::fmt; -const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); - struct ReduceSpec { + using LookFor = DenseSingleReduceFunction; size_t outer_size; size_t reduce_size; size_t inner_size; Aggr aggr; -}; - -void verify_impl(const vespalib::string &expr, - const std::vector &spec_list, - const std::vector &with_cell_types) -{ - auto fun = Function::parse(expr); - ASSERT_EQUAL(fun->num_params(), 1u); - vespalib::string param_name = fun->param_name(0); - const auto param_spec = GenSpec::from_desc(param_name); - for (CellType ct: with_cell_types) { - EvalFixture::ParamRepo param_repo; - param_repo.add(param_name, param_spec.cpy().cells(ct)); - EvalFixture slow_fixture(prod_factory, expr, param_repo, false); - EvalFixture fixture(prod_factory, expr, param_repo, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - EXPECT_EQUAL(fixture.result(), slow_fixture.result()); - auto info = fixture.find_all(); - ASSERT_EQUAL(info.size(), spec_list.size()); - for (size_t i = 0; i < spec_list.size(); ++i) { - EXPECT_TRUE(info[i]->result_is_mutable()); - EXPECT_EQUAL(info[i]->outer_size(), spec_list[i].outer_size); - EXPECT_EQUAL(info[i]->reduce_size(), spec_list[i].reduce_size); - EXPECT_EQUAL(info[i]->inner_size(), spec_list[i].inner_size); - EXPECT_EQUAL(int(info[i]->aggr()), int(spec_list[i].aggr)); - } + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + EXPECT_EQUAL(fun.outer_size(), outer_size); + EXPECT_EQUAL(fun.reduce_size(), reduce_size); + EXPECT_EQUAL(fun.inner_size(), inner_size); + EXPECT_EQUAL(int(fun.aggr()), int(aggr)); } -} +}; void verify_not_optimized(const vespalib::string &expr, std::vector with_cell_types = {CellType::DOUBLE}) { - verify_impl(expr, {}, with_cell_types); + EvalFixture::verify(expr, {}, CellTypeSpace(with_cell_types, 1)); } void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec, std::vector with_cell_types = CellTypeUtils::list_types()) { - verify_impl(expr, {spec}, with_cell_types); + EvalFixture::verify(expr, {spec}, CellTypeSpace(with_cell_types, 1)); } void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec1, const ReduceSpec &spec2, std::vector with_cell_types = CellTypeUtils::list_types()) { - verify_impl(expr, {spec1, spec2}, with_cell_types); + EvalFixture::verify(expr, {spec1, spec2}, CellTypeSpace(with_cell_types, 1)); } TEST("require that reduce to scalar is not optimized") { diff --git a/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp index 770ba337a2d..aa4e175d0a7 100644 --- a/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -16,55 +16,25 @@ using namespace vespalib::eval; using namespace vespalib::eval::test; using namespace vespalib::eval::tensor_function; -const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); - -GenSpec::seq_t lhs_seq = [] (size_t i) noexcept { return (3.0 + i) * 7.0; }; -GenSpec::seq_t rhs_seq = [] (size_t i) noexcept { return (5.0 + i) * 43.0; }; - struct FunInfo { using LookFor = DenseXWProductFunction; size_t vec_size; size_t res_size; bool happy; - bool check(const LookFor &fun) const { - return ((fun.result_is_mutable()) && - (fun.vector_size() == vec_size) && - (fun.result_size() == res_size) && - (fun.common_inner() == happy)); + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + EXPECT_EQUAL(fun.vector_size(), vec_size); + EXPECT_EQUAL(fun.result_size(), res_size); + EXPECT_EQUAL(fun.common_inner(), happy); } }; -void verify(const vespalib::string &expr, const std::vector &fun_info, const std::vector &with_cell_types) { - auto fun = Function::parse(expr); - ASSERT_EQUAL(fun->num_params(), 2u); - vespalib::string lhs_name = fun->param_name(0); - vespalib::string rhs_name = fun->param_name(1); - const auto lhs_spec = GenSpec::from_desc(lhs_name); - const auto rhs_spec = GenSpec::from_desc(rhs_name); - for (CellType lhs_ct: with_cell_types) { - for (CellType rhs_ct: with_cell_types) { - EvalFixture::ParamRepo param_repo; - param_repo.add(lhs_name, lhs_spec.cpy().cells(lhs_ct).seq(lhs_seq)); - param_repo.add(rhs_name, rhs_spec.cpy().cells(rhs_ct).seq(rhs_seq)); - EvalFixture slow_fixture(prod_factory, expr, param_repo, false); - EvalFixture fixture(prod_factory, expr, param_repo, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - EXPECT_EQUAL(fixture.result(), slow_fixture.result()); - auto info = fixture.find_all(); - ASSERT_EQUAL(info.size(), fun_info.size()); - for (size_t i = 0; i < fun_info.size(); ++i) { - EXPECT_TRUE(fun_info[i].check(*info[i])); - } - } - } -} - void verify_not_optimized(const vespalib::string &expr) { - return verify(expr, {}, {CellType::FLOAT}); + EvalFixture::verify(expr, {}, CellTypeSpace({CellType::FLOAT}, 2)); } void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_size, bool happy) { - return verify(expr, {{vec_size, res_size, happy}}, CellTypeUtils::list_types()); + EvalFixture::verify(expr, {{vec_size, res_size, happy}}, CellTypeSpace(CellTypeUtils::list_types(), 2)); } vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common) { @@ -122,7 +92,7 @@ TEST("require that xw product can be debug dumped") { EvalFixture::ParamRepo param_repo; param_repo.add("y5", GenSpec::from_desc("y5")); param_repo.add("x8y5", GenSpec::from_desc("x8y5")); - EvalFixture fixture(prod_factory, "reduce(y5*x8y5,sum,y)", param_repo, true); + EvalFixture fixture(EvalFixture::prod_factory(), "reduce(y5*x8y5,sum,y)", param_repo, true); auto info = fixture.find_all(); ASSERT_EQUAL(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index e82b85d1890..dc0576f07a2 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(eval_eval_test OBJECT SOURCES + cell_type_space.cpp eval_fixture.cpp eval_spec.cpp gen_spec.cpp diff --git a/eval/src/vespa/eval/eval/test/cell_type_space.cpp b/eval/src/vespa/eval/eval/test/cell_type_space.cpp new file mode 100644 index 00000000000..76bb87e5170 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/cell_type_space.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cell_type_space.h" + +namespace vespalib::eval::test { + +void +CellTypeSpace::step_state() { + for (size_t idx = _state.size(); idx-- > 0; ) { + if ((++_state[idx]) < _types.size()) { + return; + } else { + _state[idx] = 0; + } + } + _done = true; +} + +bool +CellTypeSpace::should_skip() { + if (_done) { + return false; + } + bool same = true; + auto type = _state[0]; + for (auto t: _state) { + if (t != type) { + same = false; + } + } + return same ? _drop_same : _drop_different; +} + +CellTypeSpace::~CellTypeSpace() = default; + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/cell_type_space.h b/eval/src/vespa/eval/eval/test/cell_type_space.h new file mode 100644 index 00000000000..92c29b9cf9c --- /dev/null +++ b/eval/src/vespa/eval/eval/test/cell_type_space.h @@ -0,0 +1,69 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace vespalib::eval::test { + +/** + * Helper class used to span out the space describing the cell types + * of different values. + **/ +class CellTypeSpace +{ +private: + std::vector _types; + std::vector _state; + bool _drop_same; + bool _drop_different; + bool _done; + + void step_state(); // will set _done + bool should_skip(); // will check _done + void skip_unwanted() { + while (should_skip()) { + step_state(); + } + } + +public: + CellTypeSpace(const std::vector &types, size_t n) + : _types(types), _state(n, 0), _drop_same(false), _drop_different(false), _done(false) + { + assert(!types.empty()); + assert(n > 0); + skip_unwanted(); + } + ~CellTypeSpace(); + CellTypeSpace &same() { + _drop_different = true; + assert(!_drop_same); + skip_unwanted(); + return *this; + } + CellTypeSpace &different() { + _drop_same = true; + assert(!_drop_different); + skip_unwanted(); + return *this; + } + size_t n() const { return _state.size(); } + bool valid() const { return !_done; } + void next() { + assert(valid()); + step_state(); + skip_unwanted(); + } + std::vector get() const { + assert(valid()); + std::vector ret; + for (size_t idx: _state) { + ret.push_back(_types[idx]); + } + return ret; + } +}; + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 1c1d1aef18d..75626307bfd 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -1,6 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include #include "eval_fixture.h" #include "reference_evaluation.h" #include @@ -140,6 +139,28 @@ EvalFixture::ParamRepo::add_variants(const vespalib::string &name_base, return *this; } +EvalFixture::ParamRepo & +EvalFixture::ParamRepo::add(const vespalib::string &name, const vespalib::string &desc, + CellType cell_type, GenSpec::seq_t seq) +{ + bool is_mutable = ((!desc.empty()) && (desc[0] == '@')); + if (is_mutable) { + return add_mutable(name, GenSpec::from_desc(desc.substr(1)).cells(cell_type).seq(seq)); + } else { + return add(name, GenSpec::from_desc(desc).cells(cell_type).seq(seq)); + } +} + +EvalFixture::ParamRepo & +EvalFixture::ParamRepo::add(const vespalib::string &name_desc, CellType cell_type, GenSpec::seq_t seq) +{ + auto pos = name_desc.find('$'); + vespalib::string desc = (pos < name_desc.size()) + ? name_desc.substr(0, pos) + : name_desc; + return add(name_desc, desc, cell_type, seq); +} + void EvalFixture::detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const { diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 71198a160a9..9bd2facb25a 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include "gen_spec.h" +#include "cell_type_space.h" +#include namespace vespalib::eval::test { @@ -36,6 +39,26 @@ public: // produce 4 variants: float/double * mutable/const ParamRepo &add_variants(const vespalib::string &name_base, const GenSpec &spec); + + // add a parameter that is generated based on a description. + // + // the description may start with '@' to indicate that the + // parameter is mutable. The rest of the description must be a + // valid parameter to the GenSpec::from_desc() function. + ParamRepo &add(const vespalib::string &name, const vespalib::string &desc, + CellType cell_type, GenSpec::seq_t seq); + + // add a parameter that is generated based on a description, + // where the description is also the name of the parameter. + // + // This is a convenience wrapper for the function adding a + // parameter based on a descriprion. The only thing this + // function does is use the description as parameter name and + // strip an optional suffix starting with '$' from the name + // before using it as a descriprion. (to support multiple + // parameters with the same description). + ParamRepo &add(const vespalib::string &name_desc, CellType cell_type, GenSpec::seq_t seq); + ~ParamRepo() {} }; @@ -85,6 +108,42 @@ public: static TensorSpec prod(const vespalib::string &expr, const ParamRepo ¶m_repo) { return EvalFixture(FastValueBuilderFactory::get(), expr, param_repo, true, false).result(); } + + static const ValueBuilderFactory &prod_factory() { return FastValueBuilderFactory::get(); } + static const ValueBuilderFactory &test_factory() { return SimpleValueBuilderFactory::get(); } + + // Verify the evaluation result and specific tensor function + // details for the given expression with different combinations of + // cell types. Parameter names must be valid GenSpec descriptions + // ('a5b8'), with an optional mutable prefix ('@a5b8') to denote + // parameters that may be modified, and an optional non-descriptive + // trailer starting with '$' ('a5b3$2') to allow multiple + // parameters with the same description as well as scalars + // ('$this_is_a_scalar'). + + template + static void verify(const vespalib::string &expr, const std::vector &fun_info, CellTypeSpace cell_type_space) { + auto fun = Function::parse(expr); + ASSERT_EQUAL(fun->num_params(), cell_type_space.n()); + for (; cell_type_space.valid(); cell_type_space.next()) { + auto cell_types = cell_type_space.get(); + EvalFixture::ParamRepo param_repo; + for (size_t i = 0; i < fun->num_params(); ++i) { + param_repo.add(fun->param_name(i), cell_types[i], N(1 + i)); + } + EvalFixture fixture(prod_factory(), expr, param_repo, true, true); + EvalFixture slow_fixture(prod_factory(), expr, param_repo, false, false); + EvalFixture test_fixture(test_factory(), expr, param_repo, true, true); + ASSERT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + ASSERT_EQUAL(fixture.result(), slow_fixture.result()); + ASSERT_EQUAL(fixture.result(), test_fixture.result()); + auto info = fixture.find_all(); + ASSERT_EQUAL(info.size(), fun_info.size()); + for (size_t i = 0; i < fun_info.size(); ++i) { + fun_info[i].verify(*info[i]); + } + } + } }; } // namespace vespalib::eval::test -- cgit v1.2.3