common code for multi-cell-type-testing

11 files changed, 299 insertions, 71 deletions

diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index 51b8b850747..9820163725d 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -13,6 +13,7 @@ vespa_define_module(
     src/tests/ann
     src/tests/eval/aggr
     src/tests/eval/array_array_map
+    src/tests/eval/cell_type_space
     src/tests/eval/compile_cache
     src/tests/eval/compiled_function
     src/tests/eval/fast_value
diff --git a/eval/src/tests/eval/cell_type_space/CMakeLists.txt b/eval/src/tests/eval/cell_type_space/CMakeLists.txt
new file mode 100644
index 00000000000..dea89bc6937
--- /dev/null
+++ b/eval/src/tests/eval/cell_type_space/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_cell_type_space_test_app TEST
+    SOURCES
+    cell_type_space_test.cpp
+    DEPENDS
+    vespaeval
+    GTest::GTest
+)
+vespa_add_test(NAME eval_cell_type_space_test_app COMMAND eval_cell_type_space_test_app)
diff --git a/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp b/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp
new file mode 100644
index 00000000000..25747cc0304
--- /dev/null
+++ b/eval/src/tests/eval/cell_type_space/cell_type_space_test.cpp
@@ -0,0 +1,76 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/eval/test/cell_type_space.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using namespace vespalib::eval;
+using namespace vespalib::eval::test;
+
+//-----------------------------------------------------------------------------
+
+auto all_types = CellTypeUtils::list_types();
+
+//-----------------------------------------------------------------------------
+
+TEST(CellTypeSpaceTest, n_1) {
+    auto space = CellTypeSpace(all_types, 1);
+    for (auto t0: all_types) {
+        ASSERT_TRUE(space.valid());
+        auto ts = space.get();
+        ASSERT_EQ(ts.size(), 1);
+        EXPECT_EQ(t0, ts[0]);
+        space.next();
+    }
+    EXPECT_FALSE(space.valid());
+}
+
+TEST(CellTypeSpaceTest, n_2) {
+    auto space = CellTypeSpace(all_types, 2);
+    for (auto t0: all_types) {
+        for (auto t1: all_types) {
+            ASSERT_TRUE(space.valid());
+            auto ts = space.get();
+            ASSERT_EQ(ts.size(), 2);
+            EXPECT_EQ(t0, ts[0]);
+            EXPECT_EQ(t1, ts[1]);
+            space.next();
+        }
+    }
+    EXPECT_FALSE(space.valid());
+}
+
+TEST(CellTypeSpaceTest, n_2_same) {
+    auto space = CellTypeSpace(all_types, 2).same();
+    for (auto t0: all_types) {
+        for (auto t1: all_types) {
+            if (t0 != t1) continue;
+            ASSERT_TRUE(space.valid());
+            auto ts = space.get();
+            ASSERT_EQ(ts.size(), 2);
+            EXPECT_EQ(t0, ts[0]);
+            EXPECT_EQ(t1, ts[1]);
+            space.next();
+        }
+    }
+    EXPECT_FALSE(space.valid());
+}
+
+TEST(CellTypeSpaceTest, n_2_different) {
+    auto space = CellTypeSpace(all_types, 2).different();
+    for (auto t0: all_types) {
+        for (auto t1: all_types) {
+            if (t0 == t1) continue;
+            ASSERT_TRUE(space.valid());
+            auto ts = space.get();
+            ASSERT_EQ(ts.size(), 2);
+            EXPECT_EQ(t0, ts[0]);
+            EXPECT_EQ(t1, ts[1]);
+            space.next();
+        }
+    }
+    EXPECT_FALSE(space.valid());
+}
+
+//-----------------------------------------------------------------------------
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/eval/src/tests/eval/gen_spec/gen_spec_test.cpp b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp
index 0d620873b07..d89bf5b8e79 100644
--- a/eval/src/tests/eval/gen_spec/gen_spec_test.cpp
+++ b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp
@@ -263,6 +263,13 @@ TEST(GenSpecFromDescTest, capital_letter_allowed) {
     EXPECT_EQ(gen_desc, expect);
 }
 
+TEST(GenSpecFromDescTest, scalar_can_be_created) {
+    // ''
+    auto expect = GenSpec().gen();
+    auto gen_desc = GenSpec::from_desc("").gen();
+    EXPECT_EQ(gen_desc, expect);
+}
+
 //-----------------------------------------------------------------------------
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp b/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp
index c26f46f5e11..6e476f7e094 100644
--- a/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp
+++ b/eval/src/tests/instruction/dense_single_reduce_function/dense_single_reduce_function_test.cpp
@@ -16,58 +16,37 @@ using namespace vespalib::eval::test;
 using namespace vespalib::eval::tensor_function;
 using vespalib::make_string_short::fmt;
 
-const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get();
-
 struct ReduceSpec {
+    using LookFor = DenseSingleReduceFunction;
     size_t outer_size;
     size_t reduce_size;
     size_t inner_size;
     Aggr aggr;
-};
-
-void verify_impl(const vespalib::string &expr,
-                 const std::vector<ReduceSpec> &spec_list,
-                 const std::vector<CellType> &with_cell_types)
-{
-    auto fun = Function::parse(expr);
-    ASSERT_EQUAL(fun->num_params(), 1u);
-    vespalib::string param_name = fun->param_name(0);
-    const auto param_spec = GenSpec::from_desc(param_name);
-    for (CellType ct: with_cell_types) {
-        EvalFixture::ParamRepo param_repo;
-        param_repo.add(param_name, param_spec.cpy().cells(ct));
-        EvalFixture slow_fixture(prod_factory, expr, param_repo, false);
-        EvalFixture fixture(prod_factory, expr, param_repo, true);
-        EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
-        EXPECT_EQUAL(fixture.result(), slow_fixture.result());
-        auto info = fixture.find_all<DenseSingleReduceFunction>();
-        ASSERT_EQUAL(info.size(), spec_list.size());
-        for (size_t i = 0; i < spec_list.size(); ++i) {
-            EXPECT_TRUE(info[i]->result_is_mutable());
-            EXPECT_EQUAL(info[i]->outer_size(), spec_list[i].outer_size);
-            EXPECT_EQUAL(info[i]->reduce_size(), spec_list[i].reduce_size);
-            EXPECT_EQUAL(info[i]->inner_size(), spec_list[i].inner_size);
-            EXPECT_EQUAL(int(info[i]->aggr()), int(spec_list[i].aggr));
-        }
+    void verify(const LookFor &fun) const {
+        EXPECT_TRUE(fun.result_is_mutable());
+        EXPECT_EQUAL(fun.outer_size(), outer_size);
+        EXPECT_EQUAL(fun.reduce_size(), reduce_size);
+        EXPECT_EQUAL(fun.inner_size(), inner_size);
+        EXPECT_EQUAL(int(fun.aggr()), int(aggr));
     }
-}
+};
 
 void verify_not_optimized(const vespalib::string &expr,
                           std::vector<CellType> with_cell_types = {CellType::DOUBLE})
 {
-    verify_impl(expr, {}, with_cell_types);
+    EvalFixture::verify<ReduceSpec>(expr, {}, CellTypeSpace(with_cell_types, 1));
 }
 
 void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec,
                       std::vector<CellType> with_cell_types = CellTypeUtils::list_types())
 {
-    verify_impl(expr, {spec}, with_cell_types);
+    EvalFixture::verify<ReduceSpec>(expr, {spec}, CellTypeSpace(with_cell_types, 1));
 }
 
 void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec1, const ReduceSpec &spec2,
                       std::vector<CellType> with_cell_types = CellTypeUtils::list_types())
 {
-    verify_impl(expr, {spec1, spec2}, with_cell_types);
+    EvalFixture::verify<ReduceSpec>(expr, {spec1, spec2}, CellTypeSpace(with_cell_types, 1));
 }
 
 TEST("require that reduce to scalar is not optimized") {
diff --git a/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp
index 770ba337a2d..aa4e175d0a7 100644
--- a/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp
+++ b/eval/src/tests/instruction/dense_xw_product_function/dense_xw_product_function_test.cpp
@@ -16,55 +16,25 @@ using namespace vespalib::eval;
 using namespace vespalib::eval::test;
 using namespace vespalib::eval::tensor_function;
 
-const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get();
-
-GenSpec::seq_t lhs_seq = [] (size_t i) noexcept { return (3.0 + i) * 7.0; };
-GenSpec::seq_t rhs_seq = [] (size_t i) noexcept { return (5.0 + i) * 43.0; };
-
 struct FunInfo {
     using LookFor = DenseXWProductFunction;
     size_t vec_size;
     size_t res_size;
     bool happy;
-    bool check(const LookFor &fun) const {
-        return ((fun.result_is_mutable()) &&
-                (fun.vector_size() == vec_size) &&
-                (fun.result_size() == res_size) &&
-                (fun.common_inner() == happy));
+    void verify(const LookFor &fun) const {
+        EXPECT_TRUE(fun.result_is_mutable());
+        EXPECT_EQUAL(fun.vector_size(), vec_size);
+        EXPECT_EQUAL(fun.result_size(), res_size);
+        EXPECT_EQUAL(fun.common_inner(), happy);
     }
 };
 
-void verify(const vespalib::string &expr, const std::vector<FunInfo> &fun_info, const std::vector<CellType> &with_cell_types) {
-    auto fun = Function::parse(expr);
-    ASSERT_EQUAL(fun->num_params(), 2u);
-    vespalib::string lhs_name = fun->param_name(0);
-    vespalib::string rhs_name = fun->param_name(1);
-    const auto lhs_spec = GenSpec::from_desc(lhs_name);
-    const auto rhs_spec = GenSpec::from_desc(rhs_name);
-    for (CellType lhs_ct: with_cell_types) {
-        for (CellType rhs_ct: with_cell_types) {
-            EvalFixture::ParamRepo param_repo;
-            param_repo.add(lhs_name, lhs_spec.cpy().cells(lhs_ct).seq(lhs_seq));
-            param_repo.add(rhs_name, rhs_spec.cpy().cells(rhs_ct).seq(rhs_seq));
-            EvalFixture slow_fixture(prod_factory, expr, param_repo, false);
-            EvalFixture fixture(prod_factory, expr, param_repo, true);
-            EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
-            EXPECT_EQUAL(fixture.result(), slow_fixture.result());
-            auto info = fixture.find_all<FunInfo::LookFor>();
-            ASSERT_EQUAL(info.size(), fun_info.size());
-            for (size_t i = 0; i < fun_info.size(); ++i) {
-                EXPECT_TRUE(fun_info[i].check(*info[i]));
-            }
-        }
-    }
-}
-
 void verify_not_optimized(const vespalib::string &expr) {
-    return verify(expr, {}, {CellType::FLOAT});
+    EvalFixture::verify<FunInfo>(expr, {}, CellTypeSpace({CellType::FLOAT}, 2));
 }
 
 void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_size, bool happy) {
-    return verify(expr, {{vec_size, res_size, happy}}, CellTypeUtils::list_types());
+    EvalFixture::verify<FunInfo>(expr, {{vec_size, res_size, happy}}, CellTypeSpace(CellTypeUtils::list_types(), 2));
 }
 
 vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common) {
@@ -122,7 +92,7 @@ TEST("require that xw product can be debug dumped") {
     EvalFixture::ParamRepo param_repo;
     param_repo.add("y5", GenSpec::from_desc("y5"));
     param_repo.add("x8y5", GenSpec::from_desc("x8y5"));
-    EvalFixture fixture(prod_factory, "reduce(y5*x8y5,sum,y)", param_repo, true);
+    EvalFixture fixture(EvalFixture::prod_factory(), "reduce(y5*x8y5,sum,y)", param_repo, true);
     auto info = fixture.find_all<DenseXWProductFunction>();
     ASSERT_EQUAL(info.size(), 1u);
     EXPECT_TRUE(info[0]->result_is_mutable());
diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt
index e82b85d1890..dc0576f07a2 100644
--- a/eval/src/vespa/eval/eval/test/CMakeLists.txt
+++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(eval_eval_test OBJECT
     SOURCES
+    cell_type_space.cpp
     eval_fixture.cpp
     eval_spec.cpp
     gen_spec.cpp
diff --git a/eval/src/vespa/eval/eval/test/cell_type_space.cpp b/eval/src/vespa/eval/eval/test/cell_type_space.cpp
new file mode 100644
index 00000000000..76bb87e5170
--- /dev/null
+++ b/eval/src/vespa/eval/eval/test/cell_type_space.cpp
@@ -0,0 +1,36 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "cell_type_space.h"
+
+namespace vespalib::eval::test {
+
+void
+CellTypeSpace::step_state() {
+    for (size_t idx = _state.size(); idx-- > 0; ) {
+        if ((++_state[idx]) < _types.size()) {
+            return;
+        } else {
+            _state[idx] = 0;
+        }
+    }
+    _done = true;
+}
+
+bool
+CellTypeSpace::should_skip() {
+    if (_done) {
+        return false;
+    }
+    bool same = true;
+    auto type = _state[0];
+    for (auto t: _state) {
+        if (t != type) {
+            same = false;
+        }
+    }
+    return same ? _drop_same : _drop_different;
+}
+
+CellTypeSpace::~CellTypeSpace() = default;
+
+} // namespace
diff --git a/eval/src/vespa/eval/eval/test/cell_type_space.h b/eval/src/vespa/eval/eval/test/cell_type_space.h
new file mode 100644
index 00000000000..92c29b9cf9c
--- /dev/null
+++ b/eval/src/vespa/eval/eval/test/cell_type_space.h
@@ -0,0 +1,69 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/cell_type.h>
+#include <vector>
+
+namespace vespalib::eval::test {
+
+/**
+ * Helper class used to span out the space describing the cell types
+ * of different values.
+ **/
+class CellTypeSpace
+{
+private:
+    std::vector<CellType> _types;
+    std::vector<size_t> _state;
+    bool _drop_same;
+    bool _drop_different;
+    bool _done;
+
+    void step_state(); // will set _done
+    bool should_skip(); // will check _done
+    void skip_unwanted() {
+        while (should_skip()) {
+            step_state();
+        }
+    }
+
+public:
+    CellTypeSpace(const std::vector<CellType> &types, size_t n)
+        : _types(types), _state(n, 0), _drop_same(false), _drop_different(false), _done(false)
+    {
+        assert(!types.empty());
+        assert(n > 0);
+        skip_unwanted();
+    }
+    ~CellTypeSpace();
+    CellTypeSpace &same() {
+        _drop_different = true;
+        assert(!_drop_same);
+        skip_unwanted();
+        return *this;
+    }
+    CellTypeSpace &different() {
+        _drop_same = true;
+        assert(!_drop_different);
+        skip_unwanted();
+        return *this;
+    }
+    size_t n() const { return _state.size(); }
+    bool valid() const { return !_done; }
+    void next() {
+        assert(valid());
+        step_state();
+        skip_unwanted();
+    }
+    std::vector<CellType> get() const {
+        assert(valid());
+        std::vector<CellType> ret;
+        for (size_t idx: _state) {
+            ret.push_back(_types[idx]);
+        }
+        return ret;
+    }
+};
+
+} // namespace
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
index 1c1d1aef18d..75626307bfd 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
@@ -1,6 +1,5 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/vespalib/testkit/test_kit.h>
 #include "eval_fixture.h"
 #include "reference_evaluation.h"
 #include <vespa/eval/eval/make_tensor_function.h>
@@ -140,6 +139,28 @@ EvalFixture::ParamRepo::add_variants(const vespalib::string &name_base,
     return *this;
 }
 
+EvalFixture::ParamRepo &
+EvalFixture::ParamRepo::add(const vespalib::string &name, const vespalib::string &desc,
+                            CellType cell_type, GenSpec::seq_t seq)
+{
+    bool is_mutable = ((!desc.empty()) && (desc[0] == '@'));
+    if (is_mutable) {
+        return add_mutable(name, GenSpec::from_desc(desc.substr(1)).cells(cell_type).seq(seq));
+    } else {
+        return add(name, GenSpec::from_desc(desc).cells(cell_type).seq(seq));
+    }
+}
+
+EvalFixture::ParamRepo &
+EvalFixture::ParamRepo::add(const vespalib::string &name_desc, CellType cell_type, GenSpec::seq_t seq)
+{
+    auto pos = name_desc.find('$');
+    vespalib::string desc = (pos < name_desc.size())
+                            ? name_desc.substr(0, pos)
+                            : name_desc;
+    return add(name_desc, desc, cell_type, seq);
+}
+
 void
 EvalFixture::detect_param_tampering(const ParamRepo &param_repo, bool allow_mutable) const
 {
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h
index 71198a160a9..9bd2facb25a 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.h
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <vespa/eval/eval/fast_value.h>
+#include <vespa/eval/eval/simple_value.h>
 #include <vespa/eval/eval/function.h>
 #include <vespa/eval/eval/tensor_spec.h>
 #include <vespa/eval/eval/tensor_function.h>
@@ -11,6 +12,8 @@
 #include <set>
 #include <functional>
 #include "gen_spec.h"
+#include "cell_type_space.h"
+#include <vespa/vespalib/testkit/test_kit.h>
 
 namespace vespalib::eval::test {
 
@@ -36,6 +39,26 @@ public:
 
         // produce 4 variants: float/double * mutable/const
         ParamRepo &add_variants(const vespalib::string &name_base, const GenSpec &spec);
+
+        // add a parameter that is generated based on a description.
+        //
+        // the description may start with '@' to indicate that the
+        // parameter is mutable. The rest of the description must be a
+        // valid parameter to the GenSpec::from_desc() function.
+        ParamRepo &add(const vespalib::string &name, const vespalib::string &desc,
+                       CellType cell_type, GenSpec::seq_t seq);
+
+        // add a parameter that is generated based on a description,
+        // where the description is also the name of the parameter.
+        //
+        // This is a convenience wrapper for the function adding a
+        // parameter based on a descriprion. The only thing this
+        // function does is use the description as parameter name and
+        // strip an optional suffix starting with '$' from the name
+        // before using it as a descriprion. (to support multiple
+        // parameters with the same description).
+        ParamRepo &add(const vespalib::string &name_desc, CellType cell_type, GenSpec::seq_t seq);
+
         ~ParamRepo() {}
     };
 
@@ -85,6 +108,42 @@ public:
     static TensorSpec prod(const vespalib::string &expr, const ParamRepo &param_repo) {
         return EvalFixture(FastValueBuilderFactory::get(), expr, param_repo, true, false).result();
     }
+
+    static const ValueBuilderFactory &prod_factory() { return FastValueBuilderFactory::get(); }
+    static const ValueBuilderFactory &test_factory() { return SimpleValueBuilderFactory::get(); }
+
+    // Verify the evaluation result and specific tensor function
+    // details for the given expression with different combinations of
+    // cell types. Parameter names must be valid GenSpec descriptions
+    // ('a5b8'), with an optional mutable prefix ('@a5b8') to denote
+    // parameters that may be modified, and an optional non-descriptive
+    // trailer starting with '$' ('a5b3$2') to allow multiple
+    // parameters with the same description as well as scalars
+    // ('$this_is_a_scalar').
+
+    template <typename FunInfo>
+    static void verify(const vespalib::string &expr, const std::vector<FunInfo> &fun_info, CellTypeSpace cell_type_space) {
+        auto fun = Function::parse(expr);
+        ASSERT_EQUAL(fun->num_params(), cell_type_space.n());
+        for (; cell_type_space.valid(); cell_type_space.next()) {
+            auto cell_types = cell_type_space.get();
+            EvalFixture::ParamRepo param_repo;
+            for (size_t i = 0; i < fun->num_params(); ++i) {
+                param_repo.add(fun->param_name(i), cell_types[i], N(1 + i));
+            }
+            EvalFixture fixture(prod_factory(), expr, param_repo, true, true);
+            EvalFixture slow_fixture(prod_factory(), expr, param_repo, false, false);
+            EvalFixture test_fixture(test_factory(), expr, param_repo, true, true);
+            ASSERT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+            ASSERT_EQUAL(fixture.result(), slow_fixture.result());
+            ASSERT_EQUAL(fixture.result(), test_fixture.result());
+            auto info = fixture.find_all<typename FunInfo::LookFor>();
+            ASSERT_EQUAL(info.size(), fun_info.size());
+            for (size_t i = 0; i < fun_info.size(); ++i) {
+                fun_info[i].verify(*info[i]);
+            }
+        }
+    }
 };
 
 } // namespace vespalib::eval::test