diff options
Diffstat (limited to 'eval')
12 files changed, 90 insertions, 118 deletions
diff --git a/eval/src/vespa/eval/eval/aggr.cpp b/eval/src/vespa/eval/eval/aggr.cpp index d10bbc4abb8..8efb0ec9fe7 100644 --- a/eval/src/vespa/eval/eval/aggr.cpp +++ b/eval/src/vespa/eval/eval/aggr.cpp @@ -71,15 +71,11 @@ Aggregator::~Aggregator() Aggregator & Aggregator::create(Aggr aggr, Stash &stash) { - switch (aggr) { - case Aggr::AVG: return stash.create<Wrapper<aggr::Avg<double>>>(); - case Aggr::COUNT: return stash.create<Wrapper<aggr::Count<double>>>(); - case Aggr::PROD: return stash.create<Wrapper<aggr::Prod<double>>>(); - case Aggr::SUM: return stash.create<Wrapper<aggr::Sum<double>>>(); - case Aggr::MAX: return stash.create<Wrapper<aggr::Max<double>>>(); - case Aggr::MIN: return stash.create<Wrapper<aggr::Min<double>>>(); - } - LOG_ABORT("should not be reached"); + return TypifyAggr::resolve(aggr, [&stash](auto t)->Aggregator& + { + using T = typename decltype(t)::template templ<double>; + return stash.create<Wrapper<T>>(); + }); } std::vector<Aggr> diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index 8dea54d8abc..e7431c2c23b 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -118,5 +118,24 @@ public: }; } // namespave vespalib::eval::aggr + +struct TypifyAggr { + template <template<typename> typename A> struct Result { + static constexpr bool is_type = false; + template <typename T> using templ = A<T>; + }; + template <typename F> static decltype(auto) resolve(Aggr aggr, F &&f) { + switch (aggr) { + case Aggr::AVG: return f(Result<aggr::Avg>()); + case Aggr::COUNT: return f(Result<aggr::Count>()); + case Aggr::PROD: return f(Result<aggr::Prod>()); + case Aggr::SUM: return f(Result<aggr::Sum>()); + case Aggr::MAX: return f(Result<aggr::Max>()); + case Aggr::MIN: return f(Result<aggr::Min>()); + } + abort(); + } +}; + } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp index 9b93f5e7d72..84da53c8488 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp @@ -25,7 +25,7 @@ void my_cell_range_op(eval::InterpretedFunction::State &state, uint64_t param) { struct MyCellRangeOp { template <typename CT> - static auto get_fun() { return my_cell_range_op<CT>; } + static auto invoke() { return my_cell_range_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -46,7 +46,9 @@ DenseCellRangeFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(this)); assert(result_type().cell_type() == child().result_type().cell_type()); - auto op = select_1<MyCellRangeOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyCellRangeOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)this); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index c9ff57e4a65..9e30451cd67 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -48,7 +48,7 @@ void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint struct MyDotProductOp { template <typename LCT, typename RCT> - static auto get_fun() { return my_dot_product_op<LCT,RCT>; } + static auto invoke() { return my_dot_product_op<LCT,RCT>; } }; eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { @@ -60,7 +60,8 @@ eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { return my_cblas_float_dot_product_op; } } - return select_2<MyDotProductOp>(lct, rct); + using MyTypify = eval::TypifyCellType; + return typify_invoke<2,MyTypify,MyDotProductOp>(lct, rct); } } // namespace vespalib::tensor::<unnamed> diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp index b60d732d7a9..e373ca09e11 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp @@ -95,7 +95,7 @@ void my_compiled_lambda_op(eval::InterpretedFunction::State &state, uint64_t par struct MyCompiledLambdaOp { template <typename CT> - static auto get_fun() { return my_compiled_lambda_op<CT>; } + static auto invoke() { return my_compiled_lambda_op<CT>; } }; //----------------------------------------------------------------------------- @@ -131,7 +131,7 @@ void my_interpreted_lambda_op(eval::InterpretedFunction::State &state, uint64_t struct MyInterpretedLambdaOp { template <typename CT> - static auto get_fun() { return my_interpreted_lambda_op<CT>; } + static auto invoke() { return my_interpreted_lambda_op<CT>; } }; //----------------------------------------------------------------------------- @@ -163,15 +163,16 @@ DenseLambdaFunction::compile_self(const TensorEngine &engine, Stash &stash) cons { assert(&engine == &prod_engine); auto mode = eval_mode(); + using MyTypify = eval::TypifyCellType; if (mode == EvalMode::COMPILED) { CompiledParams ¶ms = stash.create<CompiledParams>(_lambda); - auto op = select_1<MyCompiledLambdaOp>(result_type().cell_type()); + auto op = typify_invoke<1,MyTypify,MyCompiledLambdaOp>(result_type().cell_type()); static_assert(sizeof(¶ms) == sizeof(uint64_t)); return Instruction(op, (uint64_t)(¶ms)); } else { assert(mode == EvalMode::INTERPRETED); InterpretedParams ¶ms = stash.create<InterpretedParams>(_lambda); - auto op = select_1<MyInterpretedLambdaOp>(result_type().cell_type()); + auto op = typify_invoke<1,MyTypify,MyInterpretedLambdaOp>(result_type().cell_type()); static_assert(sizeof(¶ms) == sizeof(uint64_t)); return Instruction(op, (uint64_t)(¶ms)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp index a5f532e643a..70bdc8ae7d6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp @@ -45,7 +45,7 @@ void my_lambda_peek_op(InterpretedFunction::State &state, uint64_t param) { struct MyLambdaPeekOp { template <typename DST_CT, typename SRC_CT> - static auto get_fun() { return my_lambda_peek_op<DST_CT, SRC_CT>; } + static auto invoke() { return my_lambda_peek_op<DST_CT, SRC_CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -64,7 +64,8 @@ InterpretedFunction::Instruction DenseLambdaPeekFunction::compile_self(const TensorEngine &, Stash &stash) const { const Self &self = stash.create<Self>(result_type(), *_idx_fun); - auto op = select_2<MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<2,MyTypify,MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); static_assert(sizeof(uint64_t) == sizeof(&self)); assert(child().result_type().is_dense()); return InterpretedFunction::Instruction(op, (uint64_t)&self); diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp index 695e0fddd08..9c18cf285d4 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -80,47 +80,6 @@ void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool lhs_common_inner, bool rhs_common_inner> -struct MyMatMulOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_matmul_op<LCT,RCT,lhs_common_inner,rhs_common_inner>; } -}; - -template <bool lhs_common_inner, bool rhs_common_inner> -eval::InterpretedFunction::op_function my_select3(CellType lct, CellType rct) -{ - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_matmul_op<lhs_common_inner,rhs_common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_matmul_op<lhs_common_inner,rhs_common_inner>; - } - } - return select_2<MyMatMulOp<lhs_common_inner,rhs_common_inner>>(lct, rct); -} - -template <bool lhs_common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct, - bool rhs_common_inner) -{ - if (rhs_common_inner) { - return my_select3<lhs_common_inner,true>(lct, rct); - } else { - return my_select3<lhs_common_inner,false>(lct, rct); - } -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, - bool lhs_common_inner, bool rhs_common_inner) -{ - if (lhs_common_inner) { - return my_select2<true>(lct, rct, rhs_common_inner); - } else { - return my_select2<false>(lct, rct, rhs_common_inner); - } -} - bool is_matrix(const ValueType &type) { return (type.is_dense() && (type.dimensions().size() == 2)); } @@ -160,6 +119,18 @@ const TensorFunction &create_matmul(const TensorFunction &a, const TensorFunctio } } +struct MyGetFun { + template<typename R1, typename R2, typename R3, typename R4> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_matmul_op<R3::value, R4::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_matmul_op<R3::value, R4::value>; + } else { + return my_matmul_op<R1, R2, R3::value, R4::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseMatMulFunction::Self::Self(const eval::ValueType &result_type_in, @@ -197,9 +168,11 @@ DenseMatMulFunction::~DenseMatMulFunction() = default; eval::InterpretedFunction::Instruction DenseMatMulFunction::compile_self(const TensorEngine &, Stash &stash) const { + using MyTypify = TypifyValue<eval::TypifyCellType,TypifyBool>; Self &self = stash.create<Self>(result_type(), _lhs_size, _common_size, _rhs_size); - auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type(), - _lhs_common_inner, _rhs_common_inner); + auto op = typify_invoke<4,MyTypify,MyGetFun>( + lhs().result_type().cell_type(), rhs().result_type().cell_type(), + _lhs_common_inner, _rhs_common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index 663993b6c26..571bcb79c9f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -2,6 +2,7 @@ #include "dense_single_reduce_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> namespace vespalib::tensor { @@ -12,6 +13,8 @@ using eval::TensorEngine; using eval::TensorFunction; using eval::Value; using eval::ValueType; +using eval::TypifyCellType; +using eval::TypifyAggr; using eval::as; using namespace eval::tensor_function; @@ -66,28 +69,13 @@ void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { state.pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); } -template <typename CT> -InterpretedFunction::op_function my_select_2(Aggr aggr) { - switch (aggr) { - case Aggr::AVG: return my_single_reduce_op<CT, Avg<CT>>; - case Aggr::COUNT: return my_single_reduce_op<CT, Count<CT>>; - case Aggr::PROD: return my_single_reduce_op<CT, Prod<CT>>; - case Aggr::SUM: return my_single_reduce_op<CT, Sum<CT>>; - case Aggr::MAX: return my_single_reduce_op<CT, Max<CT>>; - case Aggr::MIN: return my_single_reduce_op<CT, Min<CT>>; +struct MyGetFun { + template <typename R1, typename R2> static auto invoke() { + return my_single_reduce_op<R1, typename R2::template templ<R1>>; } - abort(); -} +}; -InterpretedFunction::op_function my_select(CellType cell_type, Aggr aggr) { - if (cell_type == ValueType::CellType::DOUBLE) { - return my_select_2<double>(aggr); - } - if (cell_type == ValueType::CellType::FLOAT) { - return my_select_2<float>(aggr); - } - abort(); -} +using MyTypify = TypifyValue<TypifyCellType,TypifyAggr>; bool check_input_type(const ValueType &type) { return (type.is_dense() && ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); @@ -109,7 +97,7 @@ DenseSingleReduceFunction::~DenseSingleReduceFunction() = default; InterpretedFunction::Instruction DenseSingleReduceFunction::compile_self(const TensorEngine &, Stash &stash) const { - auto op = my_select(result_type().cell_type(), _aggr); + auto op = typify_invoke<2,MyTypify,MyGetFun>(result_type().cell_type(), _aggr); auto ¶ms = stash.create<Params>(result_type(), child().result_type(), _dim_idx); static_assert(sizeof(uint64_t) == sizeof(¶ms)); return InterpretedFunction::Instruction(op, (uint64_t)¶ms); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp index 3533ab20175..7e887d4df34 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp @@ -34,7 +34,7 @@ void my_tensor_create_op(eval::InterpretedFunction::State &state, uint64_t param struct MyTensorCreateOp { template <typename CT> - static auto get_fun() { return my_tensor_create_op<CT>; } + static auto invoke() { return my_tensor_create_op<CT>; } }; size_t get_index(const TensorSpec::Address &addr, const ValueType &type) { @@ -72,7 +72,9 @@ eval::InterpretedFunction::Instruction DenseTensorCreateFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_self)); - auto op = select_1<MyTensorCreateOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorCreateOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_self); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp index 5cb1cbfd88f..16c0b01b169 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp @@ -44,7 +44,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param) struct MyTensorPeekOp { template <typename CT> - static auto get_fun() { return my_tensor_peek_op<CT>; } + static auto invoke() { return my_tensor_peek_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -71,7 +71,8 @@ eval::InterpretedFunction::Instruction DenseTensorPeekFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_spec)); - auto op = select_1<MyTensorPeekOp>(_children[0].get().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorPeekOp>(_children[0].get().result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_spec); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index a0d63a1ce1e..968308d69c9 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -76,33 +76,6 @@ void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint6 state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool common_inner> -struct MyXWProductOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; } -}; - -template <bool common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) { - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_xw_product_op<common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_xw_product_op<common_inner>; - } - } - return select_2<MyXWProductOp<common_inner>>(lct, rct); -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) { - if (common_inner) { - return my_select2<true>(lct, rct); - } else { - return my_select2<false>(lct, rct); - } -} - bool isDenseTensor(const ValueType &type, size_t d) { return (type.is_dense() && (type.dimensions().size() == d)); } @@ -132,6 +105,18 @@ const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFun common_inner); } +struct MyXWProductOp { + template<typename R1, typename R2, typename R3> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_xw_product_op<R3::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_xw_product_op<R3::value>; + } else { + return my_xw_product_op<R1, R2, R3::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in, @@ -160,8 +145,10 @@ eval::InterpretedFunction::Instruction DenseXWProductFunction::compile_self(const TensorEngine &, Stash &stash) const { Self &self = stash.create<Self>(result_type(), _vector_size, _result_size); - auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _common_inner); + using MyTypify = TypifyValue<eval::TypifyCellType,vespalib::TypifyBool>; + auto op = typify_invoke<3,MyTypify,MyXWProductOp>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + _common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index 7a4b5917f00..57f727f7968 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -19,7 +19,7 @@ namespace { struct CallVectorFromDoubles { template <typename CT> static TypedCells - call(eval::InterpretedFunction::State &state, size_t numCells) { + invoke(eval::InterpretedFunction::State &state, size_t numCells) { ArrayRef<CT> outputCells = state.stash.create_array<CT>(numCells); for (size_t i = numCells; i-- > 0; ) { outputCells[i] = (CT) state.peek(0).as_double(); @@ -33,7 +33,8 @@ void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t const auto *self = (const VectorFromDoublesFunction::Self *)(param); CellType ct = self->resultType.cell_type(); size_t numCells = self->resultSize; - TypedCells cells = dispatch_0<CallVectorFromDoubles>(ct, state, numCells); + using MyTypify = eval::TypifyCellType; + TypedCells cells = typify_invoke<1,MyTypify,CallVectorFromDoubles>(ct, state, numCells); const Value &result = state.stash.create<DenseTensorView>(self->resultType, cells); state.stack.emplace_back(result); } |