From 8753ece3f5f374a5cffd360f2009ef1de9ec9d35 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Wed, 8 Nov 2023 13:55:05 +0000 Subject: enable nested ctf meta data --- eval/src/apps/eval_expr/eval_expr.cpp | 21 ++++++--- .../interpreted_function_test.cpp | 20 +++++---- .../dense_replace_type_function_test.cpp | 4 +- .../instruction/generic_peek/generic_peek_test.cpp | 2 +- .../universal_dot_product_test.cpp | 50 ++++++++++++---------- .../instruction_benchmark.cpp | 20 ++++----- .../vespa/eval/eval/compile_tensor_function.cpp | 40 +++++++++++------ eval/src/vespa/eval/eval/compile_tensor_function.h | 28 ++++++++++-- eval/src/vespa/eval/eval/interpreted_function.cpp | 27 +++++++----- eval/src/vespa/eval/eval/interpreted_function.h | 33 +++++++++++--- .../vespa/eval/eval/optimize_tensor_function.cpp | 9 ++-- .../src/vespa/eval/eval/optimize_tensor_function.h | 5 ++- eval/src/vespa/eval/eval/tensor_function.cpp | 50 +++++++++++----------- eval/src/vespa/eval/eval/tensor_function.h | 37 ++++++++-------- eval/src/vespa/eval/eval/test/eval_fixture.cpp | 21 ++++++--- eval/src/vespa/eval/eval/test/eval_fixture.h | 10 ++--- .../eval/instruction/best_similarity_function.cpp | 4 +- .../eval/instruction/best_similarity_function.h | 2 +- .../eval/instruction/dense_cell_range_function.cpp | 2 +- .../eval/instruction/dense_cell_range_function.h | 2 +- .../instruction/dense_dot_product_function.cpp | 2 +- .../eval/instruction/dense_dot_product_function.h | 2 +- .../eval/instruction/dense_hamming_distance.cpp | 2 +- .../eval/instruction/dense_hamming_distance.h | 2 +- .../instruction/dense_lambda_peek_function.cpp | 4 +- .../eval/instruction/dense_lambda_peek_function.h | 2 +- .../eval/instruction/dense_matmul_function.cpp | 4 +- .../vespa/eval/instruction/dense_matmul_function.h | 2 +- .../instruction/dense_multi_matmul_function.cpp | 2 +- .../eval/instruction/dense_multi_matmul_function.h | 2 +- .../instruction/dense_simple_expand_function.cpp | 4 +- .../instruction/dense_simple_expand_function.h | 2 +- .../instruction/dense_single_reduce_function.cpp | 4 +- .../instruction/dense_single_reduce_function.h | 2 +- .../instruction/dense_tensor_create_function.cpp | 2 +- .../instruction/dense_tensor_create_function.h | 2 +- .../instruction/dense_tensor_peek_function.cpp | 2 +- .../eval/instruction/dense_tensor_peek_function.h | 2 +- .../eval/instruction/dense_xw_product_function.cpp | 4 +- .../eval/instruction/dense_xw_product_function.h | 2 +- eval/src/vespa/eval/instruction/generic_lambda.cpp | 9 ++-- eval/src/vespa/eval/instruction/generic_lambda.h | 3 +- .../eval/instruction/generic_map_subspaces.cpp | 9 ++-- .../vespa/eval/instruction/generic_map_subspaces.h | 3 +- .../eval/instruction/inplace_map_function.cpp | 2 +- .../vespa/eval/instruction/inplace_map_function.h | 2 +- .../eval/instruction/join_with_number_function.cpp | 4 +- .../eval/instruction/join_with_number_function.h | 2 +- eval/src/vespa/eval/instruction/l2_distance.cpp | 2 +- eval/src/vespa/eval/instruction/l2_distance.h | 2 +- eval/src/vespa/eval/instruction/mapped_lookup.cpp | 2 +- eval/src/vespa/eval/instruction/mapped_lookup.h | 2 +- .../eval/instruction/mixed_112_dot_product.cpp | 2 +- .../vespa/eval/instruction/mixed_112_dot_product.h | 2 +- .../instruction/mixed_inner_product_function.cpp | 4 +- .../instruction/mixed_inner_product_function.h | 2 +- .../vespa/eval/instruction/mixed_l2_distance.cpp | 4 +- .../src/vespa/eval/instruction/mixed_l2_distance.h | 2 +- .../instruction/mixed_simple_join_function.cpp | 4 +- .../eval/instruction/mixed_simple_join_function.h | 2 +- .../eval/instruction/replace_type_function.cpp | 2 +- .../vespa/eval/instruction/replace_type_function.h | 2 +- .../vespa/eval/instruction/simple_join_count.cpp | 2 +- .../src/vespa/eval/instruction/simple_join_count.h | 2 +- .../eval/instruction/sparse_112_dot_product.cpp | 2 +- .../eval/instruction/sparse_112_dot_product.h | 2 +- .../instruction/sparse_dot_product_function.cpp | 2 +- .../eval/instruction/sparse_dot_product_function.h | 2 +- .../sparse_full_overlap_join_function.cpp | 4 +- .../sparse_full_overlap_join_function.h | 2 +- .../eval/instruction/sparse_merge_function.cpp | 8 ++-- .../vespa/eval/instruction/sparse_merge_function.h | 2 +- .../sparse_no_overlap_join_function.cpp | 8 ++-- .../instruction/sparse_no_overlap_join_function.h | 2 +- .../eval/instruction/sparse_singledim_lookup.cpp | 2 +- .../eval/instruction/sparse_singledim_lookup.h | 2 +- .../instruction/sum_max_dot_product_function.cpp | 2 +- .../instruction/sum_max_dot_product_function.h | 2 +- .../eval/instruction/universal_dot_product.cpp | 4 +- .../vespa/eval/instruction/universal_dot_product.h | 2 +- .../eval/instruction/unpack_bits_function.cpp | 2 +- .../vespa/eval/instruction/unpack_bits_function.h | 2 +- .../instruction/vector_from_doubles_function.cpp | 2 +- .../instruction/vector_from_doubles_function.h | 2 +- 84 files changed, 328 insertions(+), 237 deletions(-) diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index 23af5a926c4..ab597070329 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -132,18 +132,15 @@ public: } return {}; } - vespalib::Stash stash; - const TensorFunction &plain_fun = make_tensor_function(factory, fun->root(), types, stash); - const TensorFunction &optimized = optimize_tensor_function(factory, plain_fun, stash); Value::UP result; if (_verbose) { - InterpretedFunction ifun(factory, optimized, &_meta); + auto ifun = InterpretedFunction::opts(factory).meta(&_meta).make(fun->root(), types); REQUIRE_EQ(_meta.steps.size(), ifun.program_size()); InterpretedFunction::ProfiledContext ctx(ifun); result = factory.copy(ifun.eval(ctx, params)); _cost = ctx.cost; } else { - InterpretedFunction ifun(factory, optimized, nullptr); + InterpretedFunction ifun(factory, *fun, types); InterpretedFunction::Context ctx(ifun); result = factory.copy(ifun.eval(ctx, params)); } @@ -190,6 +187,17 @@ void print_error(const vespalib::string &error) { fprintf(stderr, "error: %s\n", error.c_str()); } +void print_nested(const CTFMetaData &meta, const vespalib::string &indent) { + const auto &steps = meta.steps; + for (size_t i = 0; i < steps.size(); ++i) { + fprintf(stderr, "%s class: %s\n", indent.c_str(), steps[i].class_name.c_str()); + fprintf(stderr, "%s symbol: %s\n", indent.c_str(), steps[i].symbol_name.c_str()); + if (steps[i].nested) { + print_nested(*steps[i].nested, indent + " "); + } + } +} + void print_value(const Value &value, const vespalib::string &name, const CTFMetaData &meta, const CostProfile &cost) { bool with_name = !name.empty(); bool with_meta = !meta.steps.empty(); @@ -206,6 +214,9 @@ void print_value(const Value &value, const vespalib::string &name, const CTFMeta fprintf(stderr, " symbol: %s\n", steps[i].symbol_name.c_str()); fprintf(stderr, " count: %zu\n", cost[i].first); fprintf(stderr, " time_us: %g\n", vespalib::count_ns(cost[i].second)/1000.0); + if (steps[i].nested) { + print_nested(*steps[i].nested, " "); + } } } if (with_name) { diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index 4ba715ea192..ac7e0f6d126 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -199,16 +199,18 @@ TEST("require that functions with non-interpretable complex lambdas cannot be in //----------------------------------------------------------------------------- TEST("require that compilation meta-data can be collected") { - Stash stash; - const auto &x2 = tensor_function::inject(ValueType::from_spec("tensor(x[2])"), 0, stash); - const auto &x3 = tensor_function::inject(ValueType::from_spec("tensor(x[3])"), 1, stash); - const auto &concat_x5 = tensor_function::concat(x3, x2, "x", stash); - const auto &x5 = tensor_function::inject(ValueType::from_spec("tensor(x[5])"), 2, stash); - const auto &mapped_x5 = tensor_function::map(x5, operation::Relu::f, stash); - const auto &flag = tensor_function::inject(ValueType::from_spec("double"), 0, stash); - const auto &root = tensor_function::if_node(flag, concat_x5, mapped_x5, stash); + auto fun = Function::parse("if(flag,concat(x2,x3,x),map(x5,f(x)(relu(x))))"); + fprintf(stderr, "%s\n", fun->dump_as_lambda().c_str()); + ASSERT_TRUE(fun->dump_as_lambda().starts_with("f(flag,x2,x3,x5)")); + std::vector param_types({ValueType::from_spec("double"), + ValueType::from_spec("tensor(x[2])"), + ValueType::from_spec("tensor(x[3])"), + ValueType::from_spec("tensor(x[5])")}); + NodeTypes types(*fun, param_types); + ASSERT_FALSE(types.get_type(fun->root()).is_error()); + ASSERT_TRUE(types.errors().empty()); CTFMetaData meta; - InterpretedFunction ifun(FastValueBuilderFactory::get(), root, &meta); + auto ifun = InterpretedFunction::opts(FastValueBuilderFactory::get()).meta(&meta).make(fun->root(), types); fprintf(stderr, "compilation meta-data:\n"); for (const auto &step: meta.steps) { fprintf(stderr, " %s -> %s\n", step.class_name.c_str(), step.symbol_name.c_str()); diff --git a/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp b/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp index 0cb5a821136..748f38a3343 100644 --- a/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp +++ b/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp @@ -22,7 +22,7 @@ struct ChildMock : Leaf { bool is_mutable; ChildMock(const ValueType &type) : Leaf(type), is_mutable(true) {} bool result_is_mutable() const override { return is_mutable; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &, Stash &) const override { abort(); } + InterpretedFunction::Instruction compile_self(const CTFContext &) const override { abort(); } }; struct Fixture { @@ -42,7 +42,7 @@ struct Fixture { { my_fun.push_children(children); state.stack.push_back(*my_value); - my_fun.compile_self(prod_factory, state.stash).perform(state); + my_fun.compile_self(CTFContext(prod_factory, state.stash, nullptr)).perform(state); ASSERT_EQUAL(children.size(), 1u); ASSERT_EQUAL(state.stack.size(), 1u); ASSERT_TRUE(!new_type.is_error()); diff --git a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp index 7f25edd50c8..df1661cc57f 100644 --- a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp +++ b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp @@ -115,7 +115,7 @@ TensorSpec tensor_function_peek(const TensorSpec &a, const ValueType &result_typ } const auto &func_param = tensor_function::inject(param->type(), 0, stash); const auto &peek_node = tensor_function::peek(func_param, func_spec, stash); - auto my_op = peek_node.compile_self(factory, stash); + auto my_op = peek_node.compile_self(CTFContext(factory, stash, nullptr)); InterpretedFunction::EvalSingle single(factory, my_op); return spec_from_value(single.eval(my_stack)); } diff --git a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp index 99809601d9a..6b72dd9ca06 100644 --- a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp +++ b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp @@ -75,6 +75,7 @@ class Optimize private: struct ctor_tag{}; public: + using optimize_fun_t = InterpretedFunction::Options::optimize_fun_t; enum class With { NONE, CUSTOM, PROD, SPECIFIC }; With with; vespalib::string name; @@ -92,6 +93,29 @@ public: static Optimize specific(const vespalib::string &name_in, tensor_function_optimizer optimizer_in) { return {With::SPECIFIC, name_in, {}, optimizer_in, {}}; } + optimize_fun_t make_optimize_fun() const { + switch (with) { + case Optimize::With::NONE: return do_not_optimize_tensor_function; + case Optimize::With::PROD: return optimize_tensor_function; + case Optimize::With::CUSTOM: + return [options=options](const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash) + ->const TensorFunction & + { + return optimize_tensor_function_impl(factory, function, stash, options); + }; + case Optimize::With::SPECIFIC: + return [optimizer=optimizer](const ValueBuilderFactory &, const TensorFunction &function, Stash &stash) + ->const TensorFunction & + { + size_t count = 0; + const auto &result = apply_tensor_function_optimizer(function, optimizer, stash, + [&count](const auto &)noexcept{ ++count; }); + EXPECT_EQ(count, 1); + return result; + }; + } + abort(); + } ~Optimize(); }; Optimize::~Optimize() = default; @@ -201,29 +225,11 @@ void benchmark(const vespalib::string &expr, std::vector list) { } NodeTypes node_types(*fun, param_types); ASSERT_FALSE(node_types.get_type(fun->root()).is_error()); - Stash stash; - const TensorFunction &plain_fun = make_tensor_function(prod_factory, fun->root(), node_types, stash); - const TensorFunction *optimized = nullptr; - switch (optimize.with) { - case Optimize::With::NONE: - optimized = std::addressof(plain_fun); - break; - case Optimize::With::PROD: - optimized = std::addressof(optimize_tensor_function(prod_factory, plain_fun, stash)); - break; - case Optimize::With::CUSTOM: - optimized = std::addressof(optimize_tensor_function(prod_factory, plain_fun, stash, optimize.options)); - break; - case Optimize::With::SPECIFIC: - size_t count = 0; - optimized = std::addressof(apply_tensor_function_optimizer(plain_fun, optimize.optimizer, stash, - [&count](const auto &)noexcept{ ++count; })); - ASSERT_EQ(count, 1); - break; - } - ASSERT_NE(optimized, nullptr); CTFMetaData ctf_meta; - InterpretedFunction ifun(prod_factory, *optimized, &ctf_meta); + auto ifun = InterpretedFunction::opts(prod_factory) + .optimize(optimize.make_optimize_fun()) + .meta(&ctf_meta) + .make(fun->root(), node_types); InterpretedFunction::ProfiledContext pctx(ifun); ASSERT_EQ(ctf_meta.steps.size(), ifun.program_size()); std::vector prev_time(ctf_meta.steps.size(), duration::zero()); diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 733acbc09bf..9fa2ef3d56f 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -97,7 +97,7 @@ void my_multi_instruction_op(InterpretedFunction::State &state, uint64_t param_i void collect_op1_chain(const TensorFunction &node, const ValueBuilderFactory &factory, Stash &stash, std::vector &list) { if (auto op1 = as(node)) { collect_op1_chain(op1->child(), factory, stash, list); - list.push_back(node.compile_self(factory, stash)); + list.push_back(node.compile_self(CTFContext(factory, stash, nullptr))); } } @@ -125,7 +125,7 @@ struct Impl { const auto &rhs_node = tensor_function::inject(rhs, 1, stash); const auto &join_node = tensor_function::join(lhs_node, rhs_node, function, stash); const auto &node = optimize ? optimize_tensor_function(factory, join_node, stash) : join_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_reduce(const ValueType &lhs, Aggr aggr, const std::vector &dims, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction @@ -142,7 +142,7 @@ struct Impl { const auto &lhs_node = tensor_function::inject(lhs, 0, stash); const auto &rename_node = tensor_function::rename(lhs_node, from, to, stash); const auto &node = optimize ? optimize_tensor_function(factory, rename_node, stash) : rename_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_merge(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction @@ -150,23 +150,23 @@ struct Impl { const auto &rhs_node = tensor_function::inject(rhs, 1, stash); const auto &merge_node = tensor_function::merge(lhs_node, rhs_node, function, stash); const auto &node = optimize ? optimize_tensor_function(factory, merge_node, stash) : merge_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_concat(const ValueType &lhs, const ValueType &rhs, const std::string &dimension, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction const auto &lhs_node = tensor_function::inject(lhs, 0, stash); const auto &rhs_node = tensor_function::inject(rhs, 1, stash); const auto &concat_node = tensor_function::concat(lhs_node, rhs_node, dimension, stash); - return concat_node.compile_self(factory, stash); + return concat_node.compile_self(CTFContext(factory, stash, nullptr)); const auto &node = optimize ? optimize_tensor_function(factory, concat_node, stash) : concat_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_map(const ValueType &lhs, operation::op1_t function, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction const auto &lhs_node = tensor_function::inject(lhs, 0, stash); const auto &map_node = tensor_function::map(lhs_node, function, stash); const auto &node = optimize ? optimize_tensor_function(factory, map_node, stash) : map_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_tensor_create(const ValueType &proto_type, const TensorSpec &proto, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction @@ -177,7 +177,7 @@ struct Impl { } const auto &create_tensor_node = tensor_function::create(proto_type, spec, stash); const auto &node = optimize ? optimize_tensor_function(factory, create_tensor_node, stash) : create_tensor_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_tensor_lambda(const ValueType &type, const Function &function, const ValueType &p0_type, Stash &stash) const { std::vector arg_types(type.dimensions().size(), ValueType::double_type()); @@ -186,7 +186,7 @@ struct Impl { EXPECT_EQ(types.errors(), std::vector()); const auto &tensor_lambda_node = tensor_function::lambda(type, {0}, function, std::move(types), stash); const auto &node = optimize ? optimize_tensor_function(factory, tensor_lambda_node, stash) : tensor_lambda_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } Instruction create_tensor_peek(const ValueType &type, const MyPeekSpec &my_spec, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction @@ -210,7 +210,7 @@ struct Impl { } const auto &peek_node = tensor_function::peek(my_param, spec, stash); const auto &node = optimize ? optimize_tensor_function(factory, peek_node, stash) : peek_node; - return node.compile_self(factory, stash); + return node.compile_self(CTFContext(factory, stash, nullptr)); } }; diff --git a/eval/src/vespa/eval/eval/compile_tensor_function.cpp b/eval/src/vespa/eval/eval/compile_tensor_function.cpp index 425910d6249..02a38f7ad42 100644 --- a/eval/src/vespa/eval/eval/compile_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/compile_tensor_function.cpp @@ -35,18 +35,23 @@ struct Frame { }; struct ProgramCompiler { - const ValueBuilderFactory &factory; - Stash &stash; + CTFContext ctx; std::vector stack; std::vector prog; - CTFMetaData *meta; - ProgramCompiler(const ValueBuilderFactory &factory_in, Stash &stash_in, CTFMetaData *meta_in) - : factory(factory_in), stash(stash_in), stack(), prog(), meta(meta_in) {} + ProgramCompiler(const CTFContext &ctx_in) + : ctx(ctx_in), stack(), prog() {} ~ProgramCompiler(); + void add_meta(const TensorFunction &node, const Instruction &instr, CTFMetaData *nested) { + ctx.meta->steps.emplace_back(getClassName(node), instr.resolve_symbol()); + if (nested) { + ctx.meta->steps.back().nested = nested->extract(); + } + } + void maybe_add_meta(const TensorFunction &node, const Instruction &instr) { - if (meta != nullptr) { - meta->steps.emplace_back(getClassName(node), instr.resolve_symbol()); + if (ctx.meta != nullptr) { + add_meta(node, instr, nullptr); } } @@ -56,11 +61,11 @@ struct ProgramCompiler { void open(const TensorFunction &node) { if (auto if_node = as(node)) { - append(compile_tensor_function(factory, if_node->cond(), stash, meta)); + append(compile_tensor_function(if_node->cond(), ctx)); maybe_add_meta(node, Instruction(op_skip_if_false)); - auto true_prog = compile_tensor_function(factory, if_node->true_child(), stash, meta); + auto true_prog = compile_tensor_function(if_node->true_child(), ctx); maybe_add_meta(node, Instruction(op_skip)); - auto false_prog = compile_tensor_function(factory, if_node->false_child(), stash, meta); + auto false_prog = compile_tensor_function(if_node->false_child(), ctx); true_prog.emplace_back(op_skip, false_prog.size()); prog.emplace_back(op_skip_if_false, true_prog.size()); append(true_prog); @@ -71,8 +76,15 @@ struct ProgramCompiler { } void close(const TensorFunction &node) { - prog.push_back(node.compile_self(factory, stash)); - maybe_add_meta(node, prog.back()); + if (ctx.meta == nullptr) { + prog.push_back(node.compile_self(ctx)); + } else { + CTFMetaData sub_meta; + CTFContext sub_context = ctx; + sub_context.meta = &sub_meta; + prog.push_back(node.compile_self(sub_context)); + add_meta(node, prog.back(), &sub_meta); + } } std::vector compile(const TensorFunction &function) { @@ -94,8 +106,8 @@ ProgramCompiler::~ProgramCompiler() = default; CTFMetaData::~CTFMetaData() = default; -std::vector compile_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, CTFMetaData *meta) { - ProgramCompiler compiler(factory, stash, meta); +std::vector compile_tensor_function(const TensorFunction &function, const CTFContext &ctx) { + ProgramCompiler compiler(ctx); return compiler.compile(function); } diff --git a/eval/src/vespa/eval/eval/compile_tensor_function.h b/eval/src/vespa/eval/eval/compile_tensor_function.h index f6edfe49052..dcc2ee58da8 100644 --- a/eval/src/vespa/eval/eval/compile_tensor_function.h +++ b/eval/src/vespa/eval/eval/compile_tensor_function.h @@ -27,18 +27,38 @@ struct CTFMetaData { struct Step { vespalib::string class_name; vespalib::string symbol_name; + std::unique_ptr nested; Step(vespalib::string &&class_name_in, vespalib::string &&symbol_name_in) noexcept - : class_name(std::move(class_name_in)), - symbol_name(std::move(symbol_name_in)) + : class_name(std::move(class_name_in)), + symbol_name(std::move(symbol_name_in)), + nested() { } }; std::vector steps; + CTFMetaData() noexcept = default; + CTFMetaData(const CTFMetaData &) = delete; + CTFMetaData(CTFMetaData &&) noexcept = default; + CTFMetaData &operator=(const CTFMetaData &) = delete; + CTFMetaData &operator=(CTFMetaData &&) noexcept = default; + std::unique_ptr extract() { + return steps.empty() + ? std::unique_ptr(nullptr) + : std::make_unique(std::move(*this)); + } ~CTFMetaData(); }; -std::vector compile_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, - CTFMetaData *meta); +struct CTFContext { + const ValueBuilderFactory &factory; + Stash &stash; + CTFMetaData *meta; + constexpr CTFContext(const CTFContext &) noexcept = default; + constexpr CTFContext(const ValueBuilderFactory &factory_in, Stash &stash_in, CTFMetaData *meta_in) noexcept + : factory(factory_in), stash(stash_in), meta(meta_in) {} +}; + +std::vector compile_tensor_function(const TensorFunction &function, const CTFContext &ctx); } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index c0aa7d1703b..46f003224d0 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -95,22 +95,29 @@ InterpretedFunction::Instruction::nop() return Instruction(my_nop); } -InterpretedFunction::InterpretedFunction(const ValueBuilderFactory &factory, const TensorFunction &function, CTFMetaData *meta) - : _program(), - _stash(), - _factory(factory) +InterpretedFunction::Options::Options(const ValueBuilderFactory &factory_in) + : _factory(factory_in), + _optimize(optimize_tensor_function), + _meta(nullptr) +{ +} + +InterpretedFunction::InterpretedFunction(const ValueBuilderFactory &factory, const TensorFunction &function) + : _program(), + _stash(), + _factory(factory) { - _program = compile_tensor_function(factory, function, _stash, meta); + _program = compile_tensor_function(function, CTFContext(factory, _stash, nullptr)); } -InterpretedFunction::InterpretedFunction(const ValueBuilderFactory &factory, const nodes::Node &root, const NodeTypes &types) +InterpretedFunction::InterpretedFunction(const Options &opts, const nodes::Node &root, const NodeTypes &types) : _program(), _stash(), - _factory(factory) + _factory(opts.factory()) { - const TensorFunction &plain_fun = make_tensor_function(factory, root, types, _stash); - const TensorFunction &optimized = optimize_tensor_function(factory, plain_fun, _stash); - _program = compile_tensor_function(factory, optimized, _stash, nullptr); + const TensorFunction &plain_fun = make_tensor_function(opts.factory(), root, types, _stash); + const TensorFunction &optimized = opts.optimize()(opts.factory(), plain_fun, _stash); + _program = compile_tensor_function(optimized, CTFContext(opts.factory(), _stash, opts.meta())); } InterpretedFunction::~InterpretedFunction() = default; diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index 4d4c77f1116..bd54cccb7b6 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -8,6 +8,7 @@ #include "lazy_params.h" #include #include +#include namespace vespalib::eval { @@ -99,7 +100,26 @@ public: } static Instruction nop(); }; - + + class Options { + public: + using optimize_fun_t = std::function; + private: + const ValueBuilderFactory &_factory; + optimize_fun_t _optimize; + CTFMetaData *_meta; + public: + Options(const ValueBuilderFactory &factory_in); + const ValueBuilderFactory &factory() const noexcept { return _factory; } + Options &meta(CTFMetaData *value) noexcept { _meta = value; return *this; } + CTFMetaData *meta() const noexcept { return _meta; } + Options &optimize(optimize_fun_t value) { _optimize = std::move(value); return *this; } + optimize_fun_t optimize() const { return _optimize; } + InterpretedFunction make(const nodes::Node &root, const NodeTypes &types) const { + return InterpretedFunction(*this, root, types); + } + }; + private: std::vector _program; Stash _stash; @@ -107,13 +127,12 @@ private: public: using UP = std::unique_ptr; - // for testing; use with care; the tensor function must be kept alive - InterpretedFunction(const ValueBuilderFactory &factory, const TensorFunction &function, CTFMetaData *meta); - InterpretedFunction(const ValueBuilderFactory &factory, const TensorFunction &function) - : InterpretedFunction(factory, function, nullptr) {} - InterpretedFunction(const ValueBuilderFactory &factory, const nodes::Node &root, const NodeTypes &types); + static Options opts(const ValueBuilderFactory &factory) { return Options(factory); } + // for testing; make sure to keep tensor function alive + InterpretedFunction(const ValueBuilderFactory &factory, const TensorFunction &function); + InterpretedFunction(const Options &opts, const nodes::Node &root, const NodeTypes &types); InterpretedFunction(const ValueBuilderFactory &factory, const Function &function, const NodeTypes &types) - : InterpretedFunction(factory, function.root(), types) {} + : InterpretedFunction(opts(factory), function.root(), types) {} InterpretedFunction(InterpretedFunction &&rhs) = default; ~InterpretedFunction(); size_t program_size() const { return _program.size(); } diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 434ed61ff25..bfb68d63c67 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -127,8 +127,8 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te } // namespace vespalib::eval:: -const TensorFunction &optimize_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, - const OptimizeTensorFunctionOptions &options) +const TensorFunction &optimize_tensor_function_impl(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, + const OptimizeTensorFunctionOptions &options) { LOG(debug, "tensor function before optimization:\n%s\n", function.as_string().c_str()); const TensorFunction &optimized = optimize_for_factory(factory, function, stash, options); @@ -137,7 +137,10 @@ const TensorFunction &optimize_tensor_function(const ValueBuilderFactory &factor } const TensorFunction &optimize_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash) { - return optimize_tensor_function(factory, function, stash, OptimizeTensorFunctionOptions()); + return optimize_tensor_function_impl(factory, function, stash, OptimizeTensorFunctionOptions()); +} +const TensorFunction &do_not_optimize_tensor_function(const ValueBuilderFactory &, const TensorFunction &function, Stash &) { + return function; } const TensorFunction &apply_tensor_function_optimizer(const TensorFunction &function, tensor_function_optimizer optimizer, Stash &stash, tensor_function_listener listener) { diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.h b/eval/src/vespa/eval/eval/optimize_tensor_function.h index a164330e698..9a59c8db3a5 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.h +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.h @@ -17,9 +17,10 @@ struct OptimizeTensorFunctionOptions { struct ValueBuilderFactory; struct TensorFunction; -const TensorFunction &optimize_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, - const OptimizeTensorFunctionOptions &options); +const TensorFunction &optimize_tensor_function_impl(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash, + const OptimizeTensorFunctionOptions &options); const TensorFunction &optimize_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash); +const TensorFunction &do_not_optimize_tensor_function(const ValueBuilderFactory &factory, const TensorFunction &function, Stash &stash); using tensor_function_optimizer = std::function; using tensor_function_listener = std::function; diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 14d486aeb48..43879fca143 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -109,7 +109,7 @@ Op2::visit_children(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -ConstValue::compile_self(const ValueBuilderFactory &, Stash &) const +ConstValue::compile_self(const CTFContext &) const { return Instruction(op_load_const, wrap_param(_value)); } @@ -128,7 +128,7 @@ ConstValue::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Inject::compile_self(const ValueBuilderFactory &, Stash &) const +Inject::compile_self(const CTFContext &) const { return Instruction::fetch_param(_param_idx); } @@ -143,9 +143,9 @@ Inject::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Reduce::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Reduce::compile_self(const CTFContext &ctx) const { - return instruction::GenericReduce::make_instruction(result_type(), child().result_type(), aggr(), dimensions(), factory, stash); + return instruction::GenericReduce::make_instruction(result_type(), child().result_type(), aggr(), dimensions(), ctx.factory, ctx.stash); } void @@ -159,9 +159,9 @@ Reduce::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Map::compile_self(const ValueBuilderFactory &, Stash &stash) const +Map::compile_self(const CTFContext &ctx) const { - return instruction::GenericMap::make_instruction(result_type(), child().result_type(), _function, stash); + return instruction::GenericMap::make_instruction(result_type(), child().result_type(), _function, ctx.stash); } void @@ -174,9 +174,9 @@ Map::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- InterpretedFunction::Instruction -MapSubspaces::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +MapSubspaces::compile_self(const CTFContext &ctx) const { - return instruction::GenericMapSubspaces::make_instruction(*this, factory, stash); + return instruction::GenericMapSubspaces::make_instruction(*this, ctx.factory, ctx.stash, ctx.meta); } void @@ -188,9 +188,9 @@ MapSubspaces::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Join::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Join::compile_self(const CTFContext &ctx) const { - return instruction::GenericJoin::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), function(), factory, stash); + return instruction::GenericJoin::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), function(), ctx.factory, ctx.stash); } void @@ -203,9 +203,9 @@ Join::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Merge::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Merge::compile_self(const CTFContext &ctx) const { - return instruction::GenericMerge::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), function(), factory, stash); + return instruction::GenericMerge::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), function(), ctx.factory, ctx.stash); } void @@ -218,9 +218,9 @@ Merge::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Concat::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Concat::compile_self(const CTFContext &ctx) const { - return instruction::GenericConcat::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), dimension(), factory, stash); + return instruction::GenericConcat::make_instruction(result_type(), lhs().result_type(), rhs().result_type(), dimension(), ctx.factory, ctx.stash); } void @@ -233,9 +233,9 @@ Concat::visit_self(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- InterpretedFunction::Instruction -CellCast::compile_self(const ValueBuilderFactory &, Stash &stash) const +CellCast::compile_self(const CTFContext &ctx) const { - return instruction::GenericCellCast::make_instruction(result_type(), child().result_type(), cell_type(), stash); + return instruction::GenericCellCast::make_instruction(result_type(), child().result_type(), cell_type(), ctx.stash); } void @@ -267,9 +267,9 @@ Create::make_spec() const } Instruction -Create::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Create::compile_self(const CTFContext &ctx) const { - return instruction::GenericCreate::make_instruction(result_type(), make_spec(), factory, stash); + return instruction::GenericCreate::make_instruction(result_type(), make_spec(), ctx.factory, ctx.stash); } void @@ -329,9 +329,9 @@ Lambda::create_spec_impl(const ValueType &type, const LazyParams ¶ms, const } InterpretedFunction::Instruction -Lambda::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Lambda::compile_self(const CTFContext &ctx) const { - return instruction::GenericLambda::make_instruction(*this, factory, stash); + return instruction::GenericLambda::make_instruction(*this, ctx.factory, ctx.stash, ctx.meta); } void @@ -379,9 +379,9 @@ Peek::make_spec() const } Instruction -Peek::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Peek::compile_self(const CTFContext &ctx) const { - return instruction::GenericPeek::make_instruction(result_type(), param_type(), make_spec(), factory, stash); + return instruction::GenericPeek::make_instruction(result_type(), param_type(), make_spec(), ctx.factory, ctx.stash); } void @@ -408,9 +408,9 @@ Peek::visit_children(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- Instruction -Rename::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +Rename::compile_self(const CTFContext &ctx) const { - return instruction::GenericRename::make_instruction(result_type(), child().result_type(), from(), to(), factory, stash); + return instruction::GenericRename::make_instruction(result_type(), child().result_type(), from(), to(), ctx.factory, ctx.stash); } void @@ -431,7 +431,7 @@ If::push_children(std::vector &children) const } Instruction -If::compile_self(const ValueBuilderFactory &, Stash &) const +If::compile_self(const CTFContext &) const { // 'if' is handled directly by compile_tensor_function to enable // lazy-evaluation of true/false sub-expressions. diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index 2c703fbdfef..68ccc7c41e0 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -8,7 +8,7 @@ #include "value_type.h" #include "value.h" #include "aggr.h" -#include "interpreted_function.h" +#include "compile_tensor_function.h" #include "wrap_param.h" #include #include @@ -104,11 +104,10 @@ struct TensorFunction * the value stack during execution. * * @return instruction representing the operation of this node - * @param factory the value builder factory used during evaluation - * @param stash heterogeneous object store + * @param ctx context needed for compilation (stash and stuff) **/ - virtual InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const = 0; - + virtual InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const = 0; + // for debug dumping vespalib::string as_string() const; virtual void visit_self(vespalib::ObjectVisitor &visitor) const; @@ -188,7 +187,7 @@ public: ConstValue(const Value &value_in) : Super(value_in.type()), _value(value_in) {} const Value &value() const { return _value; } bool result_is_mutable() const override { return false; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -204,7 +203,7 @@ public: : Super(result_type_in), _param_idx(param_idx_in) {} size_t param_idx() const { return _param_idx; } bool result_is_mutable() const override { return false; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -225,7 +224,7 @@ public: Aggr aggr() const { return _aggr; } const std::vector &dimensions() const { return _dimensions; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -243,7 +242,7 @@ public: : Super(result_type_in, child_in), _function(function_in) {} map_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -266,7 +265,7 @@ public: const Function &lambda() const { return *_lambda; } const NodeTypes &types() const { return _lambda_types; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -285,7 +284,7 @@ public: : Super(result_type_in, lhs_in, rhs_in), _function(function_in) {} join_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -304,7 +303,7 @@ public: : Super(result_type_in, lhs_in, rhs_in), _function(function_in) {} join_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -323,7 +322,7 @@ public: : Super(result_type_in, lhs_in, rhs_in), _dimension(dimension_in) {} const vespalib::string &dimension() const { return _dimension; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -339,7 +338,7 @@ public: : Super(result_type_in, child_in), _cell_type(cell_type) {} CellType cell_type() const { return _cell_type; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -363,7 +362,7 @@ public: using Spec = std::map; Spec make_spec() const; bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void push_children(std::vector &children) const final override; void visit_children(vespalib::ObjectVisitor &visitor) const final override; }; @@ -388,7 +387,7 @@ public: return create_spec_impl(result_type(), params, _bindings, fun); } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -428,7 +427,7 @@ public: const TensorFunction ¶m() const { return _param.get(); } const ValueType ¶m_type() const { return _param.get().result_type(); } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void push_children(std::vector &children) const final override; void visit_children(vespalib::ObjectVisitor &visitor) const final override; }; @@ -450,7 +449,7 @@ public: const std::vector &from() const { return _from; } const std::vector &to() const { return _to; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -476,7 +475,7 @@ public: return (true_child().result_is_mutable() && false_child().result_is_mutable()); } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const final override; void visit_children(vespalib::ObjectVisitor &visitor) const final override; }; diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 562e92a4767..601ac2f3cdc 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -77,6 +77,16 @@ const TensorFunction &maybe_patch(bool allow_mutable, const TensorFunction &plai return apply_tensor_function_optimizer(plain_fun, optimizer, stash); } +auto make_optimize_fun(std::set mutable_set, bool allow_mutable, bool optimized, const TensorFunction **root_out) { + return [=](const ValueBuilderFactory &factory, const TensorFunction &plain_function, Stash &stash)->const TensorFunction &{ + const auto &patched_function = maybe_patch(allow_mutable, plain_function, mutable_set, stash); + const auto &optimized_root = optimized ? optimize_tensor_function(factory, patched_function, stash) : patched_function; + // will be kept alive by the static stash of the interpreted function + *root_out = &optimized_root; + return optimized_root; + }; +} + std::vector make_params(const ValueBuilderFactory &factory, const Function &function, const ParamRepo ¶m_repo) { @@ -97,7 +107,7 @@ std::vector get_refs(const std::vector &values) { return result; } -} // namespace vespalib::eval::test +} // namespace vespalib::eval::test:: ParamRepo & EvalFixture::ParamRepo::add(const vespalib::string &name, TensorSpec value) @@ -173,14 +183,11 @@ EvalFixture::EvalFixture(const ValueBuilderFactory &factory, bool optimized, bool allow_mutable) : _factory(factory), - _stash(), _function(verify_function(Function::parse(expr))), _node_types(get_types(*_function, param_repo)), - _mutable_set(get_mutable(*_function, param_repo)), - _plain_tensor_function(make_tensor_function(_factory, _function->root(), _node_types, _stash)), - _patched_tensor_function(maybe_patch(allow_mutable, _plain_tensor_function, _mutable_set, _stash)), - _tensor_function(optimized ? optimize_tensor_function(_factory, _patched_tensor_function, _stash) : _patched_tensor_function), - _ifun(_factory, _tensor_function), + _optimized_root(nullptr), + _my_optimize(make_optimize_fun(get_mutable(*_function, param_repo), allow_mutable, optimized, &_optimized_root)), + _ifun(InterpretedFunction::opts(_factory).optimize(_my_optimize), _function->root(), _node_types), _ictx(_ifun), _param_values(make_params(_factory, *_function, param_repo)), _params(get_refs(_param_values)), diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 7e33b5417b6..ec3e05cc4dd 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -64,14 +64,13 @@ public: }; private: + using optimize_fun_t = InterpretedFunction::Options::optimize_fun_t; const ValueBuilderFactory &_factory; - Stash _stash; std::shared_ptr _function; NodeTypes _node_types; std::set _mutable_set; - const TensorFunction &_plain_tensor_function; - const TensorFunction &_patched_tensor_function; - const TensorFunction &_tensor_function; + const TensorFunction *_optimized_root; + optimize_fun_t _my_optimize; InterpretedFunction _ifun; InterpretedFunction::Context _ictx; std::vector _param_values; @@ -115,7 +114,8 @@ public: template std::vector find_all() const { std::vector list; - find_all(_tensor_function, list); + REQUIRE(_optimized_root != nullptr); + find_all(*_optimized_root, list); return list; } const Value &result_value() const { return _result_value; } diff --git a/eval/src/vespa/eval/instruction/best_similarity_function.cpp b/eval/src/vespa/eval/instruction/best_similarity_function.cpp index b3a229c946c..3bdb7e8e907 100644 --- a/eval/src/vespa/eval/instruction/best_similarity_function.cpp +++ b/eval/src/vespa/eval/instruction/best_similarity_function.cpp @@ -189,9 +189,9 @@ BestSimilarityFunction::BestSimilarityFunction(const ValueType &res_type_in, } InterpretedFunction::Instruction -BestSimilarityFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +BestSimilarityFunction::compile_self(const CTFContext &ctx) const { - return InterpretedFunction::Instruction(_my_fun, make_param(stash)); + return InterpretedFunction::Instruction(_my_fun, make_param(ctx.stash)); } const TensorFunction & diff --git a/eval/src/vespa/eval/instruction/best_similarity_function.h b/eval/src/vespa/eval/instruction/best_similarity_function.h index 008feca62ae..f0b753a52a9 100644 --- a/eval/src/vespa/eval/instruction/best_similarity_function.h +++ b/eval/src/vespa/eval/instruction/best_similarity_function.h @@ -30,7 +30,7 @@ public: const TensorFunction &sec, InterpretedFunction::op_function my_fun, size_t inner_size); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_cell_range_function.cpp b/eval/src/vespa/eval/instruction/dense_cell_range_function.cpp index d60504dddce..c3a7c4a244a 100644 --- a/eval/src/vespa/eval/instruction/dense_cell_range_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_cell_range_function.cpp @@ -37,7 +37,7 @@ DenseCellRangeFunction::DenseCellRangeFunction(const ValueType &result_type, DenseCellRangeFunction::~DenseCellRangeFunction() = default; InterpretedFunction::Instruction -DenseCellRangeFunction::compile_self(const ValueBuilderFactory &, Stash &) const +DenseCellRangeFunction::compile_self(const CTFContext &) const { assert(result_type().cell_type() == child().result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/dense_cell_range_function.h b/eval/src/vespa/eval/instruction/dense_cell_range_function.h index a37e60d2e57..f132a6f7fb7 100644 --- a/eval/src/vespa/eval/instruction/dense_cell_range_function.h +++ b/eval/src/vespa/eval/instruction/dense_cell_range_function.h @@ -24,7 +24,7 @@ public: ~DenseCellRangeFunction() override; size_t offset() const { return _offset; } size_t length() const { return _length; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return child().result_is_mutable(); } }; diff --git a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp index 8395ef66117..bee7f42bd48 100644 --- a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp @@ -33,7 +33,7 @@ DenseDotProductFunction::DenseDotProductFunction(const TensorFunction &lhs_in, } InterpretedFunction::Instruction -DenseDotProductFunction::compile_self(const ValueBuilderFactory &, Stash &) const +DenseDotProductFunction::compile_self(const CTFContext &) const { auto op = typify_invoke<2,TypifyCellType,MyDotProductOp>(lhs().result_type().cell_type(), rhs().result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/dense_dot_product_function.h b/eval/src/vespa/eval/instruction/dense_dot_product_function.h index 51fdbc3f685..db09db85147 100644 --- a/eval/src/vespa/eval/instruction/dense_dot_product_function.h +++ b/eval/src/vespa/eval/instruction/dense_dot_product_function.h @@ -14,7 +14,7 @@ class DenseDotProductFunction : public tensor_function::Op2 public: DenseDotProductFunction(const TensorFunction &lhs_in, const TensorFunction &rhs_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp index 81f25241d3d..af3551466de 100644 --- a/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp +++ b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp @@ -41,7 +41,7 @@ DenseHammingDistance::DenseHammingDistance(const TensorFunction &lhs_child, } InterpretedFunction::Instruction -DenseHammingDistance::compile_self(const ValueBuilderFactory &, Stash &) const +DenseHammingDistance::compile_self(const CTFContext &) const { auto op = int8_hamming_to_double_op; const auto &lhs_type = lhs().result_type(); diff --git a/eval/src/vespa/eval/instruction/dense_hamming_distance.h b/eval/src/vespa/eval/instruction/dense_hamming_distance.h index 437c50bc52b..7dac0e0352e 100644 --- a/eval/src/vespa/eval/instruction/dense_hamming_distance.h +++ b/eval/src/vespa/eval/instruction/dense_hamming_distance.h @@ -14,7 +14,7 @@ class DenseHammingDistance : public tensor_function::Op2 public: DenseHammingDistance(const TensorFunction &lhs_child, const TensorFunction &rhs_child); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp index b0e0a9af648..50b95ded7ce 100644 --- a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp @@ -52,9 +52,9 @@ DenseLambdaPeekFunction::DenseLambdaPeekFunction(const ValueType &result_type, DenseLambdaPeekFunction::~DenseLambdaPeekFunction() = default; InterpretedFunction::Instruction -DenseLambdaPeekFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +DenseLambdaPeekFunction::compile_self(const CTFContext &ctx) const { - const Self &self = stash.create(result_type(), *_idx_fun); + const Self &self = ctx.stash.create(result_type(), *_idx_fun); using MyTypify = TypifyCellType; auto op = typify_invoke<2,MyTypify,MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); assert(child().result_type().is_dense()); diff --git a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.h b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.h index 16e3298feff..7c1ca08a661 100644 --- a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.h +++ b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.h @@ -23,7 +23,7 @@ public: const TensorFunction &child, std::shared_ptr idx_fun); ~DenseLambdaPeekFunction() override; - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; vespalib::string idx_fun_dump() const; bool result_is_mutable() const override { return true; } }; diff --git a/eval/src/vespa/eval/instruction/dense_matmul_function.cpp b/eval/src/vespa/eval/instruction/dense_matmul_function.cpp index e18ffabec69..9691d3a11a9 100644 --- a/eval/src/vespa/eval/instruction/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_matmul_function.cpp @@ -163,10 +163,10 @@ DenseMatMulFunction::DenseMatMulFunction(const ValueType &result_type, DenseMatMulFunction::~DenseMatMulFunction() = default; InterpretedFunction::Instruction -DenseMatMulFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +DenseMatMulFunction::compile_self(const CTFContext &ctx) const { using MyTypify = TypifyValue; - Self &self = stash.create(result_type(), _lhs_size, _common_size, _rhs_size); + Self &self = ctx.stash.create(result_type(), _lhs_size, _common_size, _rhs_size); auto op = typify_invoke<4,MyTypify,SelectDenseMatmul>( lhs().result_type().cell_meta().not_scalar(), rhs().result_type().cell_meta().not_scalar(), diff --git a/eval/src/vespa/eval/instruction/dense_matmul_function.h b/eval/src/vespa/eval/instruction/dense_matmul_function.h index 8e4ff5f14a9..34bf385bf67 100644 --- a/eval/src/vespa/eval/instruction/dense_matmul_function.h +++ b/eval/src/vespa/eval/instruction/dense_matmul_function.h @@ -49,7 +49,7 @@ public: bool lhs_common_inner() const { return _lhs_common_inner; } bool rhs_common_inner() const { return _rhs_common_inner; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp index d9205488dff..90f91e109cc 100644 --- a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp @@ -176,7 +176,7 @@ DenseMultiMatMulFunction::DenseMultiMatMulFunction(const ValueType &result_type, DenseMultiMatMulFunction::~DenseMultiMatMulFunction() = default; InterpretedFunction::Instruction -DenseMultiMatMulFunction::compile_self(const ValueBuilderFactory &, Stash &) const +DenseMultiMatMulFunction::compile_self(const CTFContext &) const { auto op = my_select(lhs().result_type().cell_type()); return InterpretedFunction::Instruction(op, wrap_param(*this)); diff --git a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.h b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.h index a6a1da67bb2..9aeb458270e 100644 --- a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.h +++ b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.h @@ -43,7 +43,7 @@ public: bool lhs_common_inner() const { return _lhs_common_inner; } bool rhs_common_inner() const { return _rhs_common_inner; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_simple_expand_function.cpp b/eval/src/vespa/eval/instruction/dense_simple_expand_function.cpp index e4207740fc7..64b8b8fa3e0 100644 --- a/eval/src/vespa/eval/instruction/dense_simple_expand_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_simple_expand_function.cpp @@ -98,10 +98,10 @@ DenseSimpleExpandFunction::DenseSimpleExpandFunction(const ValueType &result_typ DenseSimpleExpandFunction::~DenseSimpleExpandFunction() = default; Instruction -DenseSimpleExpandFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +DenseSimpleExpandFunction::compile_self(const CTFContext &ctx) const { size_t result_size = result_type().dense_subspace_size(); - const ExpandParams ¶ms = stash.create(result_type(), result_size, function()); + const ExpandParams ¶ms = ctx.stash.create(result_type(), result_size, function()); auto op = typify_invoke<4,MyTypify,SelectDenseSimpleExpand>(lhs().result_type().cell_meta().not_scalar(), rhs().result_type().cell_meta().not_scalar(), function(), (_inner == Inner::RHS)); diff --git a/eval/src/vespa/eval/instruction/dense_simple_expand_function.h b/eval/src/vespa/eval/instruction/dense_simple_expand_function.h index 489859bcd88..a9d98b410c0 100644 --- a/eval/src/vespa/eval/instruction/dense_simple_expand_function.h +++ b/eval/src/vespa/eval/instruction/dense_simple_expand_function.h @@ -32,7 +32,7 @@ public: Inner inner_in); ~DenseSimpleExpandFunction() override; Inner inner() const { return _inner; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp b/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp index 1f849b03737..d6e5b92da74 100644 --- a/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_single_reduce_function.cpp @@ -229,12 +229,12 @@ DenseSingleReduceFunction::DenseSingleReduceFunction(const DenseSingleReduceSpec DenseSingleReduceFunction::~DenseSingleReduceFunction() = default; InterpretedFunction::Instruction -DenseSingleReduceFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +DenseSingleReduceFunction::compile_self(const CTFContext &ctx) const { auto op = typify_invoke<4,MyTypify,MyGetFun>(child().result_type().cell_meta().not_scalar(), _aggr, (_reduce_size >= 8), (_inner_size == 1)); - auto ¶ms = stash.create(result_type(), _outer_size, _reduce_size, _inner_size); + auto ¶ms = ctx.stash.create(result_type(), _outer_size, _reduce_size, _inner_size); return InterpretedFunction::Instruction(op, wrap_param(params)); } diff --git a/eval/src/vespa/eval/instruction/dense_single_reduce_function.h b/eval/src/vespa/eval/instruction/dense_single_reduce_function.h index 465cb35edb3..3c02f1c5552 100644 --- a/eval/src/vespa/eval/instruction/dense_single_reduce_function.h +++ b/eval/src/vespa/eval/instruction/dense_single_reduce_function.h @@ -50,7 +50,7 @@ public: size_t inner_size() const { return _inner_size; } Aggr aggr() const { return _aggr; } bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_tensor_create_function.cpp b/eval/src/vespa/eval/instruction/dense_tensor_create_function.cpp index ad1e8c8b67c..bc0ce336e1f 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_tensor_create_function.cpp @@ -60,7 +60,7 @@ DenseTensorCreateFunction::push_children(std::vector &target) const } InterpretedFunction::Instruction -DenseTensorCreateFunction::compile_self(const ValueBuilderFactory &, Stash &) const +DenseTensorCreateFunction::compile_self(const CTFContext &) const { using MyTypify = TypifyCellType; auto op = typify_invoke<1,MyTypify,MyTensorCreateOp>(result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/dense_tensor_create_function.h b/eval/src/vespa/eval/instruction/dense_tensor_create_function.h index c5788767e55..115c979cfc5 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_create_function.h +++ b/eval/src/vespa/eval/instruction/dense_tensor_create_function.h @@ -26,7 +26,7 @@ public: ~DenseTensorCreateFunction(); const ValueType &result_type() const override { return _self.result_type; } void push_children(std::vector &children) const override; - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp index 68dcecb481c..80763eb6d08 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp @@ -60,7 +60,7 @@ DenseTensorPeekFunction::push_children(std::vector &target) const } InterpretedFunction::Instruction -DenseTensorPeekFunction::compile_self(const ValueBuilderFactory &, Stash &) const +DenseTensorPeekFunction::compile_self(const CTFContext &) const { using MyTypify = TypifyCellType; auto op = typify_invoke<1,MyTypify,MyTensorPeekOp>(_children[0].get().result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h index 56dc72c93f7..39bfed4f11a 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h +++ b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h @@ -27,7 +27,7 @@ public: ~DenseTensorPeekFunction() override; const ValueType &result_type() const override { return DoubleValue::shared_type(); } void push_children(std::vector &children) const override; - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp b/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp index e9b9cb12f40..fe1bdf80a2e 100644 --- a/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp @@ -141,9 +141,9 @@ DenseXWProductFunction::DenseXWProductFunction(const ValueType &result_type, } InterpretedFunction::Instruction -DenseXWProductFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +DenseXWProductFunction::compile_self(const CTFContext &ctx) const { - Self &self = stash.create(result_type(), _vector_size, _result_size); + Self &self = ctx.stash.create(result_type(), _vector_size, _result_size); assert(self.result_type.cell_meta().is_scalar == false); using MyTypify = TypifyValue; auto op = typify_invoke<3,MyTypify,MyXWProductOp>(lhs().result_type().cell_meta().not_scalar(), diff --git a/eval/src/vespa/eval/instruction/dense_xw_product_function.h b/eval/src/vespa/eval/instruction/dense_xw_product_function.h index aace80ff833..ecbe485fe54 100644 --- a/eval/src/vespa/eval/instruction/dense_xw_product_function.h +++ b/eval/src/vespa/eval/instruction/dense_xw_product_function.h @@ -43,7 +43,7 @@ public: size_t result_size() const { return _result_size; } bool common_inner() const { return _common_inner; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/generic_lambda.cpp b/eval/src/vespa/eval/instruction/generic_lambda.cpp index 77d067c2b79..637982fffc4 100644 --- a/eval/src/vespa/eval/instruction/generic_lambda.cpp +++ b/eval/src/vespa/eval/instruction/generic_lambda.cpp @@ -87,11 +87,11 @@ struct InterpretedParams { const std::vector &bindings; size_t num_cells; InterpretedFunction fun; - InterpretedParams(const Lambda &lambda, const ValueBuilderFactory &factory) + InterpretedParams(const Lambda &lambda, const ValueBuilderFactory &factory, CTFMetaData *meta) : result_type(lambda.result_type()), bindings(lambda.bindings()), num_cells(result_type.dense_subspace_size()), - fun(factory, lambda.lambda().root(), lambda.types()) + fun(InterpretedFunction::opts(factory).meta(meta), lambda.lambda().root(), lambda.types()) { assert(lambda.lambda().num_params() == (result_type.dimensions().size() + bindings.size())); } @@ -122,7 +122,8 @@ struct MyInterpretedLambdaOp { Instruction GenericLambda::make_instruction(const tensor_function::Lambda &lambda_in, - const ValueBuilderFactory &factory, Stash &stash) + const ValueBuilderFactory &factory, Stash &stash, + CTFMetaData *meta) { const ValueType & result_type = lambda_in.result_type(); assert(result_type.count_mapped_dimensions() == 0); @@ -134,7 +135,7 @@ GenericLambda::make_instruction(const tensor_function::Lambda &lambda_in, auto op = typify_invoke<1,TypifyCellType,MyCompiledLambdaOp>(result_type.cell_type()); return Instruction(op, wrap_param(params)); } else { - InterpretedParams ¶ms = stash.create(lambda_in, factory); + InterpretedParams ¶ms = stash.create(lambda_in, factory, meta); auto op = typify_invoke<1,TypifyCellType,MyInterpretedLambdaOp>(result_type.cell_type()); return Instruction(op, wrap_param(params)); } diff --git a/eval/src/vespa/eval/instruction/generic_lambda.h b/eval/src/vespa/eval/instruction/generic_lambda.h index e12151771ba..8ed879cd8d2 100644 --- a/eval/src/vespa/eval/instruction/generic_lambda.h +++ b/eval/src/vespa/eval/instruction/generic_lambda.h @@ -11,7 +11,8 @@ namespace vespalib::eval::instruction { struct GenericLambda { static InterpretedFunction::Instruction make_instruction(const tensor_function::Lambda &lambda_in, - const ValueBuilderFactory &factory, Stash &stash); + const ValueBuilderFactory &factory, Stash &stash, + CTFMetaData *meta); }; } // namespace diff --git a/eval/src/vespa/eval/instruction/generic_map_subspaces.cpp b/eval/src/vespa/eval/instruction/generic_map_subspaces.cpp index 1238d4f4e57..76a8b758ce4 100644 --- a/eval/src/vespa/eval/instruction/generic_map_subspaces.cpp +++ b/eval/src/vespa/eval/instruction/generic_map_subspaces.cpp @@ -21,10 +21,10 @@ struct InterpretedParams { size_t out_size; bool direct_in; bool direct_out; - InterpretedParams(const MapSubspaces &map_subspaces, const ValueBuilderFactory &factory) + InterpretedParams(const MapSubspaces &map_subspaces, const ValueBuilderFactory &factory, CTFMetaData *meta) : result_type(map_subspaces.result_type()), inner_type(map_subspaces.inner_type()), - fun(factory, map_subspaces.lambda().root(), map_subspaces.types()), + fun(InterpretedFunction::opts(factory).meta(meta), map_subspaces.lambda().root(), map_subspaces.types()), in_size(inner_type.dense_subspace_size()), out_size(result_type.dense_subspace_size()), direct_in(map_subspaces.child().result_type().cell_type() == inner_type.cell_type()), @@ -107,9 +107,10 @@ struct SelectGenericMapSubspacesOp { Instruction GenericMapSubspaces::make_instruction(const tensor_function::MapSubspaces &map_subspaces_in, - const ValueBuilderFactory &factory, Stash &stash) + const ValueBuilderFactory &factory, Stash &stash, + CTFMetaData *meta) { - InterpretedParams ¶ms = stash.create(map_subspaces_in, factory); + InterpretedParams ¶ms = stash.create(map_subspaces_in, factory, meta); auto op = typify_invoke<2,TypifyCellType,SelectGenericMapSubspacesOp>(map_subspaces_in.child().result_type().cell_type(), params.result_type.cell_type()); return Instruction(op, wrap_param(params)); diff --git a/eval/src/vespa/eval/instruction/generic_map_subspaces.h b/eval/src/vespa/eval/instruction/generic_map_subspaces.h index f95ded60a1b..2d0c3b101b0 100644 --- a/eval/src/vespa/eval/instruction/generic_map_subspaces.h +++ b/eval/src/vespa/eval/instruction/generic_map_subspaces.h @@ -11,7 +11,8 @@ namespace vespalib::eval::instruction { struct GenericMapSubspaces { static InterpretedFunction::Instruction make_instruction(const tensor_function::MapSubspaces &map_subspaces_in, - const ValueBuilderFactory &factory, Stash &stash); + const ValueBuilderFactory &factory, Stash &stash, + CTFMetaData *meta); }; } // namespace diff --git a/eval/src/vespa/eval/instruction/inplace_map_function.cpp b/eval/src/vespa/eval/instruction/inplace_map_function.cpp index ee013f3b753..78726098ea2 100644 --- a/eval/src/vespa/eval/instruction/inplace_map_function.cpp +++ b/eval/src/vespa/eval/instruction/inplace_map_function.cpp @@ -52,7 +52,7 @@ InplaceMapFunction::InplaceMapFunction(const ValueType &result_type, InplaceMapFunction::~InplaceMapFunction() = default; Instruction -InplaceMapFunction::compile_self(const ValueBuilderFactory &, Stash &) const +InplaceMapFunction::compile_self(const CTFContext &) const { auto input_cell_meta = child().result_type().cell_meta().limit().not_scalar(); auto op = typify_invoke<2,MyTypify,MyGetFun>(input_cell_meta, function()); diff --git a/eval/src/vespa/eval/instruction/inplace_map_function.h b/eval/src/vespa/eval/instruction/inplace_map_function.h index 172693ab2bf..74fb559248d 100644 --- a/eval/src/vespa/eval/instruction/inplace_map_function.h +++ b/eval/src/vespa/eval/instruction/inplace_map_function.h @@ -18,7 +18,7 @@ public: map_fun_t function_in); ~InplaceMapFunction() override; bool inplace() const { return child().result_is_mutable(); } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/join_with_number_function.cpp b/eval/src/vespa/eval/instruction/join_with_number_function.cpp index d24a69e33a7..c41867242f0 100644 --- a/eval/src/vespa/eval/instruction/join_with_number_function.cpp +++ b/eval/src/vespa/eval/instruction/join_with_number_function.cpp @@ -89,9 +89,9 @@ JoinWithNumberFunction::primary_is_mutable() const { using MyTypify = TypifyValue; InterpretedFunction::Instruction -JoinWithNumberFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +JoinWithNumberFunction::compile_self(const CTFContext &ctx) const { - const auto ¶m = stash.create(result_type(), _function); + const auto ¶m = ctx.stash.create(result_type(), _function); auto input_type = (_primary == Primary::LHS) ? lhs().result_type() : rhs().result_type(); assert(result_type() == ValueType::join(input_type, ValueType::double_type())); auto op = typify_invoke<4,MyTypify,SelectJoinWithNumberOp>(input_type.cell_meta(), diff --git a/eval/src/vespa/eval/instruction/join_with_number_function.h b/eval/src/vespa/eval/instruction/join_with_number_function.h index b26e626818a..0e33958879f 100644 --- a/eval/src/vespa/eval/instruction/join_with_number_function.h +++ b/eval/src/vespa/eval/instruction/join_with_number_function.h @@ -26,7 +26,7 @@ public: bool primary_is_mutable() const; bool result_is_mutable() const override { return true; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/l2_distance.cpp b/eval/src/vespa/eval/instruction/l2_distance.cpp index 9490044a39b..5298468ab02 100644 --- a/eval/src/vespa/eval/instruction/l2_distance.cpp +++ b/eval/src/vespa/eval/instruction/l2_distance.cpp @@ -65,7 +65,7 @@ L2Distance::L2Distance(const TensorFunction &lhs_in, const TensorFunction &rhs_i } InterpretedFunction::Instruction -L2Distance::compile_self(const ValueBuilderFactory &, Stash &) const +L2Distance::compile_self(const CTFContext &) const { auto lhs_t = lhs().result_type(); auto rhs_t = rhs().result_type(); diff --git a/eval/src/vespa/eval/instruction/l2_distance.h b/eval/src/vespa/eval/instruction/l2_distance.h index 5d50b4ee283..a9138a169ce 100644 --- a/eval/src/vespa/eval/instruction/l2_distance.h +++ b/eval/src/vespa/eval/instruction/l2_distance.h @@ -13,7 +13,7 @@ class L2Distance : public tensor_function::Op2 { public: L2Distance(const TensorFunction &lhs_in, const TensorFunction &rhs_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/mapped_lookup.cpp b/eval/src/vespa/eval/instruction/mapped_lookup.cpp index 49048689ef8..22d99211a3b 100644 --- a/eval/src/vespa/eval/instruction/mapped_lookup.cpp +++ b/eval/src/vespa/eval/instruction/mapped_lookup.cpp @@ -117,7 +117,7 @@ MappedLookup::MappedLookup(const ValueType &res_type, } InterpretedFunction::Instruction -MappedLookup::compile_self(const ValueBuilderFactory &, Stash &) const +MappedLookup::compile_self(const CTFContext &) const { uint64_t param = wrap_param(result_type()); if (result_type().cell_type() == CellType::FLOAT) { diff --git a/eval/src/vespa/eval/instruction/mapped_lookup.h b/eval/src/vespa/eval/instruction/mapped_lookup.h index 7fc55326e1b..ea1a0648b41 100644 --- a/eval/src/vespa/eval/instruction/mapped_lookup.h +++ b/eval/src/vespa/eval/instruction/mapped_lookup.h @@ -41,7 +41,7 @@ public: MappedLookup(const ValueType &res_type, const TensorFunction &key_in, const TensorFunction &map_in); const TensorFunction &key() const { return lhs(); } const TensorFunction &map() const { return rhs(); } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return map().result_is_mutable(); } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp b/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp index 992636b15d5..5678adda3b3 100644 --- a/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp +++ b/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp @@ -208,7 +208,7 @@ Mixed112DotProduct::Mixed112DotProduct(const TensorFunction &a_in, } InterpretedFunction::Instruction -Mixed112DotProduct::compile_self(const ValueBuilderFactory &, Stash &) const +Mixed112DotProduct::compile_self(const CTFContext &) const { REQUIRE_EQ(_a.get().result_type().cell_type(), _b.get().result_type().cell_type()); REQUIRE_EQ(_a.get().result_type().cell_type(), _c.get().result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/mixed_112_dot_product.h b/eval/src/vespa/eval/instruction/mixed_112_dot_product.h index d0c3c18b996..c43a4074a16 100644 --- a/eval/src/vespa/eval/instruction/mixed_112_dot_product.h +++ b/eval/src/vespa/eval/instruction/mixed_112_dot_product.h @@ -21,7 +21,7 @@ public: Mixed112DotProduct(const TensorFunction &a_in, const TensorFunction &b_in, const TensorFunction &c_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } void push_children(std::vector &children) const final override; void visit_children(vespalib::ObjectVisitor &visitor) const final override; diff --git a/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp b/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp index 5c90dcebfd9..efd3398ef0c 100644 --- a/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp +++ b/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp @@ -65,11 +65,11 @@ MixedInnerProductFunction::MixedInnerProductFunction(const ValueType &res_type_i } InterpretedFunction::Instruction -MixedInnerProductFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +MixedInnerProductFunction::compile_self(const CTFContext &ctx) const { const auto &mix_type = lhs().result_type(); const auto &vec_type = rhs().result_type(); - auto ¶m = stash.create(result_type(), mix_type, vec_type); + auto ¶m = ctx.stash.create(result_type(), mix_type, vec_type); using MyTypify = TypifyValue; auto op = typify_invoke<3,MyTypify,SelectMixedInnerProduct>(mix_type.cell_type(), vec_type.cell_type(), diff --git a/eval/src/vespa/eval/instruction/mixed_inner_product_function.h b/eval/src/vespa/eval/instruction/mixed_inner_product_function.h index 6eee998d4bd..00fa038ee48 100644 --- a/eval/src/vespa/eval/instruction/mixed_inner_product_function.h +++ b/eval/src/vespa/eval/instruction/mixed_inner_product_function.h @@ -37,7 +37,7 @@ public: MixedInnerProductFunction(const ValueType &res_type_in, const TensorFunction &mixed_child, const TensorFunction &vector_child); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &mixed, const ValueType &dense); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp b/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp index 1a5293789c5..1ac741d65db 100644 --- a/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp +++ b/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp @@ -90,13 +90,13 @@ MixedL2Distance::MixedL2Distance(const ValueType &result_type, const TensorFunct } InterpretedFunction::Instruction -MixedL2Distance::compile_self(const ValueBuilderFactory &, Stash &stash) const +MixedL2Distance::compile_self(const CTFContext &ctx) const { auto mix_t = lhs().result_type(); auto vec_t = rhs().result_type(); REQUIRE_EQ(mix_t.cell_type(), vec_t.cell_type()); REQUIRE_EQ(mix_t.dense_subspace_size(), vec_t.dense_subspace_size()); - const auto ¶m = stash.create(result_type(), mix_t.dense_subspace_size()); + const auto ¶m = ctx.stash.create(result_type(), mix_t.dense_subspace_size()); auto mix_cm = mix_t.cell_meta().not_scalar(); auto res_cm = mix_t.cell_meta().decay(); REQUIRE_EQ(res_cm.cell_type, result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/mixed_l2_distance.h b/eval/src/vespa/eval/instruction/mixed_l2_distance.h index a5f2e2f6273..c105a731456 100644 --- a/eval/src/vespa/eval/instruction/mixed_l2_distance.h +++ b/eval/src/vespa/eval/instruction/mixed_l2_distance.h @@ -13,7 +13,7 @@ class MixedL2Distance : public tensor_function::Op2 { public: MixedL2Distance(const ValueType &result_type, const TensorFunction &mix_in, const TensorFunction &vec_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/mixed_simple_join_function.cpp b/eval/src/vespa/eval/instruction/mixed_simple_join_function.cpp index 845c6d8472c..e1d04e580a7 100644 --- a/eval/src/vespa/eval/instruction/mixed_simple_join_function.cpp +++ b/eval/src/vespa/eval/instruction/mixed_simple_join_function.cpp @@ -208,9 +208,9 @@ MixedSimpleJoinFunction::factor() const } Instruction -MixedSimpleJoinFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +MixedSimpleJoinFunction::compile_self(const CTFContext &ctx) const { - const JoinParams ¶ms = stash.create(result_type(), factor(), function()); + const JoinParams ¶ms = ctx.stash.create(result_type(), factor(), function()); auto op = typify_invoke<6,MyTypify,SelectMixedSimpleJoin>(lhs().result_type().cell_meta().not_scalar(), rhs().result_type().cell_meta().not_scalar(), function(), (_primary == Primary::RHS), diff --git a/eval/src/vespa/eval/instruction/mixed_simple_join_function.h b/eval/src/vespa/eval/instruction/mixed_simple_join_function.h index 873ed55dbe6..19427250e6b 100644 --- a/eval/src/vespa/eval/instruction/mixed_simple_join_function.h +++ b/eval/src/vespa/eval/instruction/mixed_simple_join_function.h @@ -42,7 +42,7 @@ public: bool primary_is_mutable() const; bool inplace() const; size_t factor() const; - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/replace_type_function.cpp b/eval/src/vespa/eval/instruction/replace_type_function.cpp index 01fe43cb16d..2d66d055834 100644 --- a/eval/src/vespa/eval/instruction/replace_type_function.cpp +++ b/eval/src/vespa/eval/instruction/replace_type_function.cpp @@ -29,7 +29,7 @@ ReplaceTypeFunction::~ReplaceTypeFunction() } InterpretedFunction::Instruction -ReplaceTypeFunction::compile_self(const ValueBuilderFactory &, Stash &) const +ReplaceTypeFunction::compile_self(const CTFContext &) const { return InterpretedFunction::Instruction(my_replace_type_op, wrap_param(result_type())); } diff --git a/eval/src/vespa/eval/instruction/replace_type_function.h b/eval/src/vespa/eval/instruction/replace_type_function.h index 506c7d4e124..658494dd753 100644 --- a/eval/src/vespa/eval/instruction/replace_type_function.h +++ b/eval/src/vespa/eval/instruction/replace_type_function.h @@ -15,7 +15,7 @@ public: ReplaceTypeFunction(const ValueType &result_type, const TensorFunction &child); ~ReplaceTypeFunction(); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return child().result_is_mutable(); } static const ReplaceTypeFunction &create_compact(const ValueType &result_type, const TensorFunction &child, diff --git a/eval/src/vespa/eval/instruction/simple_join_count.cpp b/eval/src/vespa/eval/instruction/simple_join_count.cpp index 6d6e4872d28..23b816374a9 100644 --- a/eval/src/vespa/eval/instruction/simple_join_count.cpp +++ b/eval/src/vespa/eval/instruction/simple_join_count.cpp @@ -68,7 +68,7 @@ SimpleJoinCount::SimpleJoinCount(const TensorFunction &lhs_in, } InterpretedFunction::Instruction -SimpleJoinCount::compile_self(const ValueBuilderFactory &, Stash &) const +SimpleJoinCount::compile_self(const CTFContext &) const { return InterpretedFunction::Instruction(my_simple_join_count_op, _dense_factor); } diff --git a/eval/src/vespa/eval/instruction/simple_join_count.h b/eval/src/vespa/eval/instruction/simple_join_count.h index 02c81dbebe1..75f4b5cfde7 100644 --- a/eval/src/vespa/eval/instruction/simple_join_count.h +++ b/eval/src/vespa/eval/instruction/simple_join_count.h @@ -17,7 +17,7 @@ private: uint64_t _dense_factor; public: SimpleJoinCount(const TensorFunction &lhs_in, const TensorFunction &rhs_in, uint64_t dense_factor_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } uint64_t dense_factor() const { return _dense_factor; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/sparse_112_dot_product.cpp b/eval/src/vespa/eval/instruction/sparse_112_dot_product.cpp index 402d55bd447..32fd10df862 100644 --- a/eval/src/vespa/eval/instruction/sparse_112_dot_product.cpp +++ b/eval/src/vespa/eval/instruction/sparse_112_dot_product.cpp @@ -194,7 +194,7 @@ Sparse112DotProduct::Sparse112DotProduct(const TensorFunction &a_in, } InterpretedFunction::Instruction -Sparse112DotProduct::compile_self(const ValueBuilderFactory &, Stash &) const +Sparse112DotProduct::compile_self(const CTFContext &) const { REQUIRE_EQ(_a.get().result_type().cell_type(), _b.get().result_type().cell_type()); REQUIRE_EQ(_a.get().result_type().cell_type(), _c.get().result_type().cell_type()); diff --git a/eval/src/vespa/eval/instruction/sparse_112_dot_product.h b/eval/src/vespa/eval/instruction/sparse_112_dot_product.h index 16d58327e40..0bfe3cac993 100644 --- a/eval/src/vespa/eval/instruction/sparse_112_dot_product.h +++ b/eval/src/vespa/eval/instruction/sparse_112_dot_product.h @@ -21,7 +21,7 @@ public: Sparse112DotProduct(const TensorFunction &a_in, const TensorFunction &b_in, const TensorFunction &c_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } void push_children(std::vector &children) const final override; void visit_children(vespalib::ObjectVisitor &visitor) const final override; diff --git a/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp b/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp index aa441047e74..b20b065f1b1 100644 --- a/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp +++ b/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp @@ -92,7 +92,7 @@ SparseDotProductFunction::SparseDotProductFunction(const TensorFunction &lhs_in, } InterpretedFunction::Instruction -SparseDotProductFunction::compile_self(const ValueBuilderFactory &, Stash &) const +SparseDotProductFunction::compile_self(const CTFContext &) const { size_t num_dims = lhs().result_type().count_mapped_dimensions(); auto op = typify_invoke<2,MyTypify,MyGetFun>(lhs().result_type().cell_type(), diff --git a/eval/src/vespa/eval/instruction/sparse_dot_product_function.h b/eval/src/vespa/eval/instruction/sparse_dot_product_function.h index 093bbdafe47..f2fc439bb9c 100644 --- a/eval/src/vespa/eval/instruction/sparse_dot_product_function.h +++ b/eval/src/vespa/eval/instruction/sparse_dot_product_function.h @@ -14,7 +14,7 @@ class SparseDotProductFunction : public tensor_function::Op2 public: SparseDotProductFunction(const TensorFunction &lhs_in, const TensorFunction &rhs_in); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.cpp b/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.cpp index 2c31003e30d..a8a48419bf2 100644 --- a/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.cpp +++ b/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.cpp @@ -97,9 +97,9 @@ SparseFullOverlapJoinFunction::SparseFullOverlapJoinFunction(const tensor_functi } InterpretedFunction::Instruction -SparseFullOverlapJoinFunction::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +SparseFullOverlapJoinFunction::compile_self(const CTFContext &ctx) const { - const auto ¶m = stash.create(result_type(), lhs().result_type(), rhs().result_type(), function(), factory); + const auto ¶m = ctx.stash.create(result_type(), lhs().result_type(), rhs().result_type(), function(), ctx.factory); assert(result_type() == ValueType::join(lhs().result_type(), rhs().result_type())); bool single_dim = (result_type().count_mapped_dimensions() == 1); auto op = typify_invoke<3,MyTypify,SelectSparseFullOverlapJoinOp>(result_type().cell_meta().limit(), function(), single_dim); diff --git a/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.h b/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.h index 91b6b88e0c7..38c94598f95 100644 --- a/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.h +++ b/eval/src/vespa/eval/instruction/sparse_full_overlap_join_function.h @@ -13,7 +13,7 @@ class SparseFullOverlapJoinFunction : public tensor_function::Join { public: SparseFullOverlapJoinFunction(const tensor_function::Join &original); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/sparse_merge_function.cpp b/eval/src/vespa/eval/instruction/sparse_merge_function.cpp index 6dff7440d01..57d0c536e8d 100644 --- a/eval/src/vespa/eval/instruction/sparse_merge_function.cpp +++ b/eval/src/vespa/eval/instruction/sparse_merge_function.cpp @@ -108,11 +108,11 @@ SparseMergeFunction::SparseMergeFunction(const tensor_function::Merge &original) } InterpretedFunction::Instruction -SparseMergeFunction::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +SparseMergeFunction::compile_self(const CTFContext &ctx) const { - const auto ¶m = stash.create(result_type(), - lhs().result_type(), rhs().result_type(), - function(), factory); + const auto ¶m = ctx.stash.create(result_type(), + lhs().result_type(), rhs().result_type(), + function(), ctx.factory); size_t num_dims = result_type().count_mapped_dimensions(); auto op = typify_invoke<3,MyTypify,SelectSparseMergeOp>(result_type().cell_meta().limit(), num_dims == 1, diff --git a/eval/src/vespa/eval/instruction/sparse_merge_function.h b/eval/src/vespa/eval/instruction/sparse_merge_function.h index 49fa0cb0996..3db36fb2f81 100644 --- a/eval/src/vespa/eval/instruction/sparse_merge_function.h +++ b/eval/src/vespa/eval/instruction/sparse_merge_function.h @@ -13,7 +13,7 @@ class SparseMergeFunction : public tensor_function::Merge { public: SparseMergeFunction(const tensor_function::Merge &original); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp index 9153d65309d..f4bb5ec4b73 100644 --- a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp +++ b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp @@ -103,11 +103,11 @@ SparseNoOverlapJoinFunction::SparseNoOverlapJoinFunction(const tensor_function:: } InterpretedFunction::Instruction -SparseNoOverlapJoinFunction::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +SparseNoOverlapJoinFunction::compile_self(const CTFContext &ctx) const { - const auto ¶m = stash.create(result_type(), - lhs().result_type(), rhs().result_type(), - function(), factory); + const auto ¶m = ctx.stash.create(result_type(), + lhs().result_type(), rhs().result_type(), + function(), ctx.factory); auto op = typify_invoke<2,MyTypify,SelectSparseNoOverlapJoinOp>(result_type().cell_meta().limit(), function()); return {op, wrap_param(param)}; } diff --git a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h index 9a8acc08408..29e74113829 100644 --- a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h +++ b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h @@ -14,7 +14,7 @@ class SparseNoOverlapJoinFunction : public tensor_function::Join { public: SparseNoOverlapJoinFunction(const tensor_function::Join &original); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp index d7701baf5ce..2c2060cf3d5 100644 --- a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp @@ -60,7 +60,7 @@ SparseSingledimLookup::SparseSingledimLookup(const TensorFunction &tensor, } InterpretedFunction::Instruction -SparseSingledimLookup::compile_self(const ValueBuilderFactory &, Stash &) const +SparseSingledimLookup::compile_self(const CTFContext &) const { auto op = typify_invoke<1,TypifyCellType,MyGetFun>(lhs().result_type().cell_type()); return InterpretedFunction::Instruction(op); diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h index c5d07f3381c..ed2e7e1ea3f 100644 --- a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h @@ -16,7 +16,7 @@ class SparseSingledimLookup : public tensor_function::Op2 { public: SparseSingledimLookup(const TensorFunction &tensor, const TensorFunction &expr); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/sum_max_dot_product_function.cpp b/eval/src/vespa/eval/instruction/sum_max_dot_product_function.cpp index 0ca1ed9705b..74281e67389 100644 --- a/eval/src/vespa/eval/instruction/sum_max_dot_product_function.cpp +++ b/eval/src/vespa/eval/instruction/sum_max_dot_product_function.cpp @@ -91,7 +91,7 @@ SumMaxDotProductFunction::SumMaxDotProductFunction(const ValueType &res_type_in, } InterpretedFunction::Instruction -SumMaxDotProductFunction::compile_self(const ValueBuilderFactory &, Stash &) const +SumMaxDotProductFunction::compile_self(const CTFContext &) const { return InterpretedFunction::Instruction(my_sum_max_dot_product_op, _dp_size); } diff --git a/eval/src/vespa/eval/instruction/sum_max_dot_product_function.h b/eval/src/vespa/eval/instruction/sum_max_dot_product_function.h index 23bc102e1f3..a9b0152251b 100644 --- a/eval/src/vespa/eval/instruction/sum_max_dot_product_function.h +++ b/eval/src/vespa/eval/instruction/sum_max_dot_product_function.h @@ -40,7 +40,7 @@ public: const TensorFunction &query, const TensorFunction &document, size_t dp_size); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; size_t dp_size() const { return _dp_size; } bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); diff --git a/eval/src/vespa/eval/instruction/universal_dot_product.cpp b/eval/src/vespa/eval/instruction/universal_dot_product.cpp index 0271fb9ce86..828577790ef 100644 --- a/eval/src/vespa/eval/instruction/universal_dot_product.cpp +++ b/eval/src/vespa/eval/instruction/universal_dot_product.cpp @@ -200,9 +200,9 @@ UniversalDotProduct::UniversalDotProduct(const ValueType &res_type_in, } InterpretedFunction::Instruction -UniversalDotProduct::compile_self(const ValueBuilderFactory &, Stash &stash) const +UniversalDotProduct::compile_self(const CTFContext &ctx) const { - auto ¶m = stash.create(result_type(), lhs().result_type(), rhs().result_type()); + auto ¶m = ctx.stash.create(result_type(), lhs().result_type(), rhs().result_type()); using MyTypify = TypifyValue; auto op = typify_invoke<6,MyTypify,SelectUniversalDotProduct>(lhs().result_type().cell_meta(), rhs().result_type().cell_meta(), diff --git a/eval/src/vespa/eval/instruction/universal_dot_product.h b/eval/src/vespa/eval/instruction/universal_dot_product.h index 14dbd6e1766..6c40c7bdc68 100644 --- a/eval/src/vespa/eval/instruction/universal_dot_product.h +++ b/eval/src/vespa/eval/instruction/universal_dot_product.h @@ -17,7 +17,7 @@ class UniversalDotProduct : public tensor_function::Op2 { public: UniversalDotProduct(const ValueType &res_type, const TensorFunction &lhs, const TensorFunction &rhs); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } bool forward() const; bool distinct() const; diff --git a/eval/src/vespa/eval/instruction/unpack_bits_function.cpp b/eval/src/vespa/eval/instruction/unpack_bits_function.cpp index 117bac79fe3..d0a75b03c04 100644 --- a/eval/src/vespa/eval/instruction/unpack_bits_function.cpp +++ b/eval/src/vespa/eval/instruction/unpack_bits_function.cpp @@ -178,7 +178,7 @@ UnpackBitsFunction::UnpackBitsFunction(const ValueType &res_type_in, } InterpretedFunction::Instruction -UnpackBitsFunction::compile_self(const ValueBuilderFactory &, Stash &) const +UnpackBitsFunction::compile_self(const CTFContext &) const { const ValueType &res_type = result_type(); auto op = typify_invoke<2,MyTypify,MyGetFun>(res_type.cell_type(), _big_bitorder); diff --git a/eval/src/vespa/eval/instruction/unpack_bits_function.h b/eval/src/vespa/eval/instruction/unpack_bits_function.h index 52360e7a4e2..02cee50ef4d 100644 --- a/eval/src/vespa/eval/instruction/unpack_bits_function.h +++ b/eval/src/vespa/eval/instruction/unpack_bits_function.h @@ -30,7 +30,7 @@ private: bool _big_bitorder; public: UnpackBitsFunction(const ValueType &res_type_in, const TensorFunction &packed, bool big); - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/eval/src/vespa/eval/instruction/vector_from_doubles_function.cpp b/eval/src/vespa/eval/instruction/vector_from_doubles_function.cpp index cd262bcbec1..d8e1d7cb52f 100644 --- a/eval/src/vespa/eval/instruction/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/instruction/vector_from_doubles_function.cpp @@ -86,7 +86,7 @@ VectorFromDoublesFunction::push_children(std::vector &target) const } InterpretedFunction::Instruction -VectorFromDoublesFunction::compile_self(const ValueBuilderFactory &, Stash &) const +VectorFromDoublesFunction::compile_self(const CTFContext &) const { return InterpretedFunction::Instruction(my_vector_from_doubles_op, wrap_param(_self)); } diff --git a/eval/src/vespa/eval/instruction/vector_from_doubles_function.h b/eval/src/vespa/eval/instruction/vector_from_doubles_function.h index 6af873f18d1..cb61dc04ec7 100644 --- a/eval/src/vespa/eval/instruction/vector_from_doubles_function.h +++ b/eval/src/vespa/eval/instruction/vector_from_doubles_function.h @@ -32,7 +32,7 @@ public: return _self.resultType.dimensions()[0].name; } size_t size() const { return _self.resultSize; } - InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + InterpretedFunction::Instruction compile_self(const CTFContext &ctx) const override; bool result_is_mutable() const override { return true; } static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; -- cgit v1.2.3