diff options
Diffstat (limited to 'eval/src')
41 files changed, 1444 insertions, 232 deletions
diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 6fa89989bb7..4827fa3be3c 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -2,8 +2,7 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/tensor_plans.h> -#include <vespa/eval/eval/tensor_instructions.h> +#include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> @@ -11,6 +10,7 @@ using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::instruction; using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; @@ -70,7 +70,7 @@ TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_ const auto &factory = SimpleValueBuilderFactory::get(); auto lhs = value_from_spec(a, factory); auto rhs = value_from_spec(b, factory); - auto my_op = tensor_instruction::make_join(lhs->type(), rhs->type(), function, factory, stash); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); InterpretedFunction::EvalSingle single(my_op); return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); } @@ -115,48 +115,6 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { EXPECT_FALSE(view->next_result({&label}, subspace)); } -TEST(SimpleValueTest, dense_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); - auto plan = DenseJoinPlan(lhs, rhs); - std::vector<size_t> expect_loop = {30,4,6}; - std::vector<size_t> expect_lhs_stride = {6,0,1}; - std::vector<size_t> expect_rhs_stride = {4,1,0}; - EXPECT_EQ(plan.lhs_size, 180); - EXPECT_EQ(plan.rhs_size, 120); - EXPECT_EQ(plan.out_size, 720); - EXPECT_EQ(plan.loop_cnt, expect_loop); - EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); - EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); -} - -TEST(SimpleValueTest, sparse_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); - auto plan = SparseJoinPlan(lhs, rhs); - using SRC = SparseJoinPlan::Source; - std::vector<SRC> expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; - std::vector<size_t> expect_lhs_overlap = {1}; - std::vector<size_t> expect_rhs_overlap = {0}; - EXPECT_EQ(plan.sources, expect_sources); - EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); - EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); -} - -TEST(SimpleValueTest, dense_join_plan_can_be_executed) { - auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), - ValueType::from_spec("tensor(b[3])")); - std::vector<int> a({1, 2}); - std::vector<int> b({3, 4, 5}); - std::vector<int> c(6, 0); - std::vector<int> expect = {3,4,5,6,8,10}; - ASSERT_EQ(plan.out_size, 6); - int *dst = &c[0]; - auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; - plan.execute(0, 0, cell_join); - EXPECT_EQ(c, expect); -} - TEST(SimpleValueTest, new_generic_join_works_for_simple_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp index 9c87a3384c4..2b03cffe730 100644 --- a/eval/src/tests/eval/value_codec/value_codec_test.cpp +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -88,7 +88,7 @@ struct TensorExample { virtual void encode_default(nbostream &dst) const = 0; virtual void encode_with_double(nbostream &dst) const = 0; virtual void encode_with_float(nbostream &dst) const = 0; - void verify_encode_decode() const { + void verify_encode_decode(bool is_dense) const { nbostream expect_default; nbostream expect_double; nbostream expect_float; @@ -99,10 +99,15 @@ struct TensorExample { nbostream data_float; encode_value(*make_tensor(false), data_double); encode_value(*make_tensor(true), data_float); - EXPECT_EQ(Memory(data_double.peek(), data_double.size()), - Memory(expect_default.peek(), expect_default.size())); - EXPECT_EQ(Memory(data_float.peek(), data_float.size()), - Memory(expect_float.peek(), expect_float.size())); + if (is_dense) { + EXPECT_EQ(Memory(data_double.peek(), data_double.size()), + Memory(expect_default.peek(), expect_default.size())); + EXPECT_EQ(Memory(data_float.peek(), data_float.size()), + Memory(expect_float.peek(), expect_float.size())); + } else { + EXPECT_EQ(spec_from_value(*decode_value(data_double, factory)), make_spec(false)); + EXPECT_EQ(spec_from_value(*decode_value(data_float, factory)), make_spec(true)); + } EXPECT_EQ(spec_from_value(*decode_value(expect_default, factory)), make_spec(false)); EXPECT_EQ(spec_from_value(*decode_value(expect_double, factory)), make_spec(false)); EXPECT_EQ(spec_from_value(*decode_value(expect_float, factory)), make_spec(true)); @@ -156,7 +161,7 @@ struct SparseTensorExample : TensorExample { TEST(ValueCodecTest, sparse_tensors_can_be_encoded_and_decoded) { SparseTensorExample f1; - f1.verify_encode_decode(); + f1.verify_encode_decode(false); } //----------------------------------------------------------------------------- @@ -206,7 +211,7 @@ struct DenseTensorExample : TensorExample { TEST(ValueCodecTest, dense_tensors_can_be_encoded_and_decoded) { DenseTensorExample f1; - f1.verify_encode_decode(); + f1.verify_encode_decode(true); } TEST(ValueCodecTest, dense_tensors_without_values_are_filled) { @@ -277,7 +282,7 @@ struct MixedTensorExample : TensorExample { TEST(ValueCodecTest, mixed_tensors_can_be_encoded_and_decoded) { MixedTensorExample f1; - f1.verify_encode_decode(); + f1.verify_encode_decode(false); } //----------------------------------------------------------------------------- diff --git a/eval/src/tests/instruction/generic_join/CMakeLists.txt b/eval/src/tests/instruction/generic_join/CMakeLists.txt new file mode 100644 index 00000000000..13fc6550d3c --- /dev/null +++ b/eval/src/tests/instruction/generic_join/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_generic_join_test_app TEST + SOURCES + generic_join_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_generic_join_test_app COMMAND eval_generic_join_test_app) diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp new file mode 100644 index 00000000000..4821bf092da --- /dev/null +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -0,0 +1,138 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::instruction; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> join_layouts = { + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +}; + +bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, TensorSpec::Address &addr) { + for (const auto &dim_a: a) { + auto pos_b = b.find(dim_a.first); + if ((pos_b != b.end()) && !(pos_b->second == dim_a.second)) { + return false; + } + addr.insert_or_assign(dim_a.first, dim_a.second); + } + return true; +} + +TensorSpec reference_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + ValueType res_type = ValueType::join(ValueType::from_spec(a.type()), ValueType::from_spec(b.type())); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell_a: a.cells()) { + for (const auto &cell_b: b.cells()) { + TensorSpec::Address addr; + if (join_address(cell_a.first, cell_b.first, addr) && + join_address(cell_b.first, cell_a.first, addr)) + { + result.add(addr, function(cell_a.second, cell_b.second)); + } + } + } + return result; +} + +TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); + auto plan = DenseJoinPlan(lhs, rhs); + std::vector<size_t> expect_loop = {30,4,6}; + std::vector<size_t> expect_lhs_stride = {6,0,1}; + std::vector<size_t> expect_rhs_stride = {4,1,0}; + EXPECT_EQ(plan.lhs_size, 180); + EXPECT_EQ(plan.rhs_size, 120); + EXPECT_EQ(plan.out_size, 720); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); + EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); +} + +TEST(GenericJoinTest, sparse_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); + auto plan = SparseJoinPlan(lhs, rhs); + using SRC = SparseJoinPlan::Source; + std::vector<SRC> expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; + std::vector<size_t> expect_lhs_overlap = {1}; + std::vector<size_t> expect_rhs_overlap = {0}; + EXPECT_EQ(plan.sources, expect_sources); + EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); + EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_executed) { + auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), + ValueType::from_spec("tensor(b[3])")); + std::vector<int> a({1, 2}); + std::vector<int> b({3, 4, 5}); + std::vector<int> c(6, 0); + std::vector<int> expect = {3,4,5,6,8,10}; + ASSERT_EQ(plan.out_size, 6); + int *dst = &c[0]; + auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; + plan.execute(0, 0, cell_join); + EXPECT_EQ(c, expect); +} + +TEST(GenericJoinTest, generic_join_works_for_simple_values) { + ASSERT_TRUE((join_layouts.size() % 2) == 0); + for (size_t i = 0; i < join_layouts.size(); i += 2) { + TensorSpec lhs = spec(join_layouts[i], Div16(N())); + TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = reference_join(lhs, rhs, fun); + auto actual = perform_generic_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp index 31e82a01070..f61899e4dda 100644 --- a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -10,6 +10,7 @@ using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::instruction; using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; @@ -50,7 +51,7 @@ TEST(GenericRenameTest, dense_rename_plan_can_be_created_and_executed) { std::vector<vespalib::string> from({"a", "c", "e"}); std::vector<vespalib::string> to({"f", "a", "b"}); ValueType renamed = lhs.rename(from, to); - auto plan = instruction::DenseRenamePlan(lhs, renamed, from, to); + auto plan = DenseRenamePlan(lhs, renamed, from, to); std::vector<size_t> expect_loop = {15,2,7}; std::vector<size_t> expect_stride = {7,105,1}; EXPECT_EQ(plan.subspace_size, 210); @@ -80,18 +81,34 @@ TEST(GenericRenameTest, sparse_rename_plan_can_be_created) { std::vector<vespalib::string> from({"a", "c", "e"}); std::vector<vespalib::string> to({"f", "a", "b"}); ValueType renamed = lhs.rename(from, to); - auto plan = instruction::SparseRenamePlan(lhs, renamed, from, to); + auto plan = SparseRenamePlan(lhs, renamed, from, to); EXPECT_EQ(plan.mapped_dims, 4); std::vector<size_t> expect = {2,0,1,3}; EXPECT_EQ(plan.output_dimensions, expect); } -TensorSpec simple_tensor_rename(const TensorSpec &a, const FromTo &ft) { - Stash stash; - const auto &engine = SimpleTensorEngine::ref(); - auto lhs = engine.from_spec(a); - const auto &result = engine.rename(*lhs, ft.from, ft.to, stash); - return engine.to_spec(result); +vespalib::string rename_dimension(const vespalib::string &name, const FromTo &ft) { + assert(ft.from.size() == ft.to.size()); + for (size_t i = 0; i < ft.from.size(); ++i) { + if (name == ft.from[i]) { + return ft.to[i]; + } + } + return name; +} + +TensorSpec reference_rename(const TensorSpec &a, const FromTo &ft) { + ValueType res_type = ValueType::from_spec(a.type()).rename(ft.from, ft.to); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell: a.cells()) { + TensorSpec::Address addr; + for (const auto &dim: cell.first) { + addr.insert_or_assign(rename_dimension(dim.first, ft), dim.second); + } + result.add(addr, cell.second); + } + return result; } TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type, @@ -99,7 +116,7 @@ TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type { Stash stash; auto lhs = value_from_spec(a, factory); - auto my_op = instruction::GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); + auto my_op = GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); InterpretedFunction::EvalSingle single(my_op); return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs}))); } @@ -114,7 +131,7 @@ void test_generic_rename(const ValueBuilderFactory &factory) { if (renamed_type.is_error()) continue; // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = simple_tensor_rename(lhs, from_to); + auto expect = reference_rename(lhs, from_to); auto actual = perform_generic_rename(lhs, renamed_type, from_to, factory); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt b/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt new file mode 100644 index 00000000000..cd7f552ec28 --- /dev/null +++ b/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_default_value_builder_factory_test_app TEST + SOURCES + default_value_builder_factory_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_default_value_builder_factory_test_app COMMAND eval_default_value_builder_factory_test_app ) diff --git a/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp b/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp new file mode 100644 index 00000000000..d180b3f6517 --- /dev/null +++ b/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp @@ -0,0 +1,61 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/tensor/default_value_builder_factory.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor.h> +#include <vespa/eval/tensor/sparse/sparse_tensor_value.h> +#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::tensor; +using namespace vespalib::eval::packed_mixed_tensor; + +Value::UP v_of(const TensorSpec &spec) { + return value_from_spec(spec, DefaultValueBuilderFactory::get()); +} + +TEST(DefaultValueBuilderFactoryTest, all_built_value_types_are_correct) { + auto dbl = v_of(TensorSpec("double").add({}, 3.0)); + auto trivial = v_of(TensorSpec("tensor(x[1])").add({{"x",0}}, 7.0)); + auto dense = v_of(TensorSpec("tensor<float>(x[2],y[3])").add({{"x",1},{"y",2}}, 17.0)); + auto sparse = v_of(TensorSpec("tensor(x{},y{})").add({{"x","foo"},{"y","bar"}}, 31.0)); + auto mixed = v_of(TensorSpec("tensor<float>(x[2],y{})").add({{"x",1},{"y","quux"}}, 42.0)); + + EXPECT_TRUE(dynamic_cast<DoubleValue *>(dbl.get())); + EXPECT_TRUE(dynamic_cast<DenseTensorView *>(trivial.get())); + EXPECT_TRUE(dynamic_cast<DenseTensorView *>(dense.get())); + EXPECT_TRUE(dynamic_cast<SparseTensorValue<double> *>(sparse.get())); + EXPECT_TRUE(dynamic_cast<PackedMixedTensor *>(mixed.get())); + + EXPECT_EQ(dbl->as_double(), 3.0); + EXPECT_EQ(trivial->cells().typify<double>()[0], 7.0); + EXPECT_EQ(dense->cells().typify<float>()[5], 17.0); + EXPECT_EQ(sparse->cells().typify<double>()[0], 31.0); + EXPECT_EQ(mixed->cells().typify<float>()[1], 42.0); + + stringref y_look = "bar"; + stringref x_res = "xxx"; + auto view = sparse->index().create_view({1}); + view->lookup({&y_look}); + size_t ss = 12345; + bool br = view->next_result({&x_res}, ss); + EXPECT_TRUE(br); + EXPECT_EQ(ss, 0); + EXPECT_EQ(x_res, "foo"); + br = view->next_result({&x_res}, ss); + EXPECT_FALSE(br); + + ss = 12345; + view = mixed->index().create_view({}); + view->lookup({}); + br = view->next_result({&x_res}, ss); + EXPECT_TRUE(br); + EXPECT_EQ(ss, 0); + EXPECT_EQ(x_res, "quux"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/instruction_benchmark/.gitignore b/eval/src/tests/tensor/instruction_benchmark/.gitignore new file mode 100644 index 00000000000..31b087883e0 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/.gitignore @@ -0,0 +1 @@ +/eval_instruction_benchmark_app diff --git a/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt new file mode 100644 index 00000000000..d2384eaf129 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_instruction_benchmark_app TEST + SOURCES + instruction_benchmark.cpp + DEPENDS + vespaeval + GTest::GTest +) diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp new file mode 100644 index 00000000000..31777e233f6 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -0,0 +1,400 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// Microbenchmark exploring performance differences between +// interpreted function instructions. + +// This benchmark was initially written to measure the difference in +// performance between (old) instructions using the TensorEngine +// immediate API and (new) instructions using the Value API +// directly. Note that all previous optimizations for dense tensors +// are trivially transformed to use the Value API, and thus only the +// generic cases need to be compared. Specifically; we want to make +// sure join performance for sparse tensors with full dimensional +// overlap does not suffer too much. Also, we want to showcase an +// improvement in generic dense join and possibly also in sparse join +// with partial dimensional overlap. Benchmarks are done using float +// cells since this is what gives best overall performance in +// production. Also, we use the multiply operation since it is the +// most optimized operations across all implementations. When +// benchmarking different implementations against each other, a smoke +// test is performed by verifying that all implementations produce the +// same result. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/default_value_builder_factory.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <optional> +#include <algorithm> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::tensor; +using namespace vespalib::eval::instruction; +using vespalib::make_string_short::fmt; + +using Instruction = InterpretedFunction::Instruction; +using EvalSingle = InterpretedFunction::EvalSingle; + +template <typename T> using CREF = std::reference_wrapper<const T>; + +//----------------------------------------------------------------------------- + +struct Impl { + size_t order; + vespalib::string name; + vespalib::string short_name; + Impl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in) + : order(order_in), name(name_in), short_name(short_name_in) {} + virtual Value::UP create_value(const TensorSpec &spec) const = 0; + virtual TensorSpec create_spec(const Value &value) const = 0; + virtual Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0; + virtual const TensorEngine &engine() const { return SimpleTensorEngine::ref(); } // engine used by EvalSingle + virtual ~Impl() {} +}; + +struct ValueImpl : Impl { + const ValueBuilderFactory &my_factory; + ValueImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const ValueBuilderFactory &factory) + : Impl(order_in, name_in, short_name_in), my_factory(factory) {} + Value::UP create_value(const TensorSpec &spec) const override { return value_from_spec(spec, my_factory); } + TensorSpec create_spec(const Value &value) const override { return spec_from_value(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + return GenericJoin::make_instruction(lhs, rhs, function, my_factory, stash); + } +}; + +struct EngineImpl : Impl { + const TensorEngine &my_engine; + EngineImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const TensorEngine &engine_in) + : Impl(order_in, name_in, short_name_in), my_engine(engine_in) {} + Value::UP create_value(const TensorSpec &spec) const override { return my_engine.from_spec(spec); } + TensorSpec create_spec(const Value &value) const override { return my_engine.to_spec(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + // create a complete tensor function joining two parameters, but only compile the join instruction itself + const auto &lhs_node = tensor_function::inject(lhs, 0, stash); + const auto &rhs_node = tensor_function::inject(rhs, 1, stash); + const auto &join_node = tensor_function::join(lhs_node, rhs_node, function, stash); + return join_node.compile_self(my_engine, stash); + } + const TensorEngine &engine() const override { return my_engine; } +}; + +//----------------------------------------------------------------------------- + +EngineImpl simple_tensor_engine_impl(4, " SimpleTensorEngine", " SimpleT", SimpleTensorEngine::ref()); +EngineImpl default_tensor_engine_impl(1, "DefaultTensorEngine", "OLD PROD", DefaultTensorEngine::ref()); +ValueImpl simple_value_impl(3, " SimpleValue", " SimpleV", SimpleValueBuilderFactory::get()); +ValueImpl packed_mixed_tensor_impl(2, " PackedMixedTensor", " Packed", PackedMixedTensorBuilderFactory::get()); +ValueImpl default_tensor_value_impl(0, " DefaultValue", "NEW PROD", DefaultValueBuilderFactory::get()); +vespalib::string short_header("--------"); + +double budget = 5.0; +std::vector<CREF<Impl>> impl_list = {simple_tensor_engine_impl, + default_tensor_engine_impl, + simple_value_impl, + packed_mixed_tensor_impl, + default_tensor_value_impl}; + +//----------------------------------------------------------------------------- + +struct BenchmarkHeader { + std::vector<vespalib::string> short_names; + BenchmarkHeader() : short_names() { + short_names.resize(impl_list.size()); + for (const Impl &impl: impl_list) { + short_names[impl.order] = impl.short_name; + } + } + void print_trailer() const { + for (size_t i = 0; i < short_names.size(); ++i) { + fprintf(stderr, "+%s", short_header.c_str()); + } + fprintf(stderr, "+------------------------------------------------\n"); + } + void print() const { + for (const auto &name: short_names) { + fprintf(stderr, "|%s", name.c_str()); + } + fprintf(stderr, "| Benchmark description\n"); + print_trailer(); + } +}; + +struct BenchmarkResult { + vespalib::string desc; + std::optional<double> ref_time; + std::vector<double> relative_perf; + BenchmarkResult(const vespalib::string &desc_in, size_t num_values) + : desc(desc_in), ref_time(std::nullopt), relative_perf(num_values, 0.0) {} + ~BenchmarkResult(); + void sample(size_t order, double time) { + relative_perf[order] = time; + if (order == 1) { + if (ref_time.has_value()) { + ref_time = std::min(ref_time.value(), time); + } else { + ref_time = time; + } + } + } + void normalize() { + for (double &perf: relative_perf) { + perf = ref_time.value() / perf; + } + } + void print() const { + for (double perf: relative_perf) { + fprintf(stderr, "|%8.2f", perf); + } + fprintf(stderr, "| %s\n", desc.c_str()); + } +}; +BenchmarkResult::~BenchmarkResult() = default; + +std::vector<BenchmarkResult> benchmark_results; + +//----------------------------------------------------------------------------- + +struct EvalOp { + using UP = std::unique_ptr<EvalOp>; + const Impl &impl; + std::vector<Value::UP> values; + std::vector<Value::CREF> stack; + EvalSingle single; + EvalOp(const EvalOp &) = delete; + EvalOp &operator=(const EvalOp &) = delete; + EvalOp(Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) + : impl(impl_in), values(), stack(), single(impl.engine(), op) + { + for (const TensorSpec &spec: stack_spec) { + values.push_back(impl.create_value(spec)); + } + for (const auto &value: values) { + stack.push_back(*value.get()); + } + } + TensorSpec result() { return impl.create_spec(single.eval(stack)); } + double estimate_cost_us() { + auto actual = [&](){ single.eval(stack); }; + return BenchmarkTimer::benchmark(actual, budget) * 1000.0 * 1000.0; + } +}; + +//----------------------------------------------------------------------------- + +void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list) { + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "Benchmark Case: [%s]\n", desc.c_str()); + std::optional<TensorSpec> expect = std::nullopt; + for (const auto &eval: list) { + if (expect.has_value()) { + ASSERT_EQ(eval->result(), expect.value()); + } else { + expect = eval->result(); + } + } + BenchmarkResult result(desc, list.size()); + for (const auto &eval: list) { + double time = eval->estimate_cost_us(); + result.sample(eval->impl.order, time); + fprintf(stderr, " %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time); + } + result.normalize(); + benchmark_results.push_back(result); + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +void benchmark_join(const vespalib::string &desc, const TensorSpec &lhs, + const TensorSpec &rhs, operation::op2_t function) +{ + Stash stash; + ValueType lhs_type = ValueType::from_spec(lhs.type()); + ValueType rhs_type = ValueType::from_spec(rhs.type()); + ValueType res_type = ValueType::join(lhs_type, rhs_type); + ASSERT_FALSE(lhs_type.is_error()); + ASSERT_FALSE(rhs_type.is_error()); + ASSERT_FALSE(res_type.is_error()); + std::vector<EvalOp::UP> list; + for (const Impl &impl: impl_list) { + auto op = impl.create_join(lhs_type, rhs_type, function, stash); + std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); + list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + } + benchmark(desc, list); +} + +//----------------------------------------------------------------------------- + +struct D { + vespalib::string name; + bool mapped; + size_t size; + size_t stride; + static D map(const vespalib::string &name_in, size_t size_in, size_t stride_in) { return D{name_in, true, size_in, stride_in}; } + static D idx(const vespalib::string &name_in, size_t size_in) { return D{name_in, false, size_in, 1}; } + operator ValueType::Dimension() const { + if (mapped) { + return ValueType::Dimension(name); + } else { + return ValueType::Dimension(name, size); + } + } + TensorSpec::Label operator()(size_t idx) const { + if (mapped) { + return TensorSpec::Label(fmt("label_%zu", idx)); + } else { + return TensorSpec::Label(idx); + } + } +}; + +void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr) { + spec.add(addr, seq); + seq += 1.0; +} + +template <typename ...Ds> void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr, const D &d, const Ds &...ds) { + for (size_t i = 0, idx = 0; i < d.size; ++i, idx += d.stride) { + addr.insert_or_assign(d.name, d(idx)); + add_cells(spec, seq, addr, ds...); + } +} + +template <typename ...Ds> TensorSpec make_spec(double seq, const Ds &...ds) { + TensorSpec spec(ValueType::tensor_type({ds...}, ValueType::CellType::FLOAT).to_spec()); + add_cells(spec, seq, TensorSpec::Address(), ds...); + return spec; +} + +TensorSpec make_vector(const D &d1, double seq) { return make_spec(seq, d1); } +TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { return make_spec(seq, d1, d2, d3); } + +//----------------------------------------------------------------------------- + +TEST(MakeInputTest, print_some_test_input) { + auto number = make_spec(5.0); + auto sparse = make_vector(D::map("x", 5, 3), 1.0); + auto dense = make_vector(D::idx("x", 5), 10.0); + auto mixed = make_cube(D::map("x", 3, 7), D::idx("y", 2), D::idx("z", 2), 100.0); + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "simple number: %s\n", number.to_string().c_str()); + fprintf(stderr, "sparse vector: %s\n", sparse.to_string().c_str()); + fprintf(stderr, "dense vector: %s\n", dense.to_string().c_str()); + fprintf(stderr, "mixed cube: %s\n", mixed.to_string().c_str()); + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +TEST(NumberJoin, plain_op2) { + auto lhs = make_spec(2.0); + auto rhs = make_spec(3.0); + benchmark_join("simple numbers multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(DenseJoin, small_vectors) { + auto lhs = make_vector(D::idx("x", 10), 1.0); + auto rhs = make_vector(D::idx("x", 10), 2.0); + benchmark_join("small dense vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, full_overlap) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 2.0); + benchmark_join("dense full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, partial_overlap) { + auto lhs = make_cube(D::idx("a", 8), D::idx("c", 8), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::idx("b", 8), D::idx("c", 8), D::idx("d", 8), 2.0); + benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, no_overlap) { + auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); + benchmark_join("dense no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(SparseJoin, small_vectors) { + auto lhs = make_vector(D::map("x", 10, 1), 1.0); + auto rhs = make_vector(D::map("x", 10, 2), 2.0); + benchmark_join("small sparse vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::map("c", 16, 2), 2.0); + benchmark_join("sparse full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap_big_vs_small) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 2, 1), D::map("b", 2, 1), D::map("c", 2, 1), 2.0); + benchmark_join("sparse full overlap big vs small multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, partial_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::map("d", 8, 1), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::map("d", 8, 2), 2.0); + benchmark_join("sparse partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::map("f", 4, 1), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::map("d", 4, 1), 2.0); + benchmark_join("sparse no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(MixedJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::idx("c", 16), 2.0); + benchmark_join("mixed full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, partial_sparse_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::idx("d", 8), 2.0); + benchmark_join("mixed partial sparse overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::idx("d", 4), 2.0); + benchmark_join("mixed no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(PrintResults, print_results) { + BenchmarkHeader header; + std::sort(benchmark_results.begin(), benchmark_results.end(), + [](const auto &a, const auto &b){ return (a.relative_perf[0] < b.relative_perf[0]); }); + header.print(); + for (const auto &result: benchmark_results) { + result.print(); + } + header.print_trailer(); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index 70f98904809..c8814372bf5 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -4,8 +4,8 @@ #include <vespa/eval/tensor/mixed/packed_mappings.h> #include <vespa/eval/tensor/mixed/packed_mappings_builder.h> #include <vespa/eval/tensor/mixed/packed_mixed_tensor.h> -#include <vespa/eval/tensor/mixed/packed_mixed_builder.h> -#include <vespa/eval/tensor/mixed/packed_mixed_factory.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> #include <vespa/vespalib/gtest/gtest.h> #include <stdlib.h> #include <assert.h> @@ -137,7 +137,7 @@ TEST_F(MappingsBuilderTest, some_random) class MixedBuilderTest : public ::testing::Test { public: - std::unique_ptr<PackedMixedBuilder<float>> builder; + std::unique_ptr<PackedMixedTensorBuilder<float>> builder; std::unique_ptr<Value> built; MixedBuilderTest() = default; @@ -170,7 +170,7 @@ TEST_F(MixedBuilderTest, empty_mapping) size_t dsss = type.dense_subspace_size(); EXPECT_GT(dims, 0); EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedBuilder<float>>(type, dims, dsss, 3); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); build_and_compare(0); } } @@ -183,7 +183,7 @@ TEST_F(MixedBuilderTest, just_one) size_t dims = type.count_mapped_dimensions(); size_t dsss = type.dense_subspace_size(); EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedBuilder<float>>(type, dims, dsss, 3); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); auto address = generate_random_address(dims); auto ref = builder->add_subspace(address); EXPECT_EQ(ref.size(), dsss); @@ -203,7 +203,7 @@ TEST_F(MixedBuilderTest, some_random) uint32_t dsss = type.dense_subspace_size(); EXPECT_GT(dims, 0); EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedBuilder<float>>(type, dims, dsss, 3); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); uint32_t cnt = random_range(dims*5, dims*20); printf("MixBuild: generate %u addresses for %u dims\n", cnt, dims); diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp index eb7607c13b8..bc1efdaba1d 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/test/tensor_model.hpp> -#include <vespa/eval/tensor/mixed/packed_mixed_factory.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib::eval; @@ -27,7 +27,7 @@ std::vector<Layout> layouts = { TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, PackedMixedFactory()); + std::unique_ptr<Value> value = value_from_spec(expect, PackedMixedTensorBuilderFactory::get()); TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } @@ -35,7 +35,7 @@ TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_s TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); - PackedMixedFactory factory; + const auto & factory = PackedMixedTensorBuilderFactory::get(); std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); float seq = 0.0; for (vespalib::string x: {"a", "b", "c"}) { diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 53ee6ed87c0..d108c516e73 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_library(eval_eval OBJECT call_nodes.cpp compile_tensor_function.cpp delete_node.cpp + double_value_builder.cpp fast_forest.cpp function.cpp gbdt.cpp @@ -25,9 +26,7 @@ vespa_add_library(eval_eval OBJECT tensor.cpp tensor_engine.cpp tensor_function.cpp - tensor_instructions.cpp tensor_nodes.cpp - tensor_plans.cpp tensor_spec.cpp value.cpp value_codec.cpp diff --git a/eval/src/vespa/eval/eval/double_value_builder.cpp b/eval/src/vespa/eval/eval/double_value_builder.cpp new file mode 100644 index 00000000000..24215104cee --- /dev/null +++ b/eval/src/vespa/eval/eval/double_value_builder.cpp @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "double_value_builder.h" + +namespace vespalib::eval { + +DoubleValueBuilder::~DoubleValueBuilder() = default; + +} diff --git a/eval/src/vespa/eval/eval/double_value_builder.h b/eval/src/vespa/eval/eval/double_value_builder.h new file mode 100644 index 00000000000..ba85d5838ad --- /dev/null +++ b/eval/src/vespa/eval/eval/double_value_builder.h @@ -0,0 +1,29 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" + +namespace vespalib::eval { + +/** + * A trivial builder for DoubleValue objects + **/ +class DoubleValueBuilder : public ValueBuilder<double> +{ +private: + double _value; +public: + DoubleValueBuilder() : _value(0.0) {} + ~DoubleValueBuilder() override; + ArrayRef<double> + add_subspace(const std::vector<vespalib::stringref> &) override { + return ArrayRef<double>(&_value, 1); + } + std::unique_ptr<Value> + build(std::unique_ptr<ValueBuilder<double>>) override { + return std::make_unique<DoubleValue>(_value); + } +}; + +} // namespace diff --git a/eval/src/vespa/eval/eval/tensor_instructions.h b/eval/src/vespa/eval/eval/tensor_instructions.h deleted file mode 100644 index f5c5e88d07c..00000000000 --- a/eval/src/vespa/eval/eval/tensor_instructions.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "interpreted_function.h" - -namespace vespalib { - -class Stash; - -namespace eval { - -struct JoinPlan; - -namespace tensor_instruction { - -using join_fun_t = double (*)(double, double); - -InterpretedFunction::Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, - const ValueBuilderFactory &factory, Stash &stash); - -} -} -} diff --git a/eval/src/vespa/eval/eval/tensor_plans.cpp b/eval/src/vespa/eval/eval/tensor_plans.cpp deleted file mode 100644 index 203ca72c92a..00000000000 --- a/eval/src/vespa/eval/eval/tensor_plans.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor_plans.h" -#include <vespa/vespalib/util/visit_ranges.h> -#include <vespa/vespalib/util/overload.h> -#include <cassert> - -namespace vespalib::eval { - -//----------------------------------------------------------------------------- - -DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() -{ - enum class Case { NONE, LHS, RHS, BOTH }; - Case prev_case = Case::NONE; - auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { - if (my_case == prev_case) { - assert(!loop_cnt.empty()); - loop_cnt.back() *= my_size; - } else { - loop_cnt.push_back(my_size); - lhs_stride.push_back(in_lhs); - rhs_stride.push_back(in_rhs); - prev_case = my_case; - } - }; - auto visitor = overload - { - [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, - [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, - [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } - }; - auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); - auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); - for (size_t i = loop_cnt.size(); i-- > 0; ) { - out_size *= loop_cnt[i]; - if (lhs_stride[i] != 0) { - lhs_stride[i] = lhs_size; - lhs_size *= loop_cnt[i]; - } - if (rhs_stride[i] != 0) { - rhs_stride[i] = rhs_size; - rhs_size *= loop_cnt[i]; - } - } -} - -DenseJoinPlan::~DenseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : sources(), lhs_overlap(), rhs_overlap() -{ - size_t lhs_idx = 0; - size_t rhs_idx = 0; - auto visitor = overload - { - [&](visit_ranges_first, const auto &) { - sources.push_back(Source::LHS); - ++lhs_idx; - }, - [&](visit_ranges_second, const auto &) { - sources.push_back(Source::RHS); - ++rhs_idx; - }, - [&](visit_ranges_both, const auto &, const auto &) { - sources.push_back(Source::BOTH); - lhs_overlap.push_back(lhs_idx++); - rhs_overlap.push_back(rhs_idx++); - } - }; - auto lhs_dims = lhs_type.mapped_dimensions(); - auto rhs_dims = rhs_type.mapped_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); -} - -SparseJoinPlan::~SparseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 20923fcd621..a084d267cec 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -100,6 +100,23 @@ public: }; /** + * A generic value without any mapped dimensions referencing its + * components without owning anything. + **/ +class DenseValueView final : public Value +{ +private: + const ValueType &_type; + TypedCells _cells; +public: + DenseValueView(const ValueType &type_in, TypedCells cells_in) + : _type(type_in), _cells(cells_in) {} + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return _cells; } + const Index &index() const final override { return TrivialIndex::get(); } +}; + +/** * Tagging interface used as return type from factories before * downcasting to actual builder with specialized cell type. **/ @@ -162,3 +179,4 @@ protected: } VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); +VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DenseValueView); diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index f5a7de96f0b..e5aae50750d 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -2,5 +2,6 @@ vespa_add_library(eval_instruction OBJECT SOURCES + generic_join generic_rename ) diff --git a/eval/src/vespa/eval/eval/tensor_instructions.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index d839bfdad50..b54f7d8952a 100644 --- a/eval/src/vespa/eval/eval/tensor_instructions.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -1,18 +1,19 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "tensor_instructions.h" -#include "tensor_plans.h" -#include "inline_operation.h" +#include "generic_join.h" +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <cassert> -namespace vespalib::eval::tensor_instruction { +namespace vespalib::eval::instruction { +using TypedCells = Value::TypedCells; using State = InterpretedFunction::State; using Instruction = InterpretedFunction::Instruction; -//----------------------------------------------------------------------------- - namespace { //----------------------------------------------------------------------------- @@ -96,7 +97,7 @@ struct SparseJoinState { SparseJoinState::~SparseJoinState() = default; template <typename LCT, typename RCT, typename OCT, typename Fun> -void my_generic_join(State &state, uint64_t param_in) { +void my_mixed_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); Fun fun(param.function); const Value &lhs = state.peek(1); @@ -121,9 +122,39 @@ void my_generic_join(State &state, uint64_t param_in) { state.pop_pop_push(result_ref); }; -struct SelectGenericJoin { - template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke() { - return my_generic_join<LCT,RCT,OCT,Fun>; +template <typename LCT, typename RCT, typename OCT, typename Fun> +void my_dense_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinParam>(param_in); + Fun fun(param.function); + auto lhs_cells = state.peek(1).cells().typify<LCT>(); + auto rhs_cells = state.peek(0).cells().typify<RCT>(); + ArrayRef<OCT> out_cells = state.stash.create_array<OCT>(param.dense_plan.out_size); + OCT *dst = out_cells.begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + param.dense_plan.execute(0, 0, join_cells); + state.pop_pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); +}; + +template <typename Fun> +void my_double_join_op(State &state, uint64_t param_in) { + Fun fun(unwrap_param<JoinParam>(param_in).function); + state.pop_pop_push(state.stash.create<DoubleValue>(fun(state.peek(1).cells().typify<double>()[0], + state.peek(0).cells().typify<double>()[0]))); +}; + +struct SelectGenericJoinOp { + template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke(const JoinParam ¶m) { + if (param.res_type.is_double()) { + bool all_double = (std::is_same_v<LCT, double> && + std::is_same_v<RCT, double> && + std::is_same_v<OCT, double>); + assert(all_double); + return my_double_join_op<Fun>; + } + if (param.sparse_plan.sources.empty()) { + return my_dense_join_op<LCT,RCT,OCT,Fun>; + } + return my_mixed_join_op<LCT,RCT,OCT,Fun>; } }; @@ -133,16 +164,95 @@ struct SelectGenericJoin { //----------------------------------------------------------------------------- -using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; +DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() +{ + enum class Case { NONE, LHS, RHS, BOTH }; + Case prev_case = Case::NONE; + auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { + if (my_case == prev_case) { + assert(!loop_cnt.empty()); + loop_cnt.back() *= my_size; + } else { + loop_cnt.push_back(my_size); + lhs_stride.push_back(in_lhs); + rhs_stride.push_back(in_rhs); + prev_case = my_case; + } + }; + auto visitor = overload + { + [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, + [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, + [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } + }; + auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + for (size_t i = loop_cnt.size(); i-- > 0; ) { + out_size *= loop_cnt[i]; + if (lhs_stride[i] != 0) { + lhs_stride[i] = lhs_size; + lhs_size *= loop_cnt[i]; + } + if (rhs_stride[i] != 0) { + rhs_stride[i] = rhs_size; + rhs_size *= loop_cnt[i]; + } + } +} -Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, - const ValueBuilderFactory &factory, Stash &stash) +DenseJoinPlan::~DenseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : sources(), lhs_overlap(), rhs_overlap() { - auto ¶m = stash.create<JoinParam>(lhs_type, rhs_type, function, factory); - auto fun = typify_invoke<4,JoinTypify,SelectGenericJoin>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); - return Instruction(fun, wrap_param<JoinParam>(param)); + size_t lhs_idx = 0; + size_t rhs_idx = 0; + auto visitor = overload + { + [&](visit_ranges_first, const auto &) { + sources.push_back(Source::LHS); + ++lhs_idx; + }, + [&](visit_ranges_second, const auto &) { + sources.push_back(Source::RHS); + ++rhs_idx; + }, + [&](visit_ranges_both, const auto &, const auto &) { + sources.push_back(Source::BOTH); + lhs_overlap.push_back(lhs_idx++); + rhs_overlap.push_back(rhs_idx++); + } + }; + auto lhs_dims = lhs_type.mapped_dimensions(); + auto rhs_dims = rhs_type.mapped_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); } +SparseJoinPlan::~SparseJoinPlan() = default; + //----------------------------------------------------------------------------- } + +//----------------------------------------------------------------------------- + +namespace vespalib::eval::instruction { + +using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; + +Instruction +GenericJoin::make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create<JoinParam>(lhs_type, rhs_type, function, factory); + auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function, param); + return Instruction(fun, wrap_param<JoinParam>(param)); +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/tensor_plans.h b/eval/src/vespa/eval/instruction/generic_join.h index 9a924a8ba48..25647452dff 100644 --- a/eval/src/vespa/eval/eval/tensor_plans.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -2,13 +2,23 @@ #pragma once -#include "value_type.h" -#include <vespa/vespalib/stllike/string.h> -#include <vector> +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/interpreted_function.h> -namespace vespalib::eval { +namespace vespalib { class Stash; } +namespace vespalib::eval { struct ValueBuilderFactory; } -class ValueBuilderFactory; +namespace vespalib::eval::instruction { + +using join_fun_t = double (*)(double, double); + +//----------------------------------------------------------------------------- + +struct GenericJoin { + static InterpretedFunction::Instruction + make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash); +}; //----------------------------------------------------------------------------- @@ -74,4 +84,4 @@ struct SparseJoinPlan { //----------------------------------------------------------------------------- -} +} // namespace diff --git a/eval/src/vespa/eval/instruction/generic_rename.h b/eval/src/vespa/eval/instruction/generic_rename.h index e4f0146b9a8..ca9f45bd341 100644 --- a/eval/src/vespa/eval/instruction/generic_rename.h +++ b/eval/src/vespa/eval/instruction/generic_rename.h @@ -7,7 +7,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -namespace vespalib::eval { class ValueBuilderFactory; } +namespace vespalib::eval { struct ValueBuilderFactory; } namespace vespalib::eval::instruction { diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index bc0a4d340b8..810dfd6d0b3 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(eval_tensor OBJECT SOURCES default_tensor_engine.cpp + default_value_builder_factory.cpp tensor.cpp tensor_address.cpp tensor_apply.cpp diff --git a/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp b/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp new file mode 100644 index 00000000000..74fd371e9a0 --- /dev/null +++ b/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "default_value_builder_factory.h" +#include <vespa/vespalib/util/typify.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/double_value_builder.h> +#include <vespa/eval/tensor/dense/dense_tensor_value_builder.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h> +#include <vespa/eval/tensor/sparse/sparse_tensor_value_builder.h> + +using namespace vespalib::eval; + +namespace vespalib::tensor { + +//----------------------------------------------------------------------------- + +namespace { + +struct CreateDefaultValueBuilderBase { + template <typename T> static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, + size_t num_mapped_dims, + size_t subspace_size, + size_t expected_subspaces) + { + assert(check_cell_type<T>(type.cell_type())); + if (type.is_double()) { + return std::make_unique<DoubleValueBuilder>(); + } + if (num_mapped_dims == 0) { + return std::make_unique<DenseTensorValueBuilder<T>>(type, subspace_size); + } + if (subspace_size == 1) { + return std::make_unique<SparseTensorValueBuilder<T>>(type, num_mapped_dims, expected_subspaces); + } + return std::make_unique<packed_mixed_tensor::PackedMixedTensorBuilder<T>>(type, num_mapped_dims, subspace_size, expected_subspaces); + } +}; + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +DefaultValueBuilderFactory::DefaultValueBuilderFactory() = default; +DefaultValueBuilderFactory DefaultValueBuilderFactory::_factory; + +std::unique_ptr<ValueBuilderBase> +DefaultValueBuilderFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped_dims, + size_t subspace_size, + size_t expected_subspaces) const +{ + return typify_invoke<1,TypifyCellType,CreateDefaultValueBuilderBase>(type.cell_type(), type, num_mapped_dims, subspace_size, expected_subspaces); +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/tensor/default_value_builder_factory.h b/eval/src/vespa/eval/tensor/default_value_builder_factory.h new file mode 100644 index 00000000000..67b1391ed78 --- /dev/null +++ b/eval/src/vespa/eval/tensor/default_value_builder_factory.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_type.h> + +namespace vespalib::tensor { + +/** + * A factory that can generate ValueBuilder + * objects appropriate for the requested type. + */ +struct DefaultValueBuilderFactory : eval::ValueBuilderFactory { +private: + DefaultValueBuilderFactory(); + static DefaultValueBuilderFactory _factory; + ~DefaultValueBuilderFactory() override {} +protected: + std::unique_ptr<eval::ValueBuilderBase> create_value_builder_base(const eval::ValueType &type, + size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +public: + static const DefaultValueBuilderFactory &get() { return _factory; } +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index b4e849a1dde..3a41fed132e 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -26,6 +26,7 @@ vespa_add_library(eval_tensor_dense OBJECT dense_tensor_modify.cpp dense_tensor_peek_function.cpp dense_tensor_reduce.cpp + dense_tensor_value_builder.cpp dense_tensor_view.cpp dense_xw_product_function.cpp index_lookup_table.cpp diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.cpp new file mode 100644 index 00000000000..6a5bb33ca06 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.cpp @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_tensor_value_builder.h" + +namespace vespalib::tensor { + +template<typename T> +DenseTensorValueBuilder<T>::DenseTensorValueBuilder(const eval::ValueType &type, + size_t subspace_size_in) + : _type(type), + _cells(subspace_size_in) +{ +} + +template<typename T> +DenseTensorValueBuilder<T>::~DenseTensorValueBuilder() = default; + +template class DenseTensorValueBuilder<float>; +template class DenseTensorValueBuilder<double>; + +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.h new file mode 100644 index 00000000000..c420be2c582 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_value_builder.h @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "dense_tensor.h" + +namespace vespalib::tensor { + +/** + * A builder for DenseTensor objects + **/ +template<typename T> +class DenseTensorValueBuilder : public eval::ValueBuilder<T> +{ +private: + eval::ValueType _type; + std::vector<T> _cells; +public: + DenseTensorValueBuilder(const eval::ValueType &type, size_t subspace_size_in); + ~DenseTensorValueBuilder() override; + ArrayRef<T> + add_subspace(const std::vector<vespalib::stringref> &) override { + return _cells; + } + std::unique_ptr<eval::Value> + build(std::unique_ptr<eval::ValueBuilder<T>>) override { + return std::make_unique<DenseTensor<T>>(std::move(_type), std::move(_cells)); + } +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt index 2621ec616c0..ceded3a7380 100644 --- a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(eval_tensor_mixed OBJECT packed_labels.cpp packed_mappings.cpp packed_mappings_builder.cpp - packed_mixed_factory.cpp + packed_mixed_tensor_builder_factory.cpp packed_mixed_tensor.cpp - packed_mixed_builder.cpp + packed_mixed_tensor_builder.cpp ) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp index a78a7423520..fdfe5957a3f 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -54,8 +54,8 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const ssize_t avail_sz = mem_end - mem_start; assert(needs_sz <= avail_sz); - uint32_t * int_store_mem = (uint32_t *) mem_start; - uint32_t * offsets_mem = (uint32_t *) (mem_start + int_store_size); + uint32_t * int_store_mem = (uint32_t *) (void *) mem_start; + uint32_t * offsets_mem = (uint32_t *) (void *) (mem_start + int_store_size); char * labels_mem = mem_start + int_store_size + label_offsets_size; ArrayRef<uint32_t> int_store_data(int_store_mem, int_store_cnt); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h index 60916b23565..604b1c94aeb 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -15,7 +15,7 @@ namespace vespalib::eval::packed_mixed_tensor { * An implementation of Value modeling a mixed tensor, * where all the data (cells and sparse address mappings) * can reside in a self-contained, contigous block of memory. - * Currently must be built by a PackedMixedBuilder. + * Currently must be built by a PackedMixedTensorBuilder. * Immutable (all data always const). **/ class PackedMixedTensor : public Value, public Value::Index @@ -33,7 +33,7 @@ private: _mappings(mappings) {} - template<typename T> friend class PackedMixedBuilder; + template<typename T> friend class PackedMixedTensorBuilder; public: ~PackedMixedTensor() override; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp index 1816e06bf25..75b307b1aa7 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp @@ -1,12 +1,12 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "packed_mixed_builder.h" +#include "packed_mixed_tensor_builder.h" namespace vespalib::eval::packed_mixed_tensor { template <typename T> ArrayRef<T> -PackedMixedBuilder<T>::add_subspace(const std::vector<vespalib::stringref> &addr) +PackedMixedTensorBuilder<T>::add_subspace(const std::vector<vespalib::stringref> &addr) { uint32_t idx = _mappings_builder.add_mapping_for(addr); size_t offset = idx * _subspace_size; @@ -20,7 +20,7 @@ PackedMixedBuilder<T>::add_subspace(const std::vector<vespalib::stringref> &addr template <typename T> std::unique_ptr<Value> -PackedMixedBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) +PackedMixedTensorBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) { size_t self_size = sizeof(PackedMixedTensor); size_t mappings_size = _mappings_builder.extra_memory(); @@ -39,7 +39,7 @@ PackedMixedBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) // copy cells: memcpy(cells_mem, &_cells[0], cells_size); - ConstArrayRef<T> cells((T *)cells_mem, _cells.size()); + ConstArrayRef<T> cells((T *)(void *) cells_mem, _cells.size()); PackedMixedTensor * built = new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); @@ -47,7 +47,7 @@ PackedMixedBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) return std::unique_ptr<PackedMixedTensor>(built); } -template class PackedMixedBuilder<float>; -template class PackedMixedBuilder<double>; +template class PackedMixedTensorBuilder<float>; +template class PackedMixedTensorBuilder<double>; } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h index c851b839756..c99762b7e8b 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h @@ -11,7 +11,7 @@ namespace vespalib::eval::packed_mixed_tensor { * appropriate for cell type T. **/ template <typename T> -class PackedMixedBuilder : public ValueBuilder<T> +class PackedMixedTensorBuilder : public ValueBuilder<T> { private: const ValueType & _type; @@ -19,7 +19,7 @@ private: std::vector<T> _cells; PackedMappingsBuilder _mappings_builder; public: - PackedMixedBuilder(const ValueType &type, + PackedMixedTensorBuilder(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, size_t expected_subspaces) @@ -31,7 +31,7 @@ public: _cells.reserve(_subspace_size * expected_subspaces); } - ~PackedMixedBuilder() override = default; + ~PackedMixedTensorBuilder() override = default; ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) override; std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>> self) override; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp index 75e6b1e996e..48eedd86f7f 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "packed_mixed_factory.h" -#include "packed_mixed_builder.h" +#include "packed_mixed_tensor_builder_factory.h" +#include "packed_mixed_tensor_builder.h" #include <vespa/vespalib/util/typify.h> @@ -9,24 +9,27 @@ namespace vespalib::eval { namespace { -struct CreatePackedMixedBuilder { +struct CreatePackedMixedTensorBuilder { template <typename T, typename ...Args> static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, Args &&...args) { assert(check_cell_type<T>(type.cell_type())); - return std::make_unique<packed_mixed_tensor::PackedMixedBuilder<T>>(type, std::forward<Args>(args)...); + return std::make_unique<packed_mixed_tensor::PackedMixedTensorBuilder<T>>(type, std::forward<Args>(args)...); } }; } // namespace <unnamed> +PackedMixedTensorBuilderFactory::PackedMixedTensorBuilderFactory() = default; +PackedMixedTensorBuilderFactory PackedMixedTensorBuilderFactory::_factory; + std::unique_ptr<ValueBuilderBase> -PackedMixedFactory::create_value_builder_base(const ValueType &type, +PackedMixedTensorBuilderFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, size_t expected_subspaces) const { - return typify_invoke<1,TypifyCellType,CreatePackedMixedBuilder>(type.cell_type(), + return typify_invoke<1,TypifyCellType,CreatePackedMixedTensorBuilder>(type.cell_type(), type, num_mapped_in, subspace_size_in, expected_subspaces); } diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h index ae90e5e243b..20a581e2b35 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h @@ -7,14 +7,19 @@ namespace vespalib::eval { /** - * A factory that can generate PackedMixedBuilder + * A factory that can generate PackedMixedTensorBuilder * objects appropriate for the requested CellType. */ -struct PackedMixedFactory : ValueBuilderFactory { - ~PackedMixedFactory() override {} +struct PackedMixedTensorBuilderFactory : ValueBuilderFactory { +private: + PackedMixedTensorBuilderFactory(); + static PackedMixedTensorBuilderFactory _factory; + ~PackedMixedTensorBuilderFactory() override {} protected: std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +public: + static const PackedMixedTensorBuilderFactory &get() { return _factory; } }; } // namespace diff --git a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt index a25d2abb477..91c609a59b7 100644 --- a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt @@ -11,4 +11,6 @@ vespa_add_library(eval_tensor_sparse OBJECT sparse_tensor_match.cpp sparse_tensor_modify.cpp sparse_tensor_remove.cpp + sparse_tensor_value.cpp + sparse_tensor_value_builder.cpp ) diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h index e053caf8604..32b9b57fb26 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h @@ -26,9 +26,10 @@ private: protected: void append(vespalib::stringref str) { - for (size_t i(0); i < str.size() + 1; i++) { + for (size_t i(0); i < str.size(); i++) { _address.push_back_fast(str[i]); } + _address.push_back_fast('\0'); } void ensure_room(size_t additional) { if (_address.capacity() < (_address.size() + additional)) { diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.cpp new file mode 100644 index 00000000000..62e3c786262 --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.cpp @@ -0,0 +1,260 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_tensor_value.h" +#include "sparse_tensor_address_builder.h" +#include "sparse_tensor_address_decoder.h" + +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/hash_map_equal.hpp> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.tensor.sparse.sparse_tensor_value"); + +namespace vespalib::tensor { + +using SubspaceMap = SparseTensorValueIndex::SubspaceMap; +using View = vespalib::eval::Value::Index::View; + +namespace { + +void copyMap(SubspaceMap &map, const SubspaceMap &map_in, Stash &to_stash) { + // copy the exact hashtable structure: + map = map_in; + // copy the actual contents of the addresses, + // and update the pointers inside the hashtable + // keys so they point to our copy: + for (auto & kv : map) { + SparseTensorAddressRef oldRef = kv.first; + SparseTensorAddressRef newRef(oldRef, to_stash); + kv.first = newRef; + } +} + +template<typename T> +size_t needed_memory_for(const SubspaceMap &map, ConstArrayRef<T> cells) { + size_t needs = cells.size() * sizeof(T); + for (const auto & kv : map) { + needs += kv.first.size(); + } + return needs; +} + +//----------------------------------------------------------------------------- + +class SparseTensorValueView : public View +{ +private: + const SubspaceMap ↦ + SubspaceMap::const_iterator iter; + const std::vector<size_t> lookup_dims; + std::vector<vespalib::stringref> lookup_refs; +public: + SparseTensorValueView(const SubspaceMap & map_in, + const std::vector<size_t> &dims) + : map(map_in), iter(map.end()), lookup_dims(dims), lookup_refs() {} + ~SparseTensorValueView(); + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; +}; + +SparseTensorValueView::~SparseTensorValueView() = default; + +void +SparseTensorValueView::lookup(const std::vector<const vespalib::stringref*> &addr) +{ + lookup_refs.clear(); + for (auto ptr : addr) { + lookup_refs.push_back(*ptr); + } + iter = map.begin(); + +} + +bool +SparseTensorValueView::next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) +{ + size_t total_dims = lookup_refs.size() + addr_out.size(); + while (iter != map.end()) { + const auto & ref = iter->first; + SparseTensorAddressDecoder decoder(ref); + idx_out = iter->second; + ++iter; + bool couldmatch = true; + size_t vd_idx = 0; + size_t ao_idx = 0; + for (size_t i = 0; i < total_dims; ++i) { + const auto label = decoder.decodeLabel(); + if (vd_idx < lookup_dims.size()) { + size_t next_view_dim = lookup_dims[vd_idx]; + if (i == next_view_dim) { + if (label == lookup_refs[vd_idx]) { + // match in this dimension + ++vd_idx; + continue; + } else { + // does not match + couldmatch = false; + break; + } + } + } + // not a view dimension: + *addr_out[ao_idx] = label; + ++ao_idx; + } + if (couldmatch) { + assert(vd_idx == lookup_dims.size()); + assert(ao_idx == addr_out.size()); + return true; + } + } + return false; +} + +//----------------------------------------------------------------------------- + +class SparseTensorValueLookup : public View +{ +private: + const SubspaceMap ↦ + SubspaceMap::const_iterator iter; +public: + SparseTensorValueLookup(const SubspaceMap & map_in) : map(map_in), iter(map.end()) {} + ~SparseTensorValueLookup(); + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; +}; + +SparseTensorValueLookup::~SparseTensorValueLookup() = default; + +void +SparseTensorValueLookup::lookup(const std::vector<const vespalib::stringref*> &addr) +{ + SparseTensorAddressBuilder builder; + for (const auto & label : addr) { + builder.add(*label); + } + auto ref = builder.getAddressRef(); + iter = map.find(ref); +} + +bool +SparseTensorValueLookup::next_result(const std::vector<vespalib::stringref*> &, size_t &idx_out) +{ + if (iter != map.end()) { + idx_out = iter->second; + iter = map.end(); + return true; + } + return false; +} + +//----------------------------------------------------------------------------- + +class SparseTensorValueAllMappings : public View +{ +private: + const SubspaceMap ↦ + SubspaceMap::const_iterator iter; +public: + SparseTensorValueAllMappings(const SubspaceMap & map_in) : map(map_in), iter(map.end()) {} + ~SparseTensorValueAllMappings(); + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; +}; + +SparseTensorValueAllMappings::~SparseTensorValueAllMappings() = default; + +void +SparseTensorValueAllMappings::lookup(const std::vector<const vespalib::stringref*> &) +{ + iter = map.begin(); +} + +bool +SparseTensorValueAllMappings::next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) +{ + if (iter != map.end()) { + const auto & ref = iter->first; + idx_out = iter->second; + ++iter; + SparseTensorAddressDecoder decoder(ref); + for (const auto ptr : addr_out) { + const auto label = decoder.decodeLabel(); + *ptr = label; + } + return true; + } + return false; +} + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +SparseTensorValueIndex::SparseTensorValueIndex(size_t num_mapped_in) + : _stash(), _map(), _num_mapped_dims(num_mapped_in) {} + +SparseTensorValueIndex::SparseTensorValueIndex(const SparseTensorValueIndex & index_in) + : _stash(), _map(), _num_mapped_dims(index_in._num_mapped_dims) +{ + copyMap(_map, index_in._map, _stash); +} + +SparseTensorValueIndex::~SparseTensorValueIndex() = default; + +size_t SparseTensorValueIndex::size() const { + return _map.size(); +} + +std::unique_ptr<View> +SparseTensorValueIndex::create_view(const std::vector<size_t> &dims) const +{ + if (dims.size() == _num_mapped_dims) { + return std::make_unique<SparseTensorValueLookup>(_map); + } + if (dims.size() == 0) { + return std::make_unique<SparseTensorValueAllMappings>(_map); + } + return std::make_unique<SparseTensorValueView>(_map, dims); +} + +void +SparseTensorValueIndex::add_subspace(SparseTensorAddressRef tmp_ref, size_t idx) +{ + SparseTensorAddressRef ref(tmp_ref, _stash); + assert(_map.find(ref) == _map.end()); + assert(_map.size() == idx); + _map[ref] = idx; +} + +//----------------------------------------------------------------------------- + +template<typename T> +SparseTensorValue<T>::SparseTensorValue(const eval::ValueType &type_in, + const SparseTensorValueIndex &index_in, + const std::vector<T> &cells_in) + : _type(type_in), + _index(index_in), + _cells(cells_in) +{ +} + +template<typename T> +SparseTensorValue<T>::SparseTensorValue(eval::ValueType &&type_in, SparseTensorValueIndex &&index_in, std::vector<T> &&cells_in) + : _type(std::move(type_in)), + _index(std::move(index_in)), + _cells(std::move(cells_in)) +{ +} + +template<typename T> SparseTensorValue<T>::~SparseTensorValue() = default; + +template class SparseTensorValue<float>; +template class SparseTensorValue<double>; + +//----------------------------------------------------------------------------- + +} // namespace + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::tensor::SparseTensorAddressRef, uint32_t); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.h new file mode 100644 index 00000000000..61e412b0191 --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value.h @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "sparse_tensor_address_ref.h" +#include <vespa/eval/eval/value.h> +#include <vespa/eval/tensor/types.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/stash.h> + +namespace vespalib::tensor { + +struct SparseTensorValueIndex : public vespalib::eval::Value::Index +{ + using View = vespalib::eval::Value::Index::View; + using SubspaceMap = hash_map<SparseTensorAddressRef, uint32_t, hash<SparseTensorAddressRef>, + std::equal_to<>, hashtable_base::and_modulator>; + + Stash _stash; + SubspaceMap _map; + size_t _num_mapped_dims; + + explicit SparseTensorValueIndex(size_t num_mapped_dims_in); + SparseTensorValueIndex(const SparseTensorValueIndex & index_in); + SparseTensorValueIndex(SparseTensorValueIndex && index_in) = default; + ~SparseTensorValueIndex(); + size_t size() const override; + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; + void add_subspace(SparseTensorAddressRef tmp_ref, size_t idx); +}; + +/** + * A tensor implementation using serialized tensor addresses to + * improve CPU cache and TLB hit ratio, relative to SimpleTensor + * implementation. + */ +template<typename T> +class SparseTensorValue : public vespalib::eval::Value +{ +private: + eval::ValueType _type; + SparseTensorValueIndex _index; + std::vector<T> _cells; +public: + SparseTensorValue(const eval::ValueType &type_in, const SparseTensorValueIndex &index_in, const std::vector<T> &cells_in); + + SparseTensorValue(eval::ValueType &&type_in, SparseTensorValueIndex &&index_in, std::vector<T> &&cells_in); + + ~SparseTensorValue() override; + + TypedCells cells() const override { return TypedCells(_cells); } + + const Index &index() const override { return _index; } + + const eval::ValueType &type() const override { return _type; } +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.cpp new file mode 100644 index 00000000000..07ba2b217ac --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.cpp @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_tensor_value_builder.h" + +namespace vespalib::tensor { + +template <typename T> +ArrayRef<T> +SparseTensorValueBuilder<T>::add_subspace(const std::vector<vespalib::stringref> &addr) +{ + uint32_t idx = _cells.size(); + _cells.resize(idx + 1); + _addr_builder.clear(); + for (const auto & label : addr) { + _addr_builder.add(label); + } + auto tmp_ref = _addr_builder.getAddressRef(); + _index.add_subspace(tmp_ref, idx); + return ArrayRef<T>(&_cells[idx], 1); +} + +template <typename T> +std::unique_ptr<eval::Value> +SparseTensorValueBuilder<T>::build(std::unique_ptr<eval::ValueBuilder<T>>) +{ + return std::make_unique<SparseTensorValue<T>>(std::move(_type), + std::move(_index), + std::move(_cells)); + +} + +template class SparseTensorValueBuilder<float>; +template class SparseTensorValueBuilder<double>; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.h new file mode 100644 index 00000000000..46d79482f3d --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_value_builder.h @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "sparse_tensor_value.h" +#include "sparse_tensor_address_builder.h" + +namespace vespalib::tensor { + +/** + * A builder for SparseTensorValue objects + * appropriate for cell type T. + **/ +template <typename T> +class SparseTensorValueBuilder : public eval::ValueBuilder<T> +{ +private: + eval::ValueType _type; + SparseTensorValueIndex _index; + std::vector<T> _cells; + SparseTensorAddressBuilder _addr_builder; +public: + SparseTensorValueBuilder(const eval::ValueType &type, + size_t num_mapped_in, + size_t expected_subspaces) + : _type(type), + _index(num_mapped_in), + _cells() + { + assert(num_mapped_in > 0); + _cells.reserve(expected_subspaces); + } + + ~SparseTensorValueBuilder() override = default; + + ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) override; + std::unique_ptr<eval::Value> build(std::unique_ptr<eval::ValueBuilder<T>> self) override; +}; + +} // namespace |