diff options
Diffstat (limited to 'eval/src')
6 files changed, 202 insertions, 392 deletions
diff --git a/eval/src/tests/tensor/direct_dense_tensor_builder/CMakeLists.txt b/eval/src/tests/tensor/direct_dense_tensor_builder/CMakeLists.txt new file mode 100644 index 00000000000..70ccbddd617 --- /dev/null +++ b/eval/src/tests/tensor/direct_dense_tensor_builder/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_direct_dense_tensor_builder_test_app TEST + SOURCES + direct_dense_tensor_builder_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_direct_dense_tensor_builder_test_app COMMAND eval_direct_dense_tensor_builder_test_app) diff --git a/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp new file mode 100644 index 00000000000..a0b7b60e2da --- /dev/null +++ b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp @@ -0,0 +1,162 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/tensor/dense/direct_dense_tensor_builder.h> +#include <vespa/vespalib/util/exceptions.h> + +using namespace vespalib::tensor; +using vespalib::IllegalArgumentException; +using Builder = DirectDenseTensorBuilder; +using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; +using vespalib::ConstArrayRef; + +template <typename T> std::vector<T> make_vector(const ConstArrayRef<T> &ref) { + std::vector<T> vec; + for (const T &t: ref) { + vec.push_back(t); + } + return vec; +} + +void assertTensor(const vespalib::string &type_spec, + const DenseTensor::Cells &expCells, + const Tensor &tensor) +{ + const DenseTensor &realTensor = dynamic_cast<const DenseTensor &>(tensor); + EXPECT_EQUAL(ValueType::from_spec(type_spec), realTensor.type()); + EXPECT_EQUAL(expCells, make_vector(realTensor.cellsRef())); +} + +void assertTensorSpec(const TensorSpec &expSpec, const Tensor &tensor) { + TensorSpec actSpec = tensor.toSpec(); + EXPECT_EQUAL(expSpec, actSpec); +} + +Tensor::UP build1DTensor() { + Builder builder(ValueType::from_spec("tensor(x[3])")); + builder.insertCell(0, 10); + builder.insertCell(1, 11); + builder.insertCell(2, 12); + return builder.build(); +} + +TEST("require that 1d tensor can be constructed") { + assertTensor("tensor(x[3])", {10,11,12}, *build1DTensor()); +} + +TEST("require that 1d tensor can be converted to tensor spec") { + assertTensorSpec(TensorSpec("tensor(x[3])"). + add({{"x", 0}}, 10). + add({{"x", 1}}, 11). + add({{"x", 2}}, 12), + *build1DTensor()); +} + +Tensor::UP build2DTensor() { + Builder builder(ValueType::from_spec("tensor(x[3],y[2])")); + builder.insertCell({0, 0}, 10); + builder.insertCell({0, 1}, 11); + builder.insertCell({1, 0}, 12); + builder.insertCell({1, 1}, 13); + builder.insertCell({2, 0}, 14); + builder.insertCell({2, 1}, 15); + return builder.build(); +} + +TEST("require that 2d tensor can be constructed") { + assertTensor("tensor(x[3],y[2])", {10,11,12,13,14,15}, *build2DTensor()); +} + +TEST("require that 2d tensor can be converted to tensor spec") { + assertTensorSpec(TensorSpec("tensor(x[3],y[2])"). + add({{"x", 0},{"y", 0}}, 10). + add({{"x", 0},{"y", 1}}, 11). + add({{"x", 1},{"y", 0}}, 12). + add({{"x", 1},{"y", 1}}, 13). + add({{"x", 2},{"y", 0}}, 14). + add({{"x", 2},{"y", 1}}, 15), + *build2DTensor()); +} + +TEST("require that 3d tensor can be constructed") { + Builder builder(ValueType::from_spec("tensor(x[3],y[2],z[2])")); + builder.insertCell({0, 0, 0}, 10); + builder.insertCell({0, 0, 1}, 11); + builder.insertCell({0, 1, 0}, 12); + builder.insertCell({0, 1, 1}, 13); + builder.insertCell({1, 0, 0}, 14); + builder.insertCell({1, 0, 1}, 15); + builder.insertCell({1, 1, 0}, 16); + builder.insertCell({1, 1, 1}, 17); + builder.insertCell({2, 0, 0}, 18); + builder.insertCell({2, 0, 1}, 19); + builder.insertCell({2, 1, 0}, 20); + builder.insertCell({2, 1, 1}, 21); + assertTensor("tensor(x[3],y[2],z[2])", + {10,11,12,13,14,15,16,17,18,19,20,21}, + *builder.build()); +} + +TEST("require that cells get default value 0 if not specified") { + Builder builder(ValueType::from_spec("tensor(x[3])")); + builder.insertCell(1, 11); + assertTensor("tensor(x[3])", {0,11,0}, + *builder.build()); +} + +void assertTensorCell(const DenseTensor::Address &expAddress, + double expCell, + const DenseTensor::CellsIterator &itr) +{ + EXPECT_TRUE(itr.valid()); + EXPECT_EQUAL(expAddress, itr.address()); + EXPECT_EQUAL(expCell, itr.cell()); +} + +TEST("require that dense tensor cells iterator works for 1d tensor") { + Tensor::UP tensor; + { + Builder builder(ValueType::from_spec("tensor(x[2])")); + builder.insertCell(0, 2); + builder.insertCell(1, 3); + tensor = builder.build(); + } + + const DenseTensor &denseTensor = dynamic_cast<const DenseTensor &>(*tensor); + DenseTensor::CellsIterator itr = denseTensor.cellsIterator(); + + assertTensorCell({0}, 2, itr); + itr.next(); + assertTensorCell({1}, 3, itr); + itr.next(); + EXPECT_FALSE(itr.valid()); +} + +TEST("require that dense tensor cells iterator works for 2d tensor") { + Tensor::UP tensor; + { + Builder builder(ValueType::from_spec("tensor(x[2],y[2])")); + builder.insertCell({0, 0}, 2); + builder.insertCell({0, 1}, 3); + builder.insertCell({1, 0}, 5); + builder.insertCell({1, 1}, 7); + tensor = builder.build(); + } + + const DenseTensor &denseTensor = dynamic_cast<const DenseTensor &>(*tensor); + DenseTensor::CellsIterator itr = denseTensor.cellsIterator(); + + assertTensorCell({0,0}, 2, itr); + itr.next(); + assertTensorCell({0,1}, 3, itr); + itr.next(); + assertTensorCell({1,0}, 5, itr); + itr.next(); + assertTensorCell({1,1}, 7, itr); + itr.next(); + EXPECT_FALSE(itr.valid()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_performance/.gitignore b/eval/src/tests/tensor/tensor_performance/.gitignore deleted file mode 100644 index c9401246324..00000000000 --- a/eval/src/tests/tensor/tensor_performance/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_performance_test_app diff --git a/eval/src/tests/tensor/tensor_performance/CMakeLists.txt b/eval/src/tests/tensor/tensor_performance/CMakeLists.txt deleted file mode 100644 index 17e58280f35..00000000000 --- a/eval/src/tests/tensor/tensor_performance/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_performance_test_app TEST - SOURCES - tensor_performance_test.cpp - DEPENDS - vespaeval -) -vespa_add_test( - NAME eval_tensor_performance_test_app - COMMAND eval_tensor_performance_test_app - ENVIRONMENT "TEST_SUBSET=SMOKETEST" -) diff --git a/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp b/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp deleted file mode 100644 index 774f8200c01..00000000000 --- a/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp +++ /dev/null @@ -1,370 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/eval/function.h> -#include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/tensor_nodes.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/eval/tensor/dense/dense_tensor_builder.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/vespalib/util/benchmark_timer.h> -#include <vespa/eval/tensor/default_tensor_engine.h> - -using namespace vespalib; -using namespace vespalib::eval; -using namespace vespalib::tensor; - -//----------------------------------------------------------------------------- - -const vespalib::string dot_product_match_expr = "reduce(query*document,sum)"; -const vespalib::string dot_product_multiply_expr = "reduce(query*document,sum)"; -const vespalib::string model_match_expr = "reduce((query*document)*model,sum)"; -const vespalib::string matrix_product_expr = "reduce(reduce((query+document)*model,sum,x),sum)"; - -//----------------------------------------------------------------------------- - -struct Params { - std::map<vespalib::string, Value::UP> map; - Params &add(const vespalib::string &name, Value::UP value) { - map.emplace(name, std::move(value)); - return *this; - } -}; - -SimpleObjectParams make_params(const Function &function, const Params ¶ms) -{ - SimpleObjectParams fun_params({}); - EXPECT_EQUAL(params.map.size(), function.num_params()); - for (size_t i = 0; i < function.num_params(); ++i) { - auto param = params.map.find(function.param_name(i)); - ASSERT_TRUE(param != params.map.end()); - fun_params.params.push_back(*param->second); - } - return fun_params; -} - -std::vector<ValueType> extract_param_types(const Function &function, const Params ¶ms) { - std::vector<ValueType> result; - EXPECT_EQUAL(params.map.size(), function.num_params()); - for (size_t i = 0; i < function.num_params(); ++i) { - auto param = params.map.find(function.param_name(i)); - ASSERT_TRUE(param != params.map.end()); - result.push_back(param->second->type()); - } - return result; -} - -double calculate_expression(const vespalib::string &expression, const Params ¶ms) { - const Function function = Function::parse(expression); - const NodeTypes types(function, extract_param_types(function, params)); - const InterpretedFunction interpreted(tensor::DefaultTensorEngine::ref(), function, types); - InterpretedFunction::Context context(interpreted); - auto fun_params = make_params(function, params); - const Value &result = interpreted.eval(context, fun_params); - EXPECT_TRUE(result.is_double()); - return result.as_double(); -} - -DoubleValue dummy_result(0.0); -const Value &dummy_ranking(InterpretedFunction::Context &, LazyParams &) { return dummy_result; } - -double benchmark_expression_us(const vespalib::string &expression, const Params ¶ms) { - const Function function = Function::parse(expression); - const NodeTypes types(function, extract_param_types(function, params)); - const InterpretedFunction interpreted(tensor::DefaultTensorEngine::ref(), function, types); - InterpretedFunction::Context context(interpreted); - auto fun_params = make_params(function, params); - auto ranking = [&](){ interpreted.eval(context, fun_params); }; - auto baseline = [&](){ dummy_ranking(context, fun_params); }; - return BenchmarkTimer::benchmark(ranking, baseline, 5.0) * 1000.0 * 1000.0; -} - -//----------------------------------------------------------------------------- - -Value::UP make_tensor(TensorSpec spec) { - return DefaultTensorEngine::ref().from_spec(spec); -} - -//----------------------------------------------------------------------------- - -TEST("SMOKETEST - require that dot product benchmark expressions produce expected results") { - Params params; - params.add("query", make_tensor(TensorSpec("tensor(x{})") - .add({{"x","0"}}, 1.0) - .add({{"x","1"}}, 2.0) - .add({{"x","2"}}, 3.0))); - params.add("document", make_tensor(TensorSpec("tensor(x{})") - .add({{"x","0"}}, 2.0) - .add({{"x","1"}}, 2.0) - .add({{"x","2"}}, 2.0))); - EXPECT_EQUAL(calculate_expression(dot_product_match_expr, params), 12.0); - EXPECT_EQUAL(calculate_expression(dot_product_multiply_expr, params), 12.0); -} - -TEST("SMOKETEST - require that model match benchmark expression produces expected result") { - Params params; - params.add("query", make_tensor(TensorSpec("tensor(x{})") - .add({{"x","0"}}, 1.0) - .add({{"x","1"}}, 2.0))); - params.add("document", make_tensor(TensorSpec("tensor(y{})") - .add({{"y","0"}}, 3.0) - .add({{"y","1"}}, 4.0))); - params.add("model", make_tensor(TensorSpec("tensor(x{},y{})") - .add({{"x","0"},{"y","0"}}, 2.0) - .add({{"x","0"},{"y","1"}}, 2.0) - .add({{"x","1"},{"y","0"}}, 2.0) - .add({{"x","1"},{"y","1"}}, 2.0))); - EXPECT_EQUAL(calculate_expression(model_match_expr, params), 42.0); -} - -TEST("SMOKETEST - require that matrix product benchmark expression produces expected result") { - Params params; - params.add("query", make_tensor(TensorSpec("tensor(x{})") - .add({{"x","0"}}, 1.0) - .add({{"x","1"}}, 0.0))); - params.add("document", make_tensor(TensorSpec("tensor(x{})") - .add({{"x","0"}}, 0.0) - .add({{"x","1"}}, 2.0))); - params.add("model", make_tensor(TensorSpec("tensor(x{},y{})") - .add({{"x","0"},{"y","0"}}, 1.0) - .add({{"x","0"},{"y","1"}}, 2.0) - .add({{"x","1"},{"y","0"}}, 3.0) - .add({{"x","1"},{"y","1"}}, 4.0))); - EXPECT_EQUAL(calculate_expression(matrix_product_expr, params), 17.0); -} - -//----------------------------------------------------------------------------- - -struct DummySparseBuilder -{ - using Dimension = SparseTensorBuilder::Dimension; - Dimension define_dimension(const vespalib::string &) { return 0; } - DummySparseBuilder &add_label(Dimension, const vespalib::string &) { return *this; } - DummySparseBuilder &add_cell(double) { return *this; } - tensor::Tensor::UP build() { return tensor::Tensor::UP(); } -}; - - -struct DummyDenseTensorBuilder -{ - using Dimension = SparseTensorBuilder::Dimension; - Dimension defineDimension(const vespalib::string &, size_t) { return 0; } - DummyDenseTensorBuilder &addLabel(Dimension, size_t) { return *this; } - DummyDenseTensorBuilder &addCell(double) { return *this; } - tensor::Tensor::UP build() { return tensor::Tensor::UP(); } -}; - -struct DimensionSpec { - vespalib::string name; - size_t count; - size_t offset; - DimensionSpec(const vespalib::string &name_in, size_t count_in, size_t offset_in = 0) - : name(name_in), count(count_in), offset(offset_in) {} -}; - -struct StringBinding { - SparseTensorBuilder::Dimension dimension; - vespalib::string label; - template <typename Builder> - StringBinding(Builder &builder, const DimensionSpec &dimension_in) - : dimension(builder.define_dimension(dimension_in.name)), - label() - { - } - void set_label(size_t id) { - label = vespalib::make_string("%zu", id); - } - template <typename Builder> - static void add_cell(Builder &builder, double value) { - builder.add_cell(value); - } - template <typename Builder> - void add_label(Builder &builder) const { - builder.add_label(dimension, label); - } -}; - -struct NumberBinding { - SparseTensorBuilder::Dimension dimension; - size_t label; - template <typename Builder> - NumberBinding(Builder &builder, const DimensionSpec &dimension_in) - : dimension(builder.defineDimension(dimension_in.name, - dimension_in.offset + - dimension_in.count)), - label() - { - } - void set_label(size_t id) { - label = id; - } - template <typename Builder> - static void add_cell(Builder &builder, double value) { - builder.addCell(value); - } - template <typename Builder> - void add_label(Builder &builder) const { - builder.addLabel(dimension, label); - } -}; - - -template <typename Builder, typename Binding> -void build_tensor(Builder &builder, const std::vector<DimensionSpec> &dimensions, - std::vector<Binding> &bindings) -{ - if (bindings.size() == dimensions.size()) { - for (const auto &bound: bindings) { - bound.add_label(builder); - } - Binding::add_cell(builder, 42); - } else { - const auto &spec = dimensions[bindings.size()]; - bindings.emplace_back(builder, spec); - for (size_t i = 0; i < spec.count; ++i) { - bindings.back().set_label(spec.offset + i); - build_tensor(builder, dimensions, bindings); - } - bindings.pop_back(); - } -} - -template <typename Builder, typename Binding> -tensor::Tensor::UP make_tensor_impl(const std::vector<DimensionSpec> &dimensions) { - Builder builder; - std::vector<Binding> bindings; - bindings.reserve(dimensions.size()); - build_tensor<Builder, Binding>(builder, dimensions, bindings); - return builder.build(); -} - -//----------------------------------------------------------------------------- - -enum class BuilderType { DUMMY, SPARSE, NUMBERDUMMY, - DENSE }; - -const BuilderType DUMMY = BuilderType::DUMMY; -const BuilderType SPARSE = BuilderType::SPARSE; -const BuilderType NUMBERDUMMY = BuilderType::NUMBERDUMMY; -const BuilderType DENSE = BuilderType::DENSE; - -const char *name(BuilderType type) { - switch (type) { - case BuilderType::DUMMY: return " dummy"; - case BuilderType::SPARSE: return "sparse"; - case BuilderType::NUMBERDUMMY: return "numberdummy"; - case BuilderType::DENSE: return "dense"; - } - abort(); -} - -tensor::Tensor::UP make_tensor(BuilderType type, const std::vector<DimensionSpec> &dimensions) { - switch (type) { - case BuilderType::DUMMY: - return make_tensor_impl<DummySparseBuilder, StringBinding> - (dimensions); - case BuilderType::SPARSE: - return make_tensor_impl<SparseTensorBuilder, StringBinding>(dimensions); - case BuilderType::NUMBERDUMMY: - return make_tensor_impl<DummyDenseTensorBuilder, NumberBinding>(dimensions); - case BuilderType::DENSE: - return make_tensor_impl<DenseTensorBuilder, NumberBinding>(dimensions); - } - abort(); -} - -//----------------------------------------------------------------------------- - -struct BuildTask { - BuilderType type; - std::vector<DimensionSpec> spec; - BuildTask(BuilderType type_in, const std::vector<DimensionSpec> &spec_in) : type(type_in), spec(spec_in) {} - void operator()() { tensor::Tensor::UP tensor = make_tensor(type, spec); } -}; - -double benchmark_build_us(BuilderType type, const std::vector<DimensionSpec> &spec) { - BuildTask build_task(type, spec); - BuildTask dummy_task((type == DENSE) ? NUMBERDUMMY : DUMMY, spec); - return BenchmarkTimer::benchmark(build_task, dummy_task, 5.0) * 1000.0 * 1000.0; -} - -TEST("benchmark create/destroy time for 1d tensors") { - for (size_t size: {5, 10, 25, 50, 100, 250, 500}) { - for (auto type: {SPARSE, DENSE}) { - double time_us = benchmark_build_us(type, {DimensionSpec("x", size)}); - fprintf(stderr, "-- 1d tensor create/destroy (%s) with size %zu: %g us\n", name(type), size, time_us); - } - } -} - -TEST("benchmark create/destroy time for 2d tensors") { - for (size_t size: {5, 10, 25, 50, 100}) { - for (auto type: {SPARSE, DENSE}) { - double time_us = benchmark_build_us(type, {DimensionSpec("x", size), DimensionSpec("y", size)}); - fprintf(stderr, "-- 2d tensor create/destroy (%s) with size %zux%zu: %g us\n", name(type), size, size, time_us); - } - } -} - -//----------------------------------------------------------------------------- - -TEST("benchmark dot product using match") { - for (size_t size: {10, 25, 50, 100, 250}) { - for (auto type: {SPARSE, DENSE}) { - Params params; - params.add("query", make_tensor(type, {DimensionSpec("x", size)})); - params.add("document", make_tensor(type, {DimensionSpec("x", size)})); - double time_us = benchmark_expression_us(dot_product_match_expr, params); - fprintf(stderr, "-- dot product (%s) using match %zu vs %zu: %g us\n", name(type), size, size, time_us); - } - } -} - -TEST("benchmark dot product using multiply") { - for (size_t size: {10, 25, 50, 100, 250}) { - for (auto type: {SPARSE, DENSE}) { - Params params; - params.add("query", make_tensor(type, {DimensionSpec("x", size)})); - params.add("document", make_tensor(type, {DimensionSpec("x", size)})); - double time_us = benchmark_expression_us(dot_product_multiply_expr, params); - fprintf(stderr, "-- dot product (%s) using multiply %zu vs %zu: %g us\n", name(type), size, size, time_us); - } - } -} - -TEST("benchmark model match") { - for (size_t model_size: {25, 50, 100}) { - for (size_t vector_size: {5, 10, 25, 50, 100}) { - if (vector_size <= model_size) { - for (auto type: {SPARSE}) { - Params params; - params.add("query", make_tensor(type, {DimensionSpec("x", vector_size)})); - params.add("document", make_tensor(type, {DimensionSpec("y", vector_size)})); - params.add("model", make_tensor(type, {DimensionSpec("x", model_size), DimensionSpec("y", model_size)})); - double time_us = benchmark_expression_us(model_match_expr, params); - fprintf(stderr, "-- model match (%s) %zu * %zu vs %zux%zu: %g us\n", name(type), vector_size, vector_size, model_size, model_size, time_us); - } - } - } - } -} - -TEST("benchmark matrix product") { - for (size_t vector_size: {5, 10, 25, 50}) { - size_t matrix_size = vector_size * 2; - for (auto type: {SPARSE, DENSE}) { - Params params; - params.add("query", make_tensor(type, {DimensionSpec("x", matrix_size)})); - params.add("document", make_tensor(type, {DimensionSpec("x", matrix_size)})); - params.add("model", make_tensor(type, {DimensionSpec("x", matrix_size), DimensionSpec("y", matrix_size)})); - double time_us = benchmark_expression_us(matrix_product_expr, params); - fprintf(stderr, "-- matrix product (%s) %zu + %zu vs %zux%zu: %g us\n", name(type), vector_size, vector_size, matrix_size, matrix_size, time_us); - } - } -} - -//----------------------------------------------------------------------------- - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index b39fd5e02fb..b4b65f06d96 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -6,7 +6,7 @@ #include "serialization/typed_binary_format.h" #include "sparse/sparse_tensor_builder.h" #include "dense/dense_tensor.h" -#include "dense/dense_tensor_builder.h" +#include "dense/direct_dense_tensor_builder.h" #include "dense/dense_dot_product_function.h" #include "dense/dense_xw_product_function.h" #include "dense/dense_fast_rename_optimizer.h" @@ -21,6 +21,7 @@ #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/operation.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/exceptions.h> #include <cassert> #include <vespa/log/log.h> @@ -36,12 +37,16 @@ using eval::TensorFunction; using eval::TensorSpec; using eval::Value; using eval::ValueType; +using vespalib::IllegalArgumentException; +using vespalib::make_string; using map_fun_t = eval::TensorEngine::map_fun_t; using join_fun_t = eval::TensorEngine::join_fun_t; namespace { +constexpr size_t UNDEFINED_IDX = std::numeric_limits<size_t>::max(); + const eval::TensorEngine &simple_engine() { return eval::SimpleTensorEngine::ref(); } const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); } @@ -103,6 +108,27 @@ const Value &fallback_reduce(const Value &a, eval::Aggr aggr, const std::vector< return to_default(simple_engine().reduce(to_simple(a, stash), aggr, dimensions, stash), stash); } +size_t calculate_cell_index(const ValueType &type, const TensorSpec::Address &address) { + if (type.dimensions().size() != address.size()) { + return UNDEFINED_IDX; + } + size_t d = 0; + size_t idx = 0; + for (const auto &binding: address) { + const auto &dim = type.dimensions()[d++]; + if ((dim.name != binding.first) || (binding.second.index >= dim.size)) { + return UNDEFINED_IDX; + } + idx *= dim.size; + idx += binding.second.index; + } + return idx; +} + +void bad_spec(const TensorSpec &spec) { + throw IllegalArgumentException(make_string("malformed tensor spec: %s", spec.to_string().c_str())); +} + } // namespace vespalib::tensor::<unnamed> const DefaultTensorEngine DefaultTensorEngine::_engine; @@ -128,17 +154,14 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const if (!tensor::Tensor::supported({type})) { return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::create(spec)); } else if (type.is_dense()) { - DenseTensorBuilder builder; - std::map<vespalib::string,DenseTensorBuilder::Dimension> dimension_map; - for (const auto &dimension: type.dimensions()) { - dimension_map[dimension.name] = builder.defineDimension(dimension.name, dimension.size); - } + DirectDenseTensorBuilder builder(type); for (const auto &cell: spec.cells()) { const auto &address = cell.first; - for (const auto &binding: address) { - builder.addLabel(dimension_map[binding.first], binding.second.index); + size_t cell_idx = calculate_cell_index(type, address); + if (cell_idx == UNDEFINED_IDX) { + bad_spec(spec); } - builder.addCell(cell.second); + builder.insertCell(cell_idx, cell.second); } return builder.build(); } else if (type.is_sparse()) { |