diff options
author | Håvard Pettersen <havardpe@oath.com> | 2020-12-02 16:32:16 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2020-12-02 16:32:16 +0000 |
commit | 389ccc63950b710993ea1c82d20050af8094b75f (patch) | |
tree | 15dba6eb8949e24f13310b7784ae841269adc14f | |
parent | 9d59f4b003b10ae5a5a7f02874fb1b4b7e0ccccc (diff) |
remove simple tensor
34 files changed, 13 insertions, 1732 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index bbf20dc7e8a..4ef530518df 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -30,7 +30,6 @@ vespa_define_module( src/tests/eval/param_usage src/tests/eval/reference_evaluation src/tests/eval/reference_operations - src/tests/eval/simple_tensor src/tests/eval/simple_value src/tests/eval/tensor_function src/tests/eval/tensor_lambda diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index c3a20e1ce72..f5e5c5d0dfd 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -3,7 +3,6 @@ #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/simple_tensor_engine.h> using namespace vespalib::eval; diff --git a/eval/src/tests/eval/simple_tensor/.gitignore b/eval/src/tests/eval/simple_tensor/.gitignore deleted file mode 100644 index f371f5c6c6d..00000000000 --- a/eval/src/tests/eval/simple_tensor/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_simple_tensor_test_app diff --git a/eval/src/tests/eval/simple_tensor/CMakeLists.txt b/eval/src/tests/eval/simple_tensor/CMakeLists.txt deleted file mode 100644 index 08fb1c17572..00000000000 --- a/eval/src/tests/eval/simple_tensor/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_simple_tensor_test_app TEST - SOURCES - simple_tensor_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_simple_tensor_test_app COMMAND eval_simple_tensor_test_app) diff --git a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp deleted file mode 100644 index f3bea3820c8..00000000000 --- a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp +++ /dev/null @@ -1,370 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> -#include <vespa/eval/eval/operation.h> -#include <vespa/vespalib/util/stash.h> -#include <vespa/vespalib/data/memory.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <iostream> - -using namespace vespalib::eval; - -using Cell = SimpleTensor::Cell; -using Cells = SimpleTensor::Cells; -using Address = SimpleTensor::Address; -using Stash = vespalib::Stash; -using vespalib::nbostream; -using vespalib::Memory; - -TensorSpec to_spec(const Value &a) { return SimpleTensorEngine::ref().to_spec(a); } - -const Tensor &unwrap(const Value &value) { - ASSERT_TRUE(value.is_tensor()); - return *value.as_tensor(); -} - -struct CellBuilder { - Cells cells; - CellBuilder &add(const Address &addr, double value) { - cells.emplace_back(addr, value); - return *this; - } - Cells build() { return cells; } -}; - -TEST("require that simple tensors can be built using tensor spec") { - TensorSpec spec("tensor(w{},x[2],y{},z[2])"); - spec.add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) - .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - Value::UP tensor = SimpleTensorEngine::ref().from_spec(spec); - TensorSpec full_spec("tensor(w{},x[2],y{},z[2])"); - full_spec - .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) - .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) - .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) - .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) - .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 0.0) - .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) - .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) - .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 0.0) - .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 0.0) - .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) - .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) - .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 0.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) - .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - Value::UP full_tensor = SimpleTensorEngine::ref().from_spec(full_spec); - EXPECT_EQUAL(full_spec, to_spec(*tensor)); - EXPECT_EQUAL(full_spec, to_spec(*full_tensor)); -}; - -TEST("require that simple tensors can have their values negated") { - auto tensor = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, 1) - .add({{"x","2"},{"y","1"}}, -3) - .add({{"x","1"},{"y","2"}}, 5)); - auto expect = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, -1) - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, -5)); - auto result = tensor->map([](double a){ return -a; }); - EXPECT_EQUAL(to_spec(*expect), to_spec(*result)); - Stash stash; - const Value &result2 = SimpleTensorEngine::ref().map(*tensor, operation::Neg::f, stash); - EXPECT_EQUAL(to_spec(*expect), to_spec(unwrap(result2))); -} - -TEST("require that simple tensors can be multiplied with each other") { - auto lhs = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, 1) - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, 5)); - auto rhs = SimpleTensor::create( - TensorSpec("tensor(y{},z{})") - .add({{"y","1"},{"z","1"}}, 7) - .add({{"y","2"},{"z","1"}}, 11) - .add({{"y","1"},{"z","2"}}, 13)); - auto expect = SimpleTensor::create( - TensorSpec("tensor(x{},y{},z{})") - .add({{"x","1"},{"y","1"},{"z","1"}}, 7) - .add({{"x","1"},{"y","1"},{"z","2"}}, 13) - .add({{"x","2"},{"y","1"},{"z","1"}}, 21) - .add({{"x","2"},{"y","1"},{"z","2"}}, 39) - .add({{"x","1"},{"y","2"},{"z","1"}}, 55)); - auto result = SimpleTensor::join(*lhs, *rhs, [](double a, double b){ return (a * b); }); - EXPECT_EQUAL(to_spec(*expect), to_spec(*result)); - Stash stash; - const Value &result2 = SimpleTensorEngine::ref().join(*lhs, *rhs, operation::Mul::f, stash); - EXPECT_EQUAL(to_spec(*expect), to_spec(unwrap(result2))); -} - -TEST("require that simple tensors can be merged") { - auto lhs = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, 1) - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, 5)); - auto rhs = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","2"}}, 7) - .add({{"x","2"},{"y","2"}}, 11) - .add({{"x","1"},{"y","1"}}, 13)); - auto expect = SimpleTensor::create( - TensorSpec("tensor(x{},y{})") - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, 7) - .add({{"x","2"},{"y","2"}}, 11) - .add({{"x","1"},{"y","1"}}, 13)); - auto result = SimpleTensor::merge(*lhs, *rhs, [](double, double b){ return b; }); - EXPECT_EQUAL(to_spec(*expect), to_spec(*result)); - Stash stash; - const Value &result2 = SimpleTensorEngine::ref().merge(*lhs, *rhs, [](double, double b){ return b; }, stash); - EXPECT_EQUAL(to_spec(*expect), to_spec(unwrap(result2))); -} - -TEST("require that simple tensors support dimension reduction") { - auto tensor = SimpleTensor::create( - TensorSpec("tensor(x[3],y[2])") - .add({{"x",0},{"y",0}}, 1) - .add({{"x",1},{"y",0}}, 2) - .add({{"x",2},{"y",0}}, 3) - .add({{"x",0},{"y",1}}, 4) - .add({{"x",1},{"y",1}}, 5) - .add({{"x",2},{"y",1}}, 6)); - auto expect_sum_y = SimpleTensor::create( - TensorSpec("tensor(x[3])") - .add({{"x",0}}, 5) - .add({{"x",1}}, 7) - .add({{"x",2}}, 9)); - auto expect_sum_x = SimpleTensor::create( - TensorSpec("tensor(y[2])") - .add({{"y",0}}, 6) - .add({{"y",1}}, 15)); - auto expect_sum_all = SimpleTensor::create(TensorSpec("double").add({}, 21)); - Stash stash; - Aggregator &aggr_sum = Aggregator::create(Aggr::SUM, stash); - auto result_sum_y = tensor->reduce(aggr_sum, {"y"}); - auto result_sum_x = tensor->reduce(aggr_sum, {"x"}); - auto result_sum_all = tensor->reduce(aggr_sum, {"x", "y"}); - EXPECT_EQUAL(to_spec(*expect_sum_y), to_spec(*result_sum_y)); - EXPECT_EQUAL(to_spec(*expect_sum_x), to_spec(*result_sum_x)); - EXPECT_EQUAL(to_spec(*expect_sum_all), to_spec(*result_sum_all)); - const Value &result_sum_y_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"y"}, stash); - const Value &result_sum_x_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"x"}, stash); - const Value &result_sum_all_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"x", "y"}, stash); - const Value &result_sum_all_3 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {}, stash); - EXPECT_EQUAL(to_spec(*expect_sum_y), to_spec(unwrap(result_sum_y_2))); - EXPECT_EQUAL(to_spec(*expect_sum_x), to_spec(unwrap(result_sum_x_2))); - EXPECT_TRUE(result_sum_all_2.is_double()); - EXPECT_TRUE(result_sum_all_3.is_double()); - EXPECT_EQUAL(21, result_sum_all_2.as_double()); - EXPECT_EQUAL(21, result_sum_all_3.as_double()); - EXPECT_EQUAL(to_spec(*result_sum_y), to_spec(*result_sum_y)); - EXPECT_NOT_EQUAL(to_spec(*result_sum_y), to_spec(*result_sum_x)); -} - -//----------------------------------------------------------------------------- - -vespalib::string make_type_spec(bool use_float, const vespalib::string &dims) { - vespalib::string type_spec = "tensor"; - if (use_float) { - type_spec.append("<float>"); - } - type_spec.append(dims); - return type_spec; -} - -struct TensorExample { - virtual ~TensorExample(); - virtual TensorSpec make_spec(bool use_float) const = 0; - virtual std::unique_ptr<SimpleTensor> make_tensor(bool use_float) const = 0; - virtual void encode_default(nbostream &dst) const = 0; - virtual void encode_with_double(nbostream &dst) const = 0; - virtual void encode_with_float(nbostream &dst) const = 0; - void verify_encode_decode() const { - nbostream expect_default; - nbostream expect_double; - nbostream expect_float; - encode_default(expect_default); - encode_with_double(expect_double); - encode_with_float(expect_float); - nbostream data_double; - nbostream data_float; - SimpleTensor::encode(*make_tensor(false), data_double); - SimpleTensor::encode(*make_tensor(true), data_float); - EXPECT_EQUAL(Memory(data_double.peek(), data_double.size()), - Memory(expect_default.peek(), expect_default.size())); - EXPECT_EQUAL(Memory(data_float.peek(), data_float.size()), - Memory(expect_float.peek(), expect_float.size())); - EXPECT_EQUAL(to_spec(*SimpleTensor::decode(expect_default)), make_spec(false)); - EXPECT_EQUAL(to_spec(*SimpleTensor::decode(expect_double)), make_spec(false)); - EXPECT_EQUAL(to_spec(*SimpleTensor::decode(expect_float)), make_spec(true)); - } -}; -TensorExample::~TensorExample() = default; - -//----------------------------------------------------------------------------- - -struct SparseTensorExample : TensorExample { - TensorSpec make_spec(bool use_float) const override { - return TensorSpec(make_type_spec(use_float, "(x{},y{})")) - .add({{"x","a"},{"y","a"}}, 1) - .add({{"x","a"},{"y","b"}}, 2) - .add({{"x","b"},{"y","a"}}, 3); - } - std::unique_ptr<SimpleTensor> make_tensor(bool use_float) const override { - return SimpleTensor::create(make_spec(use_float)); - } - template <typename T> - void encode_inner(nbostream &dst) const { - dst.putInt1_4Bytes(2); - dst.writeSmallString("x"); - dst.writeSmallString("y"); - dst.putInt1_4Bytes(3); - dst.writeSmallString("a"); - dst.writeSmallString("a"); - dst << (T) 1; - dst.writeSmallString("a"); - dst.writeSmallString("b"); - dst << (T) 2; - dst.writeSmallString("b"); - dst.writeSmallString("a"); - dst << (T) 3; - } - void encode_default(nbostream &dst) const override { - dst.putInt1_4Bytes(1); - encode_inner<double>(dst); - } - void encode_with_double(nbostream &dst) const override { - dst.putInt1_4Bytes(5); - dst.putInt1_4Bytes(0); - encode_inner<double>(dst); - } - void encode_with_float(nbostream &dst) const override { - dst.putInt1_4Bytes(5); - dst.putInt1_4Bytes(1); - encode_inner<float>(dst); - } -}; - -TEST_F("require that sparse tensors can be encoded and decoded", SparseTensorExample()) { - TEST_DO(f1.verify_encode_decode()); -} - -//----------------------------------------------------------------------------- - -struct DenseTensorExample : TensorExample { - TensorSpec make_spec(bool use_float) const override { - return TensorSpec(make_type_spec(use_float, "(x[3],y[2])")) - .add({{"x",0},{"y",0}}, 1) - .add({{"x",0},{"y",1}}, 2) - .add({{"x",1},{"y",0}}, 3) - .add({{"x",1},{"y",1}}, 4) - .add({{"x",2},{"y",0}}, 5) - .add({{"x",2},{"y",1}}, 6); - } - std::unique_ptr<SimpleTensor> make_tensor(bool use_float) const override { - return SimpleTensor::create(make_spec(use_float)); - } - template <typename T> - void encode_inner(nbostream &dst) const { - dst.putInt1_4Bytes(2); - dst.writeSmallString("x"); - dst.putInt1_4Bytes(3); - dst.writeSmallString("y"); - dst.putInt1_4Bytes(2); - dst << (T) 1; - dst << (T) 2; - dst << (T) 3; - dst << (T) 4; - dst << (T) 5; - dst << (T) 6; - } - void encode_default(nbostream &dst) const override { - dst.putInt1_4Bytes(2); - encode_inner<double>(dst); - } - void encode_with_double(nbostream &dst) const override { - dst.putInt1_4Bytes(6); - dst.putInt1_4Bytes(0); - encode_inner<double>(dst); - } - void encode_with_float(nbostream &dst) const override { - dst.putInt1_4Bytes(6); - dst.putInt1_4Bytes(1); - encode_inner<float>(dst); - } -}; - -TEST_F("require that dense tensors can be encoded and decoded", DenseTensorExample()) { - TEST_DO(f1.verify_encode_decode()); -} - -//----------------------------------------------------------------------------- - -struct MixedTensorExample : TensorExample { - TensorSpec make_spec(bool use_float) const override { - return TensorSpec(make_type_spec(use_float, "(x{},y{},z[2])")) - .add({{"x","a"},{"y","a"},{"z",0}}, 1) - .add({{"x","a"},{"y","a"},{"z",1}}, 2) - .add({{"x","a"},{"y","b"},{"z",0}}, 3) - .add({{"x","a"},{"y","b"},{"z",1}}, 4) - .add({{"x","b"},{"y","a"},{"z",0}}, 5) - .add({{"x","b"},{"y","a"},{"z",1}}, 6); - } - std::unique_ptr<SimpleTensor> make_tensor(bool use_float) const override { - return SimpleTensor::create(make_spec(use_float)); - } - template <typename T> - void encode_inner(nbostream &dst) const { - dst.putInt1_4Bytes(2); - dst.writeSmallString("x"); - dst.writeSmallString("y"); - dst.putInt1_4Bytes(1); - dst.writeSmallString("z"); - dst.putInt1_4Bytes(2); - dst.putInt1_4Bytes(3); - dst.writeSmallString("a"); - dst.writeSmallString("a"); - dst << (T) 1; - dst << (T) 2; - dst.writeSmallString("a"); - dst.writeSmallString("b"); - dst << (T) 3; - dst << (T) 4; - dst.writeSmallString("b"); - dst.writeSmallString("a"); - dst << (T) 5; - dst << (T) 6; - } - void encode_default(nbostream &dst) const override { - dst.putInt1_4Bytes(3); - encode_inner<double>(dst); - } - void encode_with_double(nbostream &dst) const override { - dst.putInt1_4Bytes(7); - dst.putInt1_4Bytes(0); - encode_inner<double>(dst); - } - void encode_with_float(nbostream &dst) const override { - dst.putInt1_4Bytes(7); - dst.putInt1_4Bytes(1); - encode_inner<float>(dst); - } -}; - -TEST_F("require that mixed tensors can be encoded and decoded", MixedTensorExample()) { - TEST_DO(f1.verify_encode_decode()); -} - -//----------------------------------------------------------------------------- - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp index fc6a7e46576..2bffc376dbf 100644 --- a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp +++ b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/tensor/default_tensor_engine.h> diff --git a/eval/src/tests/instruction/dense_simple_expand_function/dense_simple_expand_function_test.cpp b/eval/src/tests/instruction/dense_simple_expand_function/dense_simple_expand_function_test.cpp index 975088f2bff..2c5566e045a 100644 --- a/eval/src/tests/instruction/dense_simple_expand_function/dense_simple_expand_function_test.cpp +++ b/eval/src/tests/instruction/dense_simple_expand_function/dense_simple_expand_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/instruction/dense_simple_expand_function.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/tests/tensor/dense_add_dimension_optimizer/dense_add_dimension_optimizer_test.cpp b/eval/src/tests/tensor/dense_add_dimension_optimizer/dense_add_dimension_optimizer_test.cpp index 1045e4159bf..4b2ca3ae4ae 100644 --- a/eval/src/tests/tensor/dense_add_dimension_optimizer/dense_add_dimension_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_add_dimension_optimizer/dense_add_dimension_optimizer_test.cpp @@ -3,8 +3,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_replace_type_function.h> #include <vespa/eval/tensor/dense/dense_fast_rename_optimizer.h> #include <vespa/eval/tensor/dense/dense_tensor.h> diff --git a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp index 8c67dc50df3..681b1987a6d 100644 --- a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_replace_type_function.h> #include <vespa/eval/tensor/dense/dense_fast_rename_optimizer.h> #include <vespa/eval/tensor/dense/dense_tensor.h> diff --git a/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp b/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp index 90ed28caa3c..faf6bed2786 100644 --- a/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp +++ b/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/typed_dense_tensor_builder.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp index 27bfa1ff8f3..0f3a200bfdd 100644 --- a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/eval/eval/test/eval_fixture.h> diff --git a/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp b/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp index 0255c33b295..2e428bb501e 100644 --- a/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_number_join_function.h> #include <vespa/eval/eval/test/eval_fixture.h> diff --git a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp index 1db815df8a5..b655c52c9d3 100644 --- a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_replace_type_function.h> #include <vespa/eval/tensor/dense/dense_fast_rename_optimizer.h> #include <vespa/eval/tensor/dense/dense_tensor.h> diff --git a/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp index 45a60b2d30a..6ad60d2e3f5 100644 --- a/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_simple_join_function.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp b/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp index e03a8dced6a..53164ad59c8 100644 --- a/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp +++ b/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp @@ -1,8 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_simple_map_function.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp b/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp index a5659b1c1f4..1948d8d010a 100644 --- a/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp +++ b/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp @@ -3,8 +3,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/operation.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_single_reduce_function.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> diff --git a/eval/src/tests/tensor/dense_tensor_create_function/dense_tensor_create_function_test.cpp b/eval/src/tests/tensor/dense_tensor_create_function/dense_tensor_create_function_test.cpp index 9a4e55745d2..2bc937a26bf 100644 --- a/eval/src/tests/tensor/dense_tensor_create_function/dense_tensor_create_function_test.cpp +++ b/eval/src/tests/tensor/dense_tensor_create_function/dense_tensor_create_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_tensor_create_function.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index f11678d3ee7..2f9d531309b 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -29,7 +29,6 @@ #include <vespa/eval/instruction/generic_rename.h> #include <vespa/eval/instruction/generic_map.h> #include <vespa/eval/instruction/generic_merge.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/operation.h> diff --git a/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp b/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp index bb727f8bb6a..9881d120a74 100644 --- a/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp +++ b/eval/src/tests/tensor/vector_from_doubles_function/vector_from_doubles_function_test.cpp @@ -2,8 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/dense/vector_from_doubles_function.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 5cf7440237b..66c807565cc 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -25,8 +25,6 @@ vespa_add_library(eval_eval OBJECT operator_nodes.cpp optimize_tensor_function.cpp param_usage.cpp - simple_tensor.cpp - simple_tensor_engine.cpp simple_value.cpp string_stuff.cpp tensor.cpp diff --git a/eval/src/vespa/eval/eval/basic_nodes.cpp b/eval/src/vespa/eval/eval/basic_nodes.cpp index b869f2abe22..d7fa76bf1cc 100644 --- a/eval/src/vespa/eval/eval/basic_nodes.cpp +++ b/eval/src/vespa/eval/eval/basic_nodes.cpp @@ -4,7 +4,7 @@ #include "node_traverser.h" #include "node_visitor.h" #include "interpreted_function.h" -#include "simple_tensor_engine.h" +#include "simple_value.h" namespace vespalib::eval::nodes { @@ -23,7 +23,8 @@ struct Frame { double Node::get_const_value() const { assert(is_const()); - InterpretedFunction function(SimpleTensorEngine::ref(), *this, NodeTypes()); + NodeTypes node_types(*this); + InterpretedFunction function(SimpleValueBuilderFactory::get(), *this, node_types); NoParams no_params; InterpretedFunction::Context ctx(function); return function.eval(ctx, no_params).as_double(); diff --git a/eval/src/vespa/eval/eval/engine_or_factory.cpp b/eval/src/vespa/eval/eval/engine_or_factory.cpp index 36251820c23..154d2a76d97 100644 --- a/eval/src/vespa/eval/eval/engine_or_factory.cpp +++ b/eval/src/vespa/eval/eval/engine_or_factory.cpp @@ -4,7 +4,6 @@ #include "fast_value.h" #include "simple_value.h" #include "value_codec.h" -#include "simple_tensor_engine.h" #include <vespa/eval/instruction/generic_concat.h> #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/instruction/generic_map.h> @@ -160,9 +159,6 @@ EngineOrFactory::to_string() const if (&engine() == &tensor::DefaultTensorEngine::ref()) { return "DefaultTensorEngine"; } - if (&engine() == &SimpleTensorEngine::ref()) { - return "SimpleTensorEngine"; - } } if (is_factory()) { if (&factory() == &FastValueBuilderFactory::get()) { diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 1016b929574..37817843ce2 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -8,7 +8,6 @@ #include "make_tensor_function.h" #include "optimize_tensor_function.h" #include "compile_tensor_function.h" -#include "simple_tensor_engine.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp index 9569518417d..7d76f2064a3 100644 --- a/eval/src/vespa/eval/eval/node_types.cpp +++ b/eval/src/vespa/eval/eval/node_types.cpp @@ -332,6 +332,15 @@ NodeTypes::NodeTypes() { } +NodeTypes::NodeTypes(const nodes::Node &const_node) + : _not_found(ValueType::error_type()), + _type_map() +{ + std::vector<ValueType> no_input_types; + nodes::TypeResolver resolver(no_input_types, _type_map, _errors); + const_node.traverse(resolver); +} + NodeTypes::NodeTypes(const Function &function, const std::vector<ValueType> &input_types) : _not_found(ValueType::error_type()), _type_map() diff --git a/eval/src/vespa/eval/eval/node_types.h b/eval/src/vespa/eval/eval/node_types.h index 72332564409..de867f48847 100644 --- a/eval/src/vespa/eval/eval/node_types.h +++ b/eval/src/vespa/eval/eval/node_types.h @@ -28,6 +28,7 @@ public: NodeTypes(); NodeTypes(NodeTypes &&rhs) = default; NodeTypes &operator=(NodeTypes &&rhs) = default; + NodeTypes(const nodes::Node &const_node); NodeTypes(const Function &function, const std::vector<ValueType> &input_types); ~NodeTypes(); const std::vector<vespalib::string> &errors() const { return _errors; } diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp deleted file mode 100644 index 98e3bc325cb..00000000000 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ /dev/null @@ -1,788 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_tensor.h" -#include "simple_tensor_engine.h" -#include "operation.h" -#include <vespa/vespalib/util/overload.h> -#include <vespa/vespalib/util/visit_ranges.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <algorithm> -#include <cassert> - -namespace vespalib { -namespace eval { - -using Address = SimpleTensor::Address; -using Cell = SimpleTensor::Cell; -using Cells = SimpleTensor::Cells; -using IndexList = std::vector<size_t>; -using Label = SimpleTensor::Label; -using CellRef = std::reference_wrapper<const Cell>; - -namespace { - -constexpr uint32_t DOUBLE_CELL_TYPE = 0; -constexpr uint32_t FLOAT_CELL_TYPE = 1; - -uint32_t cell_type_to_id(CellType cell_type) { - switch (cell_type) { - case CellType::DOUBLE: return DOUBLE_CELL_TYPE; - case CellType::FLOAT: return FLOAT_CELL_TYPE; - } - abort(); -} - -CellType id_to_cell_type(uint32_t id) { - switch (id) { - case DOUBLE_CELL_TYPE: return CellType::DOUBLE; - case FLOAT_CELL_TYPE: return CellType::FLOAT; - } - abort(); -} - -void assert_type(const ValueType &type) { - (void) type; - assert(type.is_double() || type.is_tensor()); -} - -void assert_address(const Address &address, const ValueType &type) { - assert(address.size() == type.dimensions().size()); - for (size_t i = 0; i < address.size(); ++i) { - if (type.dimensions()[i].is_mapped()) { - assert(address[i].is_mapped()); - } else { - assert(address[i].is_indexed()); - assert(address[i].index < type.dimensions()[i].size); - } - } -} - -Address select(const Address &address, const IndexList &selector) { - Address result; - for (size_t index: selector) { - result.push_back(address[index]); - } - return result; -} - -Address select(const Address &a, const Address &b, const IndexList &selector) { - Address result; - for (size_t index: selector) { - if (index < a.size()) { - result.push_back(a[index]); - } else { - result.push_back(b[index - a.size()]); - } - } - return result; -} - -size_t get_dimension_size(const ValueType &type, ValueType::Dimension::size_type dim_idx) { - if (dim_idx == ValueType::Dimension::npos) { - return 1; - } - return type.dimensions()[dim_idx].size; -} - -size_t get_dimension_index(const Address &addr, ValueType::Dimension::size_type dim_idx) { - if (dim_idx == ValueType::Dimension::npos) { - return 0; - } - return addr[dim_idx].index; -} - -const vespalib::string &reverse_rename(const vespalib::string &name, - const std::vector<vespalib::string> &from, - const std::vector<vespalib::string> &to) -{ - assert(from.size() == to.size()); - for (size_t idx = 0; idx < to.size(); ++idx) { - if (to[idx] == name) { - return from[idx]; - } - } - return name; -} - -/** - * Meta information about how a type can be decomposed into mapped and - * indexed dimensions and also how large each block is. A block is a - * dense-subspace consisting of all indexed dimensions that is - * uniquely specified by the labels of all mapped dimensions. - **/ -struct TypeMeta { - IndexList mapped; - IndexList indexed; - size_t block_size; - CellType cell_type; - explicit TypeMeta(const ValueType &type) - : mapped(), - indexed(), - block_size(1), - cell_type(type.cell_type()) - { - for (size_t i = 0; i < type.dimensions().size(); ++i) { - const auto &dimension = type.dimensions()[i]; - if (dimension.is_mapped()) { - mapped.push_back(i); - } else { - block_size *= dimension.size; - indexed.push_back(i); - } - } - } - ~TypeMeta() {} -}; - -/** - * Helper class used when building SimpleTensors. While a tensor - * in its final form simply contains a collection of cells, the - * builder keeps track of cell values as a block map instead. Each - * block is a dense multi-dimensional array that is addressed by - * the combination of all mapped Labels in a cell address. The - * indexed labels from the same cell address is used to address - * the appropriate cell value within the block. The reason for - * this is to make it easier to make sure that the indexed - * dimensions have entries for all valid Lables (densify with 0.0 - * as default value). - **/ -class Builder { -private: - using Block = std::vector<double>; - using BlockMap = std::map<Address,Block>; - - ValueType _type; - TypeMeta _meta; - BlockMap _blocks; - - size_t offset_of(const Address &address) const { - size_t offset = 0; - for (size_t index: _meta.indexed) { - size_t label = address[index].index; - size_t size = _type.dimensions()[index].size; - offset = (offset * size) + label; - } - return offset; - } - - void convert(const Block &block, Address &address, size_t n, Cells &cells_out) const { - if (n < _meta.indexed.size()) { - Label &label = address[_meta.indexed[n]]; - size_t size = _type.dimensions()[_meta.indexed[n]].size; - for (label.index = 0; label.index < size; ++label.index) { - convert(block, address, n + 1, cells_out); - } - } else { - cells_out.emplace_back(address, block[offset_of(address)]); - } - } - -public: - explicit Builder(const ValueType &type) - : _type(type), - _meta(type), - _blocks() - { - assert_type(_type); - if (_meta.mapped.empty()) { - _blocks.emplace(Address(), Block(_meta.block_size, 0.0)); - } - } - ~Builder() {} - void set(const Address &address, double value) { - assert_address(address, _type); - Address block_key = select(address, _meta.mapped); - auto pos = _blocks.find(block_key); - if (pos == _blocks.end()) { - pos = _blocks.emplace(block_key, Block(_meta.block_size, 0.0)).first; - } - pos->second[offset_of(address)] = value; - } - void set(const TensorSpec::Address &label_map, double value) { - Address address; - for (const auto &dimension: _type.dimensions()) { - auto pos = label_map.find(dimension.name); - assert(pos != label_map.end()); - address.emplace_back(pos->second); - } - set(address, value); - } - std::unique_ptr<SimpleTensor> build() { - Cells cells; - Address address(_type.dimensions().size(), Label(size_t(0))); - for (const auto &entry: _blocks) { - for (size_t i = 0; i < _meta.mapped.size(); ++i) { - address[_meta.mapped[i]] = entry.first[i]; - } - convert(entry.second, address, 0, cells); - } - return std::make_unique<SimpleTensor>(_type, std::move(cells)); - } -}; - -/** - * Helper class used to calculate which dimensions are shared between - * types and which are not. Also calculates how address elements from - * cells with the different types should be combined into a single - * address. - **/ -struct TypeAnalyzer { - static constexpr size_t npos = -1; - IndexList only_a; - IndexList overlap_a; - IndexList overlap_b; - IndexList only_b; - IndexList combine; - size_t ignored_a; - size_t ignored_b; - TypeAnalyzer(const ValueType &lhs, const ValueType &rhs, const vespalib::string &ignore = "") - : only_a(), overlap_a(), overlap_b(), only_b(), combine(), ignored_a(npos), ignored_b(npos) - { - const auto &a = lhs.dimensions(); - const auto &b = rhs.dimensions(); - size_t b_idx = 0; - for (size_t a_idx = 0; a_idx < a.size(); ++a_idx) { - while ((b_idx < b.size()) && (b[b_idx].name < a[a_idx].name)) { - if (b[b_idx].name != ignore) { - only_b.push_back(b_idx); - combine.push_back(a.size() + b_idx); - } else { - ignored_b = b_idx; - } - ++b_idx; - } - if ((b_idx < b.size()) && (b[b_idx].name == a[a_idx].name)) { - if (a[a_idx].name != ignore) { - overlap_a.push_back(a_idx); - overlap_b.push_back(b_idx); - combine.push_back(a_idx); - } else { - ignored_a = a_idx; - ignored_b = b_idx; - } - ++b_idx; - } else { - if (a[a_idx].name != ignore) { - only_a.push_back(a_idx); - combine.push_back(a_idx); - } else { - ignored_a = a_idx; - } - } - } - while (b_idx < b.size()) { - if (b[b_idx].name != ignore) { - only_b.push_back(b_idx); - combine.push_back(a.size() + b_idx); - } else { - ignored_b = b_idx; - } - ++b_idx; - } - } - ~TypeAnalyzer() {} -}; - -/** - * A view is a total ordering of cells from a SimpleTensor according - * to a subset of the dimensions in the tensor type. - **/ -class View { -public: - /** - * A range of cells within a view with equal values for all labels - * corresponding to the dimensions of the view. - **/ - class EqualRange { - private: - const CellRef *_begin; - const CellRef *_end; - public: - EqualRange(const CellRef *begin_in, const CellRef *end_in) - : _begin(begin_in), _end(end_in) {} - const CellRef *begin() const { return _begin; }; - const CellRef *end() const { return _end; } - bool empty() const { return (_begin == _end); } - }; -private: - /** - * Address comparator only looking at a subset of the labels. - **/ - struct Less { - IndexList selector; - explicit Less(const IndexList &selector_in) : selector(selector_in) {} - bool operator()(const CellRef &a, const CellRef &b) const { - for (size_t idx: selector) { - if (a.get().address[idx] != b.get().address[idx]) { - return (a.get().address[idx] < b.get().address[idx]); - } - } - return false; - } - }; - Less _less; - std::vector<CellRef> _refs; - - EqualRange make_range(const CellRef *begin) const { - const CellRef *end = (begin < refs_end()) ? (begin + 1) : begin; - while ((end < refs_end()) && !_less(*(end - 1), *end)) { - ++end; - } - return EqualRange(begin, end); - } - -public: - View(const SimpleTensor &tensor, const IndexList &selector) - : _less(selector), _refs() - { - for (const auto &cell: tensor.my_cells()) { - _refs.emplace_back(cell); - } - std::sort(_refs.begin(), _refs.end(), _less); - } - View(const EqualRange &range, const IndexList &selector) - : _less(selector), _refs() - { - for (const auto &cell: range) { - _refs.emplace_back(cell); - } - std::sort(_refs.begin(), _refs.end(), _less); - } - ~View() {} - const IndexList &selector() const { return _less.selector; } - const CellRef *refs_begin() const { return &_refs[0]; } - const CellRef *refs_end() const { return (refs_begin() + _refs.size()); } - EqualRange first_range() const { return make_range(refs_begin()); } - EqualRange next_range(const EqualRange &prev) const { return make_range(prev.end()); } -}; - -/** - * Helper class used to find matching EqualRanges from two different - * SimpleTensor Views. - **/ -class ViewMatcher { -public: - /** - * Comparator used to cross-compare addresses across two different - * views only looking at the overlapping dimensions between the - * views. - **/ - struct CrossCompare { - enum class Result { LESS, EQUAL, GREATER }; - IndexList a_selector; - IndexList b_selector; - CrossCompare(const IndexList &a_selector_in, const IndexList &b_selector_in) - : a_selector(a_selector_in), b_selector(b_selector_in) - { - assert(a_selector.size() == b_selector.size()); - } - ~CrossCompare() {} - Result compare(const Cell &a, const Cell &b) const { - for (size_t i = 0; i < a_selector.size(); ++i) { - if (a.address[a_selector[i]] != b.address[b_selector[i]]) { - if (a.address[a_selector[i]] < b.address[b_selector[i]]) { - return Result::LESS; - } else { - return Result::GREATER; - } - } - } - return Result::EQUAL; - } - }; - using EqualRange = View::EqualRange; - -private: - const View &_a; - const View &_b; - EqualRange _a_range; - EqualRange _b_range; - CrossCompare _cmp; - - bool has_a() const { return !_a_range.empty(); } - bool has_b() const { return !_b_range.empty(); } - void next_a() { _a_range = _a.next_range(_a_range); } - void next_b() { _b_range = _b.next_range(_b_range); } - - void find_match() { - while (valid()) { - switch (_cmp.compare(*get_a().begin(), *get_b().begin())) { - case CrossCompare::Result::LESS: - next_a(); - break; - case CrossCompare::Result::GREATER: - next_b(); - break; - case CrossCompare::Result::EQUAL: - return; - } - } - } - -public: - ViewMatcher(const View &a, const View &b) - : _a(a), _b(b), _a_range(_a.first_range()), _b_range(b.first_range()), - _cmp(a.selector(), b.selector()) - { - find_match(); - } - ~ViewMatcher() {} - bool valid() const { return (has_a() && has_b()); } - const EqualRange &get_a() const { return _a_range; } - const EqualRange &get_b() const { return _b_range; } - void next() { - next_a(); - next_b(); - find_match(); - } -}; - -struct Format { - bool is_sparse; - bool is_dense; - bool with_cell_type; - uint32_t tag; - explicit Format(const TypeMeta &meta) - : is_sparse(meta.mapped.size() > 0), - is_dense((meta.indexed.size() > 0) || !is_sparse), - with_cell_type(meta.cell_type != CellType::DOUBLE), - tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} - explicit Format(uint32_t tag_in) - : is_sparse((tag_in & 0x1) != 0), - is_dense((tag_in & 0x2) != 0), - with_cell_type((tag_in & 0x4) != 0), - tag(tag_in) {} - ~Format() {} -}; - -void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { - if (format.with_cell_type) { - output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); - } -} - -void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { - maybe_encode_cell_type(output, format, meta); - if (format.is_sparse) { - output.putInt1_4Bytes(meta.mapped.size()); - for (size_t idx: meta.mapped) { - output.writeSmallString(type.dimensions()[idx].name); - } - } - if (format.is_dense) { - output.putInt1_4Bytes(meta.indexed.size()); - for (size_t idx: meta.indexed) { - output.writeSmallString(type.dimensions()[idx].name); - output.putInt1_4Bytes(type.dimensions()[idx].size); - } - } -} - -void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { - if ((meta.mapped.size() > 0)) { - output.putInt1_4Bytes(num_blocks); - } -} - -void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address &addr) { - for (size_t idx: meta.mapped) { - output.writeSmallString(addr[idx].name); - } -} - -CellType maybe_decode_cell_type(nbostream &input, const Format &format) { - if (format.with_cell_type) { - return id_to_cell_type(input.getInt1_4Bytes()); - } - return CellType::DOUBLE; -} - -ValueType decode_type(nbostream &input, const Format &format) { - CellType cell_type = maybe_decode_cell_type(input, format); - std::vector<ValueType::Dimension> dim_list; - if (format.is_sparse) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name); - } - } - if (format.is_dense) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name, input.getInt1_4Bytes()); - } - } - return ValueType::tensor_type(std::move(dim_list), cell_type); -} - -size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { - if ((meta.mapped.size() > 0) || !format.is_dense) { - return input.getInt1_4Bytes(); - } - return 1; -} - -void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) { - for (size_t idx: meta.mapped) { - vespalib::string name; - input.readSmallString(name); - addr[idx] = Label(name); - } -} - -void decode_cells(nbostream &input, const ValueType &type, const TypeMeta meta, - Address &address, size_t n, Builder &builder) -{ - if (n < meta.indexed.size()) { - Label &label = address[meta.indexed[n]]; - size_t size = type.dimensions()[meta.indexed[n]].size; - for (label.index = 0; label.index < size; ++label.index) { - decode_cells(input, type, meta, address, n + 1, builder); - } - } else { - double value = (meta.cell_type == CellType::FLOAT) - ? input.readValue<float>() - : input.readValue<double>(); - builder.set(address, value); - } -} - -} // namespace vespalib::eval::<unnamed> - -constexpr size_t TensorSpec::Label::npos; -constexpr size_t SimpleTensor::Label::npos; - -SimpleTensor::SimpleTensor() - : Tensor(SimpleTensorEngine::ref()), - _type(ValueType::error_type()), - _cells() -{ -} - -SimpleTensor::SimpleTensor(double value) - : Tensor(SimpleTensorEngine::ref()), - _type(ValueType::double_type()), - _cells({Cell({},value)}) -{ -} - -SimpleTensor::SimpleTensor(const ValueType &type_in, Cells cells_in) - : Tensor(SimpleTensorEngine::ref()), - _type(type_in), - _cells(std::move(cells_in)) -{ - assert_type(_type); - for (const auto &cell: _cells) { - assert_address(cell.address, _type); - } - std::sort(_cells.begin(), _cells.end(), - [](const auto &a, const auto &b){ return (a.address < b.address); }); -} - -double -SimpleTensor::as_double() const -{ - double sum = 0.0; - for (auto &cell: _cells) { - sum += cell.value; - } - return sum; -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::map(map_fun_t function) const -{ - Cells cells(_cells); - for (auto &cell: cells) { - cell.value = function(cell.value); - } - return std::make_unique<SimpleTensor>(_type, std::move(cells)); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::reduce(Aggregator &aggr, const std::vector<vespalib::string> &dimensions) const -{ - ValueType result_type = _type.reduce(dimensions); - if (result_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(result_type); - IndexList selector = TypeAnalyzer(_type, result_type).overlap_a; - View view(*this, selector); - for (View::EqualRange range = view.first_range(); !range.empty(); range = view.next_range(range)) { - auto pos = range.begin(); - aggr.first((pos++)->get().value); - for (; pos != range.end(); ++pos) { - aggr.next(pos->get().value); - } - builder.set(select(range.begin()->get().address, selector), aggr.result()); - } - return builder.build(); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::rename(const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) const -{ - ValueType result_type = _type.rename(from, to); - if (result_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(result_type); - IndexList selector; - for (const auto &dim: result_type.dimensions()) { - selector.push_back(_type.dimension_index(reverse_rename(dim.name, from, to))); - } - for (auto &cell: _cells) { - builder.set(select(cell.address, selector), cell.value); - } - return builder.build(); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::create(const TensorSpec &spec) -{ - ValueType my_type = ValueType::from_spec(spec.type()); - if (my_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(my_type); - for (const auto &cell: spec.cells()) { - builder.set(cell.first, cell.second); - } - return builder.build(); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::join(const SimpleTensor &a, const SimpleTensor &b, join_fun_t function) -{ - ValueType result_type = ValueType::join(a.type(), b.type()); - if (result_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(result_type); - TypeAnalyzer type_info(a.type(), b.type()); - View view_a(a, type_info.overlap_a); - View view_b(b, type_info.overlap_b); - for (ViewMatcher matcher(view_a, view_b); matcher.valid(); matcher.next()) { - for (const auto &ref_a: matcher.get_a()) { - for (const auto &ref_b: matcher.get_b()) { - builder.set(select(ref_a.get().address, ref_b.get().address, type_info.combine), - function(ref_a.get().value, ref_b.get().value)); - } - } - } - return builder.build(); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::merge(const SimpleTensor &a, const SimpleTensor &b, join_fun_t function) -{ - ValueType result_type = ValueType::merge(a.type(), b.type()); - if (result_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(result_type); - auto cmp = [](const Cell &x, const Cell &y) { return (x.address < y.address); }; - auto visitor = overload{ - [&builder](visit_ranges_either, const Cell &x) { builder.set(x.address, x.value); }, - [&builder,function](visit_ranges_both, const Cell &x, const Cell &y) { - builder.set(x.address, function(x.value, y.value)); - }}; - visit_ranges(visitor, a._cells.begin(), a._cells.end(), b._cells.begin(), b._cells.end(), cmp); - return builder.build(); -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::concat(const SimpleTensor &a, const SimpleTensor &b, const vespalib::string &dimension) -{ - ValueType result_type = ValueType::concat(a.type(), b.type(), dimension); - if (result_type.is_error()) { - return std::make_unique<SimpleTensor>(); - } - Builder builder(result_type); - TypeAnalyzer type_info(a.type(), b.type(), dimension); - View view_a(a, type_info.overlap_a); - View view_b(b, type_info.overlap_b); - size_t cat_dim_idx = result_type.dimension_index(dimension); - size_t cat_offset = get_dimension_size(a.type(), type_info.ignored_a); - for (ViewMatcher matcher(view_a, view_b); matcher.valid(); matcher.next()) { - View subview_a(matcher.get_a(), type_info.only_a); - View subview_b(matcher.get_b(), type_info.only_b); - for (auto range_a = subview_a.first_range(); !range_a.empty(); range_a = subview_a.next_range(range_a)) { - for (auto range_b = subview_b.first_range(); !range_b.empty(); range_b = subview_b.next_range(range_b)) { - Address addr = select(range_a.begin()->get().address, range_b.begin()->get().address, type_info.combine); - addr.insert(addr.begin() + cat_dim_idx, Label(size_t(0))); - for (const auto &ref: range_a) { - addr[cat_dim_idx].index = get_dimension_index(ref.get().address, type_info.ignored_a); - builder.set(addr, ref.get().value); - } - for (const auto &ref: range_b) { - addr[cat_dim_idx].index = cat_offset + get_dimension_index(ref.get().address, type_info.ignored_b); - builder.set(addr, ref.get().value); - } - } - } - } - return builder.build(); -} - -void -SimpleTensor::encode(const SimpleTensor &tensor, nbostream &output) -{ - TypeMeta meta(tensor.type()); - Format format(meta); - output.putInt1_4Bytes(format.tag); - encode_type(output, format, tensor.type(), meta); - maybe_encode_num_blocks(output, meta, tensor.my_cells().size() / meta.block_size); - View view(tensor, meta.mapped); - for (auto block = view.first_range(); !block.empty(); block = view.next_range(block)) { - encode_mapped_labels(output, meta, block.begin()->get().address); - View subview(block, meta.indexed); - for (auto cell = subview.first_range(); !cell.empty(); cell = subview.next_range(cell)) { - if (meta.cell_type == CellType::FLOAT) { - output << (float) cell.begin()->get().value; - } else { - output << cell.begin()->get().value; - } - } - } -} - -std::unique_ptr<SimpleTensor> -SimpleTensor::decode(nbostream &input) -{ - Format format(input.getInt1_4Bytes()); - ValueType type = decode_type(input, format); - TypeMeta meta(type); - Builder builder(type); - size_t num_blocks = maybe_decode_num_blocks(input, meta, format); - Address address(type.dimensions().size(), Label(size_t(0))); - for (size_t i = 0; i < num_blocks; ++i) { - decode_mapped_labels(input, meta, address); - decode_cells(input, type, meta, address, 0, builder); - } - return builder.build(); -} - -vespalib::MemoryUsage -SimpleTensor::get_memory_usage() const { - size_t addr_use = sizeof(Label) * _type.dimensions().size(); - size_t cell_use = sizeof(Cell) + addr_use; - size_t cells_use = _cells.size() * cell_use; - - size_t addr_alloc = sizeof(Label) * _type.dimensions().capacity(); - size_t cell_alloc = sizeof(Cell) + addr_alloc; - size_t cells_alloc = _cells.capacity() * cell_alloc; - - size_t mine_sz = sizeof(SimpleTensor); - size_t used = mine_sz + cells_use; - size_t allocated = mine_sz + cells_alloc; - return MemoryUsage(allocated, used, 0, 0); -} - -} // namespace vespalib::eval -} // namespace vespalib diff --git a/eval/src/vespa/eval/eval/simple_tensor.h b/eval/src/vespa/eval/eval/simple_tensor.h deleted file mode 100644 index 7ab59199af2..00000000000 --- a/eval/src/vespa/eval/eval/simple_tensor.h +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "value_type.h" -#include "tensor.h" -#include "tensor_spec.h" -#include "aggr.h" -#include "operation.h" -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stash.h> -#include <memory> -#include <map> -#include <functional> - -namespace vespalib { - -class nbostream; - -namespace eval { - -struct UnaryOperation; -struct BinaryOperation; - -/** - * A tensor supporting a mix of indexed and mapped dimensions. The - * goal for this class is to be a simple, complete and correct - * reference implementation supporting all relevant tensor operations. - **/ -class SimpleTensor : public Tensor -{ -public: - /** - * A label for a single dimension. This is either a string - * (mapped) or an integer (indexed). A sequence of Labels form an - * Address. The labels must have the same order as the dimensions - * in the tensor type (which are sorted on dimension name). Labels - * for mapped dimensions must be strings and labels for indexed - * dimensions must be integers smaller than the dimension size. - **/ - struct Label { - size_t index; - vespalib::string name; - static constexpr size_t npos = -1; - Label(const TensorSpec::Label &label) noexcept - : index(label.index), name(label.name) {} - bool operator<(const Label &rhs) const { - if (index != rhs.index) { - return (index < rhs.index); - } - return (name < rhs.name); - } - bool operator==(const Label &rhs) const { - return ((index == rhs.index) && (name == rhs.name)); - } - bool operator!=(const Label &rhs) const { return !(*this == rhs); } - bool is_mapped() const { return (index == npos); } - bool is_indexed() const { return (index != npos); } - }; - using Address = std::vector<Label>; - - /** - * A tensor has a type and contains a collection of Cells. Each - * cell has an Address and a value. - **/ - struct Cell { - Address address; - double value; - Cell(const Address &address_in, double value_in) - : address(address_in), value(value_in) {} - }; - using Cells = std::vector<Cell>; - -private: - ValueType _type; - Cells _cells; - -public: - using map_fun_t = vespalib::eval::operation::op1_t; - using join_fun_t = vespalib::eval::operation::op2_t; - - SimpleTensor(); - TypedCells cells() const override { abort(); } - const Index &index() const override { abort(); } - explicit SimpleTensor(double value); - SimpleTensor(const ValueType &type_in, Cells cells_in); - double as_double() const final override; - const ValueType &type() const override { return _type; } - const Cells &my_cells() const { return _cells; } - std::unique_ptr<SimpleTensor> map(map_fun_t function) const; - std::unique_ptr<SimpleTensor> reduce(Aggregator &aggr, const std::vector<vespalib::string> &dimensions) const; - std::unique_ptr<SimpleTensor> rename(const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) const; - static std::unique_ptr<SimpleTensor> create(const TensorSpec &spec); - static std::unique_ptr<SimpleTensor> join(const SimpleTensor &a, const SimpleTensor &b, join_fun_t function); - static std::unique_ptr<SimpleTensor> merge(const SimpleTensor &a, const SimpleTensor &b, join_fun_t function); - static std::unique_ptr<SimpleTensor> concat(const SimpleTensor &a, const SimpleTensor &b, const vespalib::string &dimension); - static void encode(const SimpleTensor &tensor, nbostream &output); - static std::unique_ptr<SimpleTensor> decode(nbostream &input); - MemoryUsage get_memory_usage() const override; -}; - -} // namespace vespalib::eval -} // namespace vespalib diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp deleted file mode 100644 index 491a310dc0b..00000000000 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_tensor_engine.h" -#include "simple_tensor.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <cassert> - -namespace vespalib { -namespace eval { - -namespace { - -const SimpleTensor &as_simple(const Tensor &tensor) { - assert(&tensor.engine() == &SimpleTensorEngine::ref()); - return static_cast<const SimpleTensor&>(tensor); -} - -const SimpleTensor &to_simple(const Value &value, Stash &stash) { - if (value.is_double()) { - return stash.create<SimpleTensor>(value.as_double()); - } - if (auto tensor = value.as_tensor()) { - return as_simple(*tensor); - } - return stash.create<SimpleTensor>(); // error -} - -template <typename F> -void with_simple(const Value &value, const F &f) { - if (value.is_double()) { - f(SimpleTensor(value.as_double())); - } else if (auto tensor = value.as_tensor()) { - f(as_simple(*tensor)); - } else { - f(SimpleTensor()); - } -} - -const Value &to_value(std::unique_ptr<SimpleTensor> tensor, Stash &stash) { - if (tensor->type().is_tensor()) { - return *stash.create<Value::UP>(std::move(tensor)); - } - return stash.create<DoubleValue>(tensor->as_double()); -} - -Value::UP to_value(std::unique_ptr<SimpleTensor> tensor) { - if (tensor->type().is_tensor()) { - return tensor; - } - return std::make_unique<DoubleValue>(tensor->as_double()); -} - -} // namespace vespalib::eval::<unnamed> - -const SimpleTensorEngine SimpleTensorEngine::_engine; - -//----------------------------------------------------------------------------- - -TensorSpec -SimpleTensorEngine::to_spec(const Value &value) const -{ - TensorSpec spec(value.type().to_spec()); - const auto &dimensions = value.type().dimensions(); - with_simple(value, [&spec,&dimensions](const SimpleTensor &simple_tensor) - { - for (const auto &cell: simple_tensor.my_cells()) { - TensorSpec::Address addr; - assert(cell.address.size() == dimensions.size()); - for (size_t i = 0; i < cell.address.size(); ++i) { - const auto &label = cell.address[i]; - if (label.is_mapped()) { - addr.emplace(dimensions[i].name, TensorSpec::Label(label.name)); - } else { - addr.emplace(dimensions[i].name, TensorSpec::Label(label.index)); - } - } - spec.add(addr, cell.value); - } - }); - return spec; -} - -Value::UP -SimpleTensorEngine::from_spec(const TensorSpec &spec) const -{ - return to_value(SimpleTensor::create(spec)); -} - -//----------------------------------------------------------------------------- - -void -SimpleTensorEngine::encode(const Value &value, nbostream &output) const -{ - with_simple(value, [&output](const SimpleTensor &tensor) { SimpleTensor::encode(tensor, output); }); -} - -Value::UP -SimpleTensorEngine::decode(nbostream &input) const -{ - return to_value(SimpleTensor::decode(input)); -} - -//----------------------------------------------------------------------------- - -const Value & -SimpleTensorEngine::map(const Value &a, map_fun_t function, Stash &stash) const -{ - if (a.is_double()) { - return stash.create<DoubleValue>(function(a.as_double())); - } - return to_value(to_simple(a, stash).map(function), stash); -} - -const Value & -SimpleTensorEngine::join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const -{ - if (a.is_double() && b.is_double()) { - return stash.create<DoubleValue>(function(a.as_double(), b.as_double())); - } - return to_value(SimpleTensor::join(to_simple(a, stash), to_simple(b, stash), function), stash); -} - -const Value & -SimpleTensorEngine::merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) const -{ - return to_value(SimpleTensor::merge(to_simple(a, stash), to_simple(b, stash), function), stash); -} - -const Value & -SimpleTensorEngine::reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const -{ - return to_value(to_simple(a, stash).reduce(Aggregator::create(aggr, stash), dimensions), stash); -} - -const Value & -SimpleTensorEngine::concat(const Value &a, const Value &b, const vespalib::string &dimension, Stash &stash) const -{ - return to_value(SimpleTensor::concat(to_simple(a, stash), to_simple(b, stash), dimension), stash); -} - -const Value & -SimpleTensorEngine::rename(const Value &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const -{ - return to_value(to_simple(a, stash).rename(from, to), stash); -} - -//----------------------------------------------------------------------------- - -} // namespace vespalib::eval -} // namespace vespalib diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.h b/eval/src/vespa/eval/eval/simple_tensor_engine.h deleted file mode 100644 index 4c71e91c8d3..00000000000 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "tensor_engine.h" - -namespace vespalib { -namespace eval { - -/** - * This is a TensorEngine implementation for the SimpleTensor - * reference implementation. - **/ -class SimpleTensorEngine : public TensorEngine -{ -private: - SimpleTensorEngine() {} - static const SimpleTensorEngine _engine; -public: - static const TensorEngine &ref() { return _engine; }; - - TensorSpec to_spec(const Value &value) const override; - std::unique_ptr<Value> from_spec(const TensorSpec &spec) const override; - - void encode(const Value &value, nbostream &output) const override; - std::unique_ptr<Value> decode(nbostream &input) const override; - - const Value &map(const Value &a, map_fun_t function, Stash &stash) const override; - const Value &join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; - const Value &merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; - const Value &reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const override; - const Value &concat(const Value &a, const Value &b, const vespalib::string &dimension, Stash &stash) const override; - const Value &rename(const Value &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const override; -}; - -} // namespace vespalib::eval -} // namespace vespalib diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index bbee96eeef0..0492b810d62 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -5,7 +5,6 @@ #include "operation.h" #include "tensor.h" #include "tensor_engine.h" -#include "simple_tensor_engine.h" #include "visit_stuff.h" #include "string_stuff.h" #include <vespa/eval/instruction/generic_concat.h> diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index c4ef2082a84..b02424169a1 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -7,7 +7,6 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/vespalib/util/stash.h> #include <set> #include <functional> diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 75be3e802ea..30f36abacbf 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -5,6 +5,5 @@ vespa_add_library(eval_tensor OBJECT partial_update.cpp tensor.cpp tensor_address.cpp - wrapped_simple_tensor.cpp wrapped_simple_value.cpp ) diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp deleted file mode 100644 index fd49d1d83bc..00000000000 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "cell_values.h" -#include "tensor_address_builder.h" -#include "tensor_visitor.h" -#include "wrapped_simple_tensor.h" -#include <vespa/eval/eval/simple_tensor_engine.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/vespalib/util/stringfmt.h> - -#include <vespa/log/log.h> -LOG_SETUP(".eval.tensor.wrapped_simple_tensor"); - -namespace vespalib::tensor { - -using eval::SimpleTensor; -using eval::TensorSpec; - -eval::TensorSpec -WrappedSimpleTensor::toSpec() const -{ - return eval::SimpleTensorEngine::ref().to_spec(_tensor); -} - -double -WrappedSimpleTensor::as_double() const -{ - return _tensor.as_double(); -} - -void -WrappedSimpleTensor::accept(TensorVisitor &visitor) const -{ - TensorAddressBuilder addr; - const auto &dimensions = _tensor.type().dimensions(); - for (const auto &cell: _tensor.my_cells()) { - addr.clear(); - for (size_t i = 0; i < dimensions.size(); ++i) { - if (dimensions[i].is_indexed()) { - addr.add(dimensions[i].name, make_string("%zu", cell.address[i].index)); - } else { - addr.add(dimensions[i].name, cell.address[i].name); - } - } - visitor.visit(addr.build(), cell.value); - } -} - -MemoryUsage -WrappedSimpleTensor::get_memory_usage() const -{ - size_t used = sizeof(WrappedSimpleTensor); - if (_space) { - auto plus = _space->get_memory_usage(); - plus.incUsedBytes(used); - plus.incAllocatedBytes(used); - return plus; - } - return MemoryUsage(used, used, 0, 0); -} - -//----------------------------------------------------------------------------- - -Tensor::UP -WrappedSimpleTensor::apply(const CellFunction &) const -{ - LOG_ABORT("should not be reached"); -} - -Tensor::UP -WrappedSimpleTensor::join(join_fun_t, const Tensor &) const -{ - LOG_ABORT("should not be reached"); -} - -Tensor::UP -WrappedSimpleTensor::merge(join_fun_t, const Tensor &) const -{ - LOG_ABORT("should not be reached"); -} - -Tensor::UP -WrappedSimpleTensor::reduce(join_fun_t, const std::vector<vespalib::string> &) const -{ - LOG_ABORT("should not be reached"); -} - -namespace { - -TensorSpec::Address -convertToOnlyMappedDimensions(const TensorSpec::Address &address) -{ - TensorSpec::Address result; - for (const auto &elem : address) { - if (elem.second.is_indexed()) { - result.emplace(std::make_pair(elem.first, - TensorSpec::Label(vespalib::make_string("%zu", elem.second.index)))); - } else { - result.emplace(elem); - } - } - return result; -} - -} - -std::unique_ptr<Tensor> -WrappedSimpleTensor::modify(join_fun_t op, const CellValues &cellValues) const -{ - TensorSpec oldTensor = toSpec(); - TensorSpec toModify = cellValues.toSpec(); - TensorSpec result(type().to_spec()); - - for (const auto &cell : oldTensor.cells()) { - TensorSpec::Address mappedAddress = convertToOnlyMappedDimensions(cell.first); - auto itr = toModify.cells().find(mappedAddress); - if (itr != toModify.cells().end()) { - result.add(cell.first, op(cell.second, itr->second)); - } else { - result.add(cell.first, cell.second); - } - } - return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); -} - -std::unique_ptr<Tensor> -WrappedSimpleTensor::add(const Tensor &arg) const -{ - const auto *rhs = dynamic_cast<const WrappedSimpleTensor *>(&arg); - if (!rhs || type() != rhs->type()) { - return Tensor::UP(); - } - TensorSpec oldTensor = toSpec(); - TensorSpec argTensor = rhs->toSpec(); - TensorSpec result(type().to_spec()); - for (const auto &cell : oldTensor.cells()) { - auto argItr = argTensor.cells().find(cell.first); - if (argItr != argTensor.cells().end()) { - result.add(argItr->first, argItr->second); - } else { - result.add(cell.first, cell.second); - } - } - for (const auto &cell : argTensor.cells()) { - auto resultItr = result.cells().find(cell.first); - if (resultItr == result.cells().end()) { - result.add(cell.first, cell.second); - } - } - return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); -} - -namespace { - -TensorSpec::Address -extractMappedDimensions(const TensorSpec::Address &address) -{ - TensorSpec::Address result; - for (const auto &elem : address) { - if (elem.second.is_mapped()) { - result.emplace(elem); - } - } - return result; -} - -} - -std::unique_ptr<Tensor> -WrappedSimpleTensor::remove(const CellValues &cellAddresses) const -{ - TensorSpec oldTensor = toSpec(); - TensorSpec toRemove = cellAddresses.toSpec(); - TensorSpec result(type().to_spec()); - - for (const auto &cell : oldTensor.cells()) { - TensorSpec::Address mappedAddress = extractMappedDimensions(cell.first); - auto itr = toRemove.cells().find(mappedAddress); - if (itr == toRemove.cells().end()) { - result.add(cell.first, cell.second); - } - } - return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); -} - -} diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h deleted file mode 100644 index c2199ccb49d..00000000000 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "tensor.h" -#include <vespa/eval/eval/simple_tensor.h> - -namespace vespalib::tensor { - -/** - * A thin wrapper around a SimpleTensor (tensor reference - * implementation) to be used as fallback for tensors with data - * layouts not supported by the default tensor implementation. - * - * Tensor implementation class is currently inferred from its value - * type. Consider adding explicit tagging to the tensor::Tensor - * default implementation top-level class in the future. - **/ -class WrappedSimpleTensor : public Tensor -{ -private: - std::unique_ptr<eval::SimpleTensor> _space; - const eval::SimpleTensor &_tensor; -public: - explicit WrappedSimpleTensor(const eval::SimpleTensor &tensor) - : _space(), _tensor(tensor) {} - explicit WrappedSimpleTensor(std::unique_ptr<eval::SimpleTensor> tensor) - : _space(std::move(tensor)), _tensor(*_space) {} - eval::TypedCells cells() const override { abort(); } - const Index &index() const override { abort(); } - ~WrappedSimpleTensor() {} - const eval::SimpleTensor &get() const { return _tensor; } - const eval::ValueType &type() const override { return _tensor.type(); } - eval::TensorSpec toSpec() const override; - double as_double() const override; - void accept(TensorVisitor &visitor) const override; - MemoryUsage get_memory_usage() const override; - // functions below should not be used for this implementation - Tensor::UP apply(const CellFunction &) const override; - Tensor::UP join(join_fun_t, const Tensor &) const override; - Tensor::UP merge(join_fun_t, const Tensor &) const override; - Tensor::UP reduce(join_fun_t, const std::vector<vespalib::string> &) const override; - std::unique_ptr<Tensor> modify(join_fun_t, const CellValues &) const override; - std::unique_ptr<Tensor> add(const Tensor &arg) const override; - std::unique_ptr<Tensor> remove(const CellValues &) const override; -}; - -} // namespace vespalib::tensor |