diff options
author | Arne Juul <arnej@verizonmedia.com> | 2020-12-03 19:27:47 +0000 |
---|---|---|
committer | Arne Juul <arnej@verizonmedia.com> | 2020-12-03 19:27:47 +0000 |
commit | 9e7eacc5694ceb32f845df7a0036781c88b0cc15 (patch) | |
tree | 1ab39d22962e9bda4df0bba04bd1bb8365c34870 /eval | |
parent | 0127a1e99d83b707957b099dd436889583f2b53f (diff) |
remove DefaultTensorEngine
Diffstat (limited to 'eval')
25 files changed, 1 insertions, 1125 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index d7fb168cc9d..17f689248eb 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -74,12 +74,8 @@ vespa_define_module( src/tests/tensor/partial_add src/tests/tensor/partial_modify src/tests/tensor/partial_remove - src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance - src/tests/tensor/tensor_modify_operation - src/tests/tensor/tensor_remove_operation - src/tests/tensor/tensor_serialization src/tests/tensor/typed_cells src/tests/tensor/vector_from_doubles_function diff --git a/eval/src/tests/tensor/partial_add/partial_add_test.cpp b/eval/src/tests/tensor/partial_add/partial_add_test.cpp index 711bf7a1c49..42db77a311f 100644 --- a/eval/src/tests/tensor/partial_add/partial_add_test.cpp +++ b/eval/src/tests/tensor/partial_add/partial_add_test.cpp @@ -3,7 +3,6 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/partial_update.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp b/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp index ab04f195b1f..a39bcc87ab3 100644 --- a/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp +++ b/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp @@ -3,7 +3,6 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/partial_update.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp index fc22a364e5c..410331c5eb7 100644 --- a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp +++ b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp @@ -3,7 +3,6 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/partial_update.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt deleted file mode 100644 index 275043c8c54..00000000000 --- a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_add_operation_test_app TEST - SOURCES - tensor_add_operation_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_tensor_add_operation_test_app COMMAND eval_tensor_add_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp b/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp deleted file mode 100644 index 1d39a557114..00000000000 --- a/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/vespalib/gtest/gtest.h> - -using vespalib::eval::Value; -using vespalib::eval::TensorSpec; -using vespalib::tensor::test::makeTensor; -using namespace vespalib::tensor; - -void -assertAdd(const TensorSpec &source, const TensorSpec &arg, const TensorSpec &expected) -{ - auto sourceTensor = makeTensor<Tensor>(source); - auto argTensor = makeTensor<Tensor>(arg); - auto resultTensor = sourceTensor->add(*argTensor); - auto actual = resultTensor->toSpec(); - EXPECT_EQ(actual, expected); -} - -void -assertNullTensor(const TensorSpec &source, const TensorSpec &arg) -{ - auto sourceTensor = makeTensor<Tensor>(source); - auto argTensor = makeTensor<Tensor>(arg); - auto resultTensor = sourceTensor->add(*argTensor); - EXPECT_FALSE(resultTensor); -} - -TEST(TensorAddTest, cells_can_be_added_to_a_sparse_tensor) -{ - assertAdd(TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 2) - .add({{"x","c"},{"y","d"}}, 3), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 5) - .add({{"x","e"},{"y","f"}}, 7), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 5) - .add({{"x","c"},{"y","d"}}, 3) - .add({{"x","e"},{"y","f"}}, 7)); -} - -TEST(TensorAddTest, cells_can_be_added_to_a_mixed_tensor) -{ - assertAdd(TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3) - .add({{"x","b"},{"y",0}}, 4) - .add({{"x","b"},{"y",1}}, 5), - TensorSpec("tensor(x{},y[2])") - .add({{"x","b"},{"y",0}}, 6) - .add({{"x","b"},{"y",1}}, 7) - .add({{"x","c"},{"y",0}}, 8) - .add({{"x","c"},{"y",1}}, 9), - TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3) - .add({{"x","b"},{"y",0}}, 6) - .add({{"x","b"},{"y",1}}, 7) - .add({{"x","c"},{"y",0}}, 8) - .add({{"x","c"},{"y",1}}, 9)); -} - -TEST(TensorAddTest, cells_can_be_added_to_empty_mixed_tensor) -{ - assertAdd(TensorSpec("tensor(x{},y[2])"), - TensorSpec("tensor(x{},y[2])") - .add({{"x","b"},{"y",0}}, 6) - .add({{"x","b"},{"y",1}}, 7), - TensorSpec("tensor(x{},y[2])") - .add({{"x","b"},{"y",0}}, 6) - .add({{"x","b"},{"y",1}}, 7)); -} - -TEST(TensorAddTest, tensors_of_different_types_cannot_be_added_together) -{ - assertNullTensor(TensorSpec("tensor(x{},y[2])"), TensorSpec("tensor(x{},y{})")); - assertNullTensor(TensorSpec("tensor(x{},y[2])"), TensorSpec("tensor(x{},y[3])")); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt deleted file mode 100644 index cd4643deae0..00000000000 --- a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_modify_operation_test_app TEST - SOURCES - tensor_modify_operation_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_tensor_modify_operation_test_app COMMAND eval_tensor_modify_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp b/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp deleted file mode 100644 index 00ee55ed7af..00000000000 --- a/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/operation.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/cell_values.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/stringfmt.h> - -using vespalib::eval::Value; -using vespalib::eval::TensorSpec; -using vespalib::tensor::test::makeTensor; -using namespace vespalib::tensor; - -void -checkUpdate(const TensorSpec &source, const TensorSpec &update, const TensorSpec &expect) -{ - auto sourceTensor = makeTensor<Tensor>(source); - auto updateTensor = makeTensor<SparseTensor>(update); - const CellValues cellValues(*updateTensor); - - auto actualTensor = sourceTensor->modify(vespalib::eval::operation::Add::f, cellValues); - auto actual = actualTensor->toSpec(); - auto expectTensor = makeTensor<Tensor>(expect); - auto expectPadded = expectTensor->toSpec(); - EXPECT_EQ(actual, expectPadded); -} - -TEST(TensorModifyTest, sparse_tensors_can_be_modified) -{ - checkUpdate(TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 11) - .add({{"x","9"},{"y","9"}}, 11), - TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2), - TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 13) - .add({{"x","9"},{"y","9"}}, 11)); -} - -TEST(TensorModifyTest, dense_tensors_can_be_modified) -{ - checkUpdate(TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 11) - .add({{"x",9},{"y",9}}, 11), - TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2), - TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 13) - .add({{"x",9},{"y",9}}, 11)); -} - -TEST(TensorModifyTest, mixed_tensors_can_be_modified) -{ - checkUpdate(TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3) - .add({{"x","b"},{"y",0}}, 4) - .add({{"x","b"},{"y",1}}, 5), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","0"}}, 6) - .add({{"x","b"},{"y","1"}}, 7), - TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 8) - .add({{"x","a"},{"y",1}}, 3) - .add({{"x","b"},{"y",0}}, 4) - .add({{"x","b"},{"y",1}}, 12)); -} - -TEST(TensorModifyTest, sparse_tensors_ignore_updates_to_missing_cells) -{ - checkUpdate(TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 11) - .add({{"x","9"},{"y","9"}}, 11), - TensorSpec("tensor(x{},y{})") - .add({{"x","7"},{"y","9"}}, 2) - .add({{"x","8"},{"y","9"}}, 2), - TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 13) - .add({{"x","9"},{"y","9"}}, 11)); -} - -TEST(TensorModifyTest, dense_tensors_ignore_updates_to_out_of_range_cells) -{ - checkUpdate(TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 11) - .add({{"x",9},{"y",9}}, 11), - TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2) - .add({{"x","10"},{"y","9"}}, 2), - TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 13) - .add({{"x",9},{"y",9}}, 11)); -} - -TEST(TensorModifyTest, mixed_tensors_ignore_updates_to_missing_or_out_of_range_cells) -{ - checkUpdate(TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","2"}}, 4) - .add({{"x","c"},{"y","0"}}, 5), - TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3)); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt deleted file mode 100644 index a302cc528e0..00000000000 --- a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_remove_operation_test_app TEST - SOURCES - tensor_remove_operation_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_tensor_remove_operation_test_app COMMAND eval_tensor_remove_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp b/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp deleted file mode 100644 index ceeaf2c99da..00000000000 --- a/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/cell_values.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/vespalib/gtest/gtest.h> - -using vespalib::eval::Value; -using vespalib::eval::TensorSpec; -using vespalib::tensor::test::makeTensor; -using namespace vespalib::tensor; - -void -assertRemove(const TensorSpec &source, const TensorSpec &arg, const TensorSpec &expected) -{ - auto sourceTensor = makeTensor<Tensor>(source); - auto argTensor = makeTensor<SparseTensor>(arg); - auto resultTensor = sourceTensor->remove(CellValues(*argTensor)); - auto actual = resultTensor->toSpec(); - EXPECT_EQ(actual, expected); -} - -TEST(TensorRemoveTest, cells_can_be_removed_from_a_sparse_tensor) -{ - assertRemove(TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 2) - .add({{"x","c"},{"y","d"}}, 3), - TensorSpec("tensor(x{},y{})") - .add({{"x","c"},{"y","d"}}, 1) - .add({{"x","e"},{"y","f"}}, 1), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 2)); -} - -TEST(TensorRemoveTest, all_cells_can_be_removed_from_a_sparse_tensor) -{ - assertRemove(TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 2), - TensorSpec("tensor(x{},y{})") - .add({{"x","a"},{"y","b"}}, 1), - TensorSpec("tensor(x{},y{})")); -} - -TEST(TensorRemoveTest, cells_can_be_removed_from_a_mixed_tensor) -{ - assertRemove(TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3) - .add({{"x","b"},{"y",0}}, 4) - .add({{"x","b"},{"y",1}}, 5), - TensorSpec("tensor(x{})") - .add({{"x","b"}}, 1) - .add({{"x","c"}}, 1), - TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3)); - - assertRemove(TensorSpec("tensor(x{},y{},z[2])") - .add({{"x","a"},{"y","c"},{"z",0}}, 2) - .add({{"x","a"},{"y","c"},{"z",1}}, 3) - .add({{"x","b"},{"y","c"},{"z",0}}, 4) - .add({{"x","b"},{"y","c"},{"z",1}}, 5), - TensorSpec("tensor(x{},y{})") - .add({{"x","b"},{"y","c"}}, 1) - .add({{"x","c"},{"y","c"}}, 1), - TensorSpec("tensor(x{},y{},z[2])") - .add({{"x","a"},{"y","c"},{"z",0}}, 2) - .add({{"x","a"},{"y","c"},{"z",1}}, 3)); - - assertRemove(TensorSpec("tensor(x{},y[1],z[2])") - .add({{"x","a"},{"y",0},{"z",0}}, 2) - .add({{"x","a"},{"y",0},{"z",1}}, 3) - .add({{"x","b"},{"y",0},{"z",0}}, 4) - .add({{"x","b"},{"y",0},{"z",1}}, 5), - TensorSpec("tensor(x{})") - .add({{"x","b"}}, 1) - .add({{"x","c"}}, 1), - TensorSpec("tensor(x{},y[1],z[2])") - .add({{"x","a"},{"y",0},{"z",0}}, 2) - .add({{"x","a"},{"y",0},{"z",1}}, 3)); -} - -TEST(TensorRemoveTest, all_cells_can_be_removed_from_a_mixed_tensor) -{ - assertRemove(TensorSpec("tensor(x{},y[2])") - .add({{"x","a"},{"y",0}}, 2) - .add({{"x","a"},{"y",1}}, 3), - TensorSpec("tensor(x{})") - .add({{"x","a"}}, 1), - TensorSpec("tensor(x{},y[2])")); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_serialization/.gitignore b/eval/src/tests/tensor/tensor_serialization/.gitignore deleted file mode 100644 index f8525561c6b..00000000000 --- a/eval/src/tests/tensor/tensor_serialization/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_serialization_test_app diff --git a/eval/src/tests/tensor/tensor_serialization/CMakeLists.txt b/eval/src/tests/tensor/tensor_serialization/CMakeLists.txt deleted file mode 100644 index b9fc7e0d544..00000000000 --- a/eval/src/tests/tensor/tensor_serialization/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_serialization_test_app TEST - SOURCES - tensor_serialization_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_tensor_serialization_test_app COMMAND eval_tensor_serialization_test_app) diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp deleted file mode 100644 index 358f5d36101..00000000000 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/eval/tensor/serialization/sparse_binary_format.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/objects/hexdump.h> -#include <ostream> -#include <vespa/eval/tensor/dense/dense_tensor_view.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/simple_value.h> - -using namespace vespalib::tensor; -using vespalib::eval::TensorSpec; -using vespalib::nbostream; -using ExpBuffer = std::vector<uint8_t>; - -namespace std { - -bool operator==(const std::vector<uint8_t> &exp, const nbostream &stream) -{ - return ((exp.size() == stream.size()) && - (memcmp(&exp[0], stream.peek(), exp.size()) == 0)); -} - -std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs) -{ - out << vespalib::HexDump(&rhs[0], rhs.size()); - return out; -} - -} - -//----------------------------------------------------------------------------- - -template <typename T> -void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { - nbostream input(&exp[0], exp.size()); - std::vector<T> cells; - TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells); - ASSERT_EQUAL(cells.size(), spec.cells().size()); - size_t i = 0; - for (const auto &cell: spec.cells()) { - EXPECT_EQUAL(cells[i++], cell.second.value); - } - ASSERT_EQUAL(i, cells.size()); -} - -TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &spec) { - const auto &factory = vespalib::eval::SimpleValueBuilderFactory::get(); - auto new_value = vespalib::eval::value_from_spec(spec, factory); - auto new_value_spec = vespalib::eval::spec_from_value(*new_value); - nbostream actual; - vespalib::eval::encode_value(*new_value, actual); - ASSERT_EQUAL(exp, actual); - auto new_decoded = vespalib::eval::decode_value(actual, factory); - auto new_decoded_spec = vespalib::eval::spec_from_value(*new_decoded); - EXPECT_EQUAL(0u, actual.size()); - EXPECT_EQUAL(new_value_spec, new_decoded_spec); - if (new_value->type().is_dense()) { - TEST_DO(verify_cells_only<float>(exp, new_value_spec)); - TEST_DO(verify_cells_only<double>(exp, new_value_spec)); - } - return new_decoded_spec; -} - -void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { - auto &engine = DefaultTensorEngine::ref(); - auto value = engine.from_spec(spec); - auto value_spec = engine.to_spec(*value); - nbostream actual; - engine.encode(*value, actual); - EXPECT_EQUAL(exp, actual); - auto decoded = engine.decode(actual); - auto decoded_spec = engine.to_spec(*decoded); - EXPECT_EQUAL(0u, actual.size()); - EXPECT_EQUAL(value_spec, decoded_spec); - if (value->type().is_dense()) { - TEST_DO(verify_cells_only<float>(exp, value_spec)); - TEST_DO(verify_cells_only<double>(exp, value_spec)); - } - auto new_value_spec = verify_new_value_serialized(exp, spec); - EXPECT_EQUAL(value_spec, new_value_spec); -} - -//----------------------------------------------------------------------------- - -TEST("test tensor serialization for SparseTensor") { - TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, - TensorSpec("tensor(x{})"))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, - TensorSpec("tensor(x{},y{})"))); - TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - TensorSpec("tensor(x{})") - .add({{"x", "1"}}, 3))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, - 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00 }, - TensorSpec("tensor(x{},y{})") - .add({{"x", ""}, {"y", ""}}, 3))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, - 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - TensorSpec("tensor(x{},y{})") - .add({{"x", "1"}, {"y", ""}}, 3))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, - 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - TensorSpec("tensor(x{},y{})") - .add({{"x", ""}, {"y", "3"}}, 3))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, - 0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00 }, - TensorSpec("tensor(x{},y{})") - .add({{"x", "2"}, {"y", "4"}}, 3))); - TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, - 0x01, 0x01, 0x31, 0x00, 0x40, 0x08, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - TensorSpec("tensor(x{},y{})") - .add({{"x", "1"}, {"y", ""}}, 3))); -} - -TEST("test float cells from sparse tensor") { - TEST_DO(verify_serialized({ 0x05, 0x01, - 0x02, 0x01, 0x78, 0x01, 0x79, - 0x01, 0x01, 0x31, 0x00, - 0x40, 0x40, 0x00, 0x00 }, - TensorSpec("tensor<float>(x{},y{})") - .add({{"x", "1"}, {"y", ""}}, 3))); -} - -TEST("test tensor serialization for DenseTensor") { - TEST_DO(verify_serialized({0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("double"))); - TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[1])") - .add({{"x", 0}}, 0))); - TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[1],y[1])") - .add({{"x", 0}, {"y", 0}}, 0))); - TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[2])") - .add({{"x", 1}}, 3))); - TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[1],y[1])") - .add({{"x", 0}, {"y", 0}}, 3))); - TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[2],y[1])") - .add({{"x", 1}, {"y", 0}}, 3))); - TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x04, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[1],y[4])") - .add({{"x", 0}, {"y", 3}}, 3))); - TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - TensorSpec("tensor(x[3],y[5])") - .add({{"x", 2}, {"y", 4}}, 3))); -} - -TEST("test float cells for dense tensor") { - TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x40, 0x00, 0x00 }, - TensorSpec("tensor<float>(x[3],y[5])") - .add({{"x", 2}, {"y", 4}}, 3))); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/engine_or_factory.cpp b/eval/src/vespa/eval/eval/engine_or_factory.cpp index c40d681ab24..f7e0490435c 100644 --- a/eval/src/vespa/eval/eval/engine_or_factory.cpp +++ b/eval/src/vespa/eval/eval/engine_or_factory.cpp @@ -10,7 +10,7 @@ #include <vespa/eval/instruction/generic_merge.h> #include <vespa/eval/instruction/generic_reduce.h> #include <vespa/eval/instruction/generic_rename.h> -#include <vespa/eval/tensor/default_tensor_engine.h> +#include "tensor_engine.h" #include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> @@ -156,11 +156,6 @@ EngineOrFactory::get() vespalib::string EngineOrFactory::to_string() const { - if (is_engine()) { - if (&engine() == &tensor::DefaultTensorEngine::ref()) { - return "DefaultTensorEngine"; - } - } if (is_factory()) { if (&factory() == &FastValueBuilderFactory::get()) { return "FastValueBuilderFactory"; diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 30f36abacbf..5d332e9b582 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(eval_tensor OBJECT SOURCES - default_tensor_engine.cpp partial_update.cpp tensor.cpp tensor_address.cpp diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp deleted file mode 100644 index 49b5118f777..00000000000 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "default_tensor_engine.h" -#include "tensor.h" -#include "wrapped_simple_value.h" -#include "serialization/typed_binary_format.h" -#include "sparse/sparse_tensor_address_builder.h" -#include "sparse/direct_sparse_tensor_builder.h" -#include "dense/dense_tensor.h" -#include "dense/typed_dense_tensor_builder.h" -#include <vespa/eval/instruction/dense_dot_product_function.h> -#include <vespa/eval/instruction/dense_xw_product_function.h> -#include <vespa/eval/instruction/dense_matmul_function.h> -#include <vespa/eval/instruction/dense_multi_matmul_function.h> -#include "dense/dense_fast_rename_optimizer.h" -#include "dense/dense_add_dimension_optimizer.h" -#include "dense/dense_single_reduce_function.h" -#include "dense/dense_remove_dimension_optimizer.h" -#include <vespa/eval/instruction/dense_lambda_peek_optimizer.h> -#include "dense/dense_lambda_function.h" -#include <vespa/eval/instruction/dense_simple_expand_function.h> -#include "dense/dense_simple_join_function.h" -#include "dense/dense_number_join_function.h" -#include "dense/dense_pow_as_map_optimizer.h" -#include "dense/dense_simple_map_function.h" -#include "dense/vector_from_doubles_function.h" -#include "dense/dense_tensor_create_function.h" -#include <vespa/eval/instruction/dense_tensor_peek_function.h> -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/operation.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/exceptions.h> -#include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".eval.tensor.default_tensor_engine"); - -namespace vespalib::tensor { - -using eval::Aggr; -using eval::Aggregator; -using eval::DoubleValue; -using eval::TensorFunction; -using eval::TensorSpec; -using eval::Value; -using eval::ValueType; -using vespalib::IllegalArgumentException; -using vespalib::make_string; - -using map_fun_t = vespalib::eval::operation::op1_t; -using join_fun_t = vespalib::eval::operation::op2_t; - -namespace { - -constexpr size_t UNDEFINED_IDX = std::numeric_limits<size_t>::max(); - -const eval::EngineOrFactory &simple_engine() { - static eval::EngineOrFactory engine(eval::SimpleValueBuilderFactory::get()); - return engine; -} -const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); } - -// map tensors to simple tensors before fall-back evaluation - -const Value &to_simple(const Value &value, Stash &stash) { - if (auto tensor = value.as_tensor()) { - if (auto wrapped = dynamic_cast<const WrappedSimpleValue *>(tensor)) { - return wrapped->unwrap(); - } - nbostream data; - encode_value(*tensor, data); - return *stash.create<Value::UP>(simple_engine().decode(data)); - } - return value; -} - -// map tensors to default tensors after fall-back evaluation - -const Value &to_default(const Value &value, Stash &stash) { - if (! value.type().is_double()) { - if (! Tensor::supported({value.type()})) { - return stash.create<WrappedSimpleValue>(value); - } - nbostream data; - simple_engine().encode(value, data); - return *stash.create<Value::UP>(default_engine().decode(data)); - } - return value; -} - -const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { - assert(tensor); - if (tensor->type().is_tensor()) { - return *stash.create<Value::UP>(std::move(tensor)); - } - return stash.create<DoubleValue>(tensor->as_double()); -} - -Value::UP to_value(std::unique_ptr<Tensor> tensor) { - if (tensor->type().is_tensor()) { - return tensor; - } - return std::make_unique<DoubleValue>(tensor->as_double()); -} - -const Value &fallback_join(const Value &a, const Value &b, join_fun_t function, Stash &stash) { - return to_default(simple_engine().join(to_simple(a, stash), to_simple(b, stash), function, stash), stash); -} - -const Value &fallback_merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) { - return to_default(simple_engine().merge(to_simple(a, stash), to_simple(b, stash), function, stash), stash); -} - -const Value &fallback_reduce(const Value &a, eval::Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { - return to_default(simple_engine().reduce(to_simple(a, stash), aggr, dimensions, stash), stash); -} - -size_t calculate_cell_index(const ValueType &type, const TensorSpec::Address &address) { - if (type.dimensions().size() != address.size()) { - return UNDEFINED_IDX; - } - size_t d = 0; - size_t idx = 0; - for (const auto &binding: address) { - const auto &dim = type.dimensions()[d++]; - if ((dim.name != binding.first) || (binding.second.index >= dim.size)) { - return UNDEFINED_IDX; - } - idx *= dim.size; - idx += binding.second.index; - } - return idx; -} - -bool build_cell_address(const ValueType &type, const TensorSpec::Address &address, - SparseTensorAddressBuilder &builder) -{ - if (type.dimensions().size() != address.size()) { - return false; - } - size_t d = 0; - builder.clear(); - for (const auto &binding: address) { - const auto &dim = type.dimensions()[d++]; - if (dim.name != binding.first) { - return false; - } - builder.add(binding.second.name); - } - return true; -} - -void bad_spec(const TensorSpec &spec) { - throw IllegalArgumentException(make_string("malformed tensor spec: %s", spec.to_string().c_str())); -} - -} // namespace vespalib::tensor::<unnamed> - -const DefaultTensorEngine DefaultTensorEngine::_engine; - -TensorSpec -DefaultTensorEngine::to_spec(const Value &value) const -{ - return TensorSpec::from_value(value); -} - -struct CallDenseTensorBuilder { - template <typename CT> - static Value::UP - invoke(const ValueType &type, const TensorSpec &spec) - { - TypedDenseTensorBuilder<CT> builder(type); - for (const auto &cell: spec.cells()) { - const auto &address = cell.first; - size_t cell_idx = calculate_cell_index(type, address); - if (cell_idx == UNDEFINED_IDX) { - bad_spec(spec); - } - builder.insertCell(cell_idx, cell.second); - } - return builder.build(); - } -}; - -struct CallSparseTensorBuilder { - template <typename CT> - static Value::UP - invoke(const ValueType &type, const TensorSpec &spec) - { - DirectSparseTensorBuilder<CT> builder(type); - builder.reserve(spec.cells().size()); - SparseTensorAddressBuilder address_builder; - for (const auto &cell: spec.cells()) { - const auto &address = cell.first; - if (build_cell_address(type, address, address_builder)) { - builder.insertCell(address_builder, cell.second); - } else { - bad_spec(spec); - } - } - return builder.build(); - } -}; - -using MyTypify = eval::TypifyCellType; - -Value::UP -DefaultTensorEngine::from_spec(const TensorSpec &spec) const -{ - ValueType type = ValueType::from_spec(spec.type()); - if (type.is_error()) { - bad_spec(spec); - } else if (type.is_double()) { - double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; - return std::make_unique<DoubleValue>(value); - } else if (type.is_dense()) { - return typify_invoke<1,MyTypify,CallDenseTensorBuilder>(type.cell_type(), type, spec); - } else if (type.is_sparse()) { - return typify_invoke<1,MyTypify,CallSparseTensorBuilder>(type.cell_type(), type, spec); - } - return std::make_unique<WrappedSimpleValue>(simple_engine().from_spec(spec)); -} - -struct CellFunctionFunAdapter : tensor::CellFunction { - map_fun_t fun; - CellFunctionFunAdapter(map_fun_t fun_in) : fun(fun_in) {} - virtual double apply(double value) const override { return fun(value); } -}; - -struct CellFunctionBindLeftAdapter : tensor::CellFunction { - join_fun_t fun; - double a; - CellFunctionBindLeftAdapter(join_fun_t fun_in, double bound) : fun(fun_in), a(bound) {} - virtual double apply(double b) const override { return fun(a, b); } -}; - -struct CellFunctionBindRightAdapter : tensor::CellFunction { - join_fun_t fun; - double b; - CellFunctionBindRightAdapter(join_fun_t fun_in, double bound) : fun(fun_in), b(bound) {} - virtual double apply(double a) const override { return fun(a, b); } -}; - -//----------------------------------------------------------------------------- - -void -DefaultTensorEngine::encode(const Value &value, nbostream &output) const -{ - if (auto tensor = value.as_tensor()) { - TypedBinaryFormat::serialize(output, static_cast<const tensor::Tensor &>(*tensor)); - } else { - TypedBinaryFormat::serialize(output, DenseTensor<double>(ValueType::double_type(), {value.as_double()})); - } -} - -Value::UP -DefaultTensorEngine::decode(nbostream &input) const -{ - return to_value(TypedBinaryFormat::deserialize(input)); -} - -//----------------------------------------------------------------------------- - -const TensorFunction & -DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const -{ - using Child = TensorFunction::Child; - Child root(expr); - { - std::vector<Child::CREF> nodes({root}); - for (size_t i = 0; i < nodes.size(); ++i) { - nodes[i].get().get().push_children(nodes); - } - while (!nodes.empty()) { - const Child &child = nodes.back().get(); - child.set(eval::DenseDotProductFunction::optimize(child.get(), stash)); - child.set(eval::DenseXWProductFunction::optimize(child.get(), stash)); - child.set(eval::DenseMatMulFunction::optimize(child.get(), stash)); - child.set(eval::DenseMultiMatMulFunction::optimize(child.get(), stash)); - nodes.pop_back(); - } - } - { - std::vector<Child::CREF> nodes({root}); - for (size_t i = 0; i < nodes.size(); ++i) { - nodes[i].get().get().push_children(nodes); - } - while (!nodes.empty()) { - const Child &child = nodes.back().get(); - child.set(eval::DenseSimpleExpandFunction::optimize(child.get(), stash)); - child.set(DenseAddDimensionOptimizer::optimize(child.get(), stash)); - child.set(DenseRemoveDimensionOptimizer::optimize(child.get(), stash)); - child.set(VectorFromDoublesFunction::optimize(child.get(), stash)); - child.set(DenseTensorCreateFunction::optimize(child.get(), stash)); - child.set(eval::DenseTensorPeekFunction::optimize(child.get(), stash)); - child.set(eval::DenseLambdaPeekOptimizer::optimize(child.get(), stash)); - child.set(DenseLambdaFunction::optimize(child.get(), stash)); - child.set(DenseFastRenameOptimizer::optimize(child.get(), stash)); - child.set(DensePowAsMapOptimizer::optimize(child.get(), stash)); - child.set(DenseSimpleMapFunction::optimize(child.get(), stash)); - child.set(DenseSimpleJoinFunction::optimize(child.get(), stash)); - child.set(DenseNumberJoinFunction::optimize(child.get(), stash)); - child.set(DenseSingleReduceFunction::optimize(child.get(), stash)); - nodes.pop_back(); - } - } - return root.get(); -} - -//----------------------------------------------------------------------------- - -const Value & -DefaultTensorEngine::map(const Value &a, map_fun_t function, Stash &stash) const -{ - if (auto tensor = a.as_tensor()) { - const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor); - if (!tensor::Tensor::supported({my_a.type()})) { - return to_default(simple_engine().map(to_simple(a, stash), function, stash), stash); - } - CellFunctionFunAdapter cell_function(function); - return to_value(my_a.apply(cell_function), stash); - } else { - return stash.create<DoubleValue>(function(a.as_double())); - } -} - -const Value & -DefaultTensorEngine::join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const -{ - if (auto tensor_a = a.as_tensor()) { - const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor_a); - if (auto tensor_b = b.as_tensor()) { - const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(*tensor_b); - if (!tensor::Tensor::supported({my_a.type(), my_b.type()})) { - return fallback_join(a, b, function, stash); - } - return to_value(my_a.join(function, my_b), stash); - } else { - if (!tensor::Tensor::supported({my_a.type()})) { - return fallback_join(a, b, function, stash); - } - CellFunctionBindRightAdapter cell_function(function, b.as_double()); - return to_value(my_a.apply(cell_function), stash); - } - } else { - if (auto tensor_b = b.as_tensor()) { - const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(*tensor_b); - if (!tensor::Tensor::supported({my_b.type()})) { - return fallback_join(a, b, function, stash); - } - CellFunctionBindLeftAdapter cell_function(function, a.as_double()); - return to_value(my_b.apply(cell_function), stash); - } else { - return stash.create<DoubleValue>(function(a.as_double(), b.as_double())); - } - } -} - -const Value & -DefaultTensorEngine::merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) const -{ - if (auto tensor_a = a.as_tensor()) { - auto tensor_b = b.as_tensor(); - assert(tensor_b); - const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor_a); - const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(*tensor_b); - if (!tensor::Tensor::supported({my_a.type(), my_b.type()})) { - return fallback_merge(a, b, function, stash); - } - return to_value(my_a.merge(function, my_b), stash); - } else { - return stash.create<DoubleValue>(function(a.as_double(), b.as_double())); - } -} - -const Value & -DefaultTensorEngine::reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const -{ - if (auto tensor = a.as_tensor()) { - const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor); - if (!tensor::Tensor::supported({my_a.type()})) { - return fallback_reduce(a, aggr, dimensions, stash); - } - switch (aggr) { - case Aggr::PROD: return to_value(my_a.reduce(eval::operation::Mul::f, dimensions), stash); - case Aggr::SUM: - if (dimensions.empty()) { - return stash.create<eval::DoubleValue>(my_a.as_double()); - } else { - return to_value(my_a.reduce(eval::operation::Add::f, dimensions), stash); - } - case Aggr::MAX: return to_value(my_a.reduce(eval::operation::Max::f, dimensions), stash); - case Aggr::MIN: return to_value(my_a.reduce(eval::operation::Min::f, dimensions), stash); - default: - return fallback_reduce(a, aggr, dimensions, stash); - } - } else { - Aggregator &aggregator = Aggregator::create(aggr, stash); - aggregator.first(a.as_double()); - return stash.create<DoubleValue>(aggregator.result()); - } -} - -size_t vector_size(const ValueType &type, const vespalib::string &dimension) { - if (type.is_double()) { - return 1; - } else if ((type.dimensions().size() == 1) && - (type.dimensions()[0].is_indexed()) && - (type.dimensions()[0].name == dimension)) - { - return type.dimensions()[0].size; - } else { - return 0; - } -} - -template <typename OCT> -struct CallAppendVector { - template <typename CT> - static void call(const ConstArrayRef<CT> &arr, OCT *&pos) { - for (CT cell: arr) { *pos++ = cell; } - } -}; - -template <typename OCT> -void append_vector(OCT *&pos, const Value &value) { - if (auto tensor = value.as_tensor()) { - dispatch_1<CallAppendVector<OCT> >(tensor->cells(), pos); - } else { - *pos++ = value.as_double(); - } -} - -template <typename OCT> -const Value &concat_vectors(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { - ArrayRef<OCT> cells = stash.create_uninitialized_array<OCT>(vector_size); - OCT *pos = cells.begin(); - append_vector<OCT>(pos, a); - append_vector<OCT>(pos, b); - assert(pos == cells.end()); - const ValueType &type = stash.create<ValueType>(ValueType::tensor_type({ValueType::Dimension(dimension, vector_size)}, ValueType::unify_cell_types(a.type(), b.type()))); - return stash.create<DenseTensorView>(type, TypedCells(cells)); -} - -struct CallConcatVectors { - template <typename OCT> - static const Value &invoke(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { - return concat_vectors<OCT>(a, b, dimension, vector_size, stash); - } -}; - -const Value & -DefaultTensorEngine::concat(const Value &a, const Value &b, const vespalib::string &dimension, Stash &stash) const -{ - size_t a_size = vector_size(a.type(), dimension); - size_t b_size = vector_size(b.type(), dimension); - if ((a_size > 0) && (b_size > 0)) { - CellType result_cell_type = ValueType::unify_cell_types(a.type(), b.type()); - return typify_invoke<1,MyTypify,CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); - } - return to_default(simple_engine().concat(to_simple(a, stash), to_simple(b, stash), dimension, stash), stash); -} - -const Value & -DefaultTensorEngine::rename(const Value &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const -{ - return to_default(simple_engine().rename(to_simple(a, stash), from, to, stash), stash); -} - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.h b/eval/src/vespa/eval/tensor/default_tensor_engine.h deleted file mode 100644 index 5c39706b326..00000000000 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/tensor_engine.h> - -namespace vespalib::tensor { - -/** - * This is a tensor engine implementation wrapping the default tensor - * implementations (dense/sparse). - **/ -class DefaultTensorEngine : public eval::TensorEngine -{ -private: - DefaultTensorEngine() {} - static const DefaultTensorEngine _engine; -public: - static const TensorEngine &ref() { return _engine; }; - - TensorSpec to_spec(const Value &value) const override; - std::unique_ptr<Value> from_spec(const TensorSpec &spec) const override; - - void encode(const Value &value, nbostream &output) const override; - std::unique_ptr<Value> decode(nbostream &input) const override; - - const TensorFunction &optimize(const TensorFunction &expr, Stash &stash) const override; - - const Value &map(const Value &a, map_fun_t function, Stash &stash) const override; - const Value &join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; - const Value &merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; - const Value &reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const override; - const Value &concat(const Value &a, const Value &b, const vespalib::string &dimension, Stash &stash) const override; - const Value &rename(const Value &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const override; -}; - -} diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp index d4b1ee4506e..e0e66aae01c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -15,7 +15,6 @@ using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; -using eval::TensorEngine; using eval::TypifyCellType; using eval::as; diff --git a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp index c12ad21c7fb..d7bc7dc860d 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { using eval::Value; using eval::ValueType; using eval::TensorFunction; -using eval::TensorEngine; using eval::as; using namespace eval::tensor_function; diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 02867f646b4..8e7bd72a0f6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -18,7 +18,6 @@ using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; -using eval::TensorEngine; using eval::TypifyCellType; using eval::as; diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp index 4c7b1b3dee8..b6376326468 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp @@ -14,7 +14,6 @@ using vespalib::ArrayRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; -using eval::TensorEngine; using eval::TypifyCellType; using eval::as; diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index c692f7c4a0f..82d4505dc80 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -10,7 +10,6 @@ namespace vespalib::tensor { using eval::Aggr; using eval::InterpretedFunction; -using eval::TensorEngine; using eval::TensorFunction; using eval::Value; using eval::ValueType; diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp index f425ce4fd1c..4cdaa01eea8 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp @@ -12,7 +12,6 @@ using eval::DoubleValue; using eval::ValueType; using eval::TensorSpec; using eval::TensorFunction; -using eval::TensorEngine; using Child = eval::TensorFunction::Child; using eval::as; using namespace eval::tensor_function; diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index a5e26665741..0d653acee4d 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { using eval::Value; using eval::ValueType; using eval::TensorFunction; -using eval::TensorEngine; using Child = eval::TensorFunction::Child; using eval::as; using namespace eval::tensor_function; diff --git a/eval/src/vespa/eval/tensor/test/test_utils.h b/eval/src/vespa/eval/tensor/test/test_utils.h deleted file mode 100644 index 5daae74284b..00000000000 --- a/eval/src/vespa/eval/tensor/test/test_utils.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/vespalib/testkit/test_kit.h> - -namespace vespalib::tensor::test { - -template <typename T> -std::unique_ptr<T> -makeTensor(const vespalib::eval::TensorSpec &spec) -{ - auto value = DefaultTensorEngine::ref().from_spec(spec); - T *tensor = dynamic_cast<T *>(value.get()); - ASSERT_TRUE(tensor); - value.release(); - return std::unique_ptr<T>(tensor); -} - -} |