diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-02-08 12:09:56 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-02-08 12:09:56 +0000 |
commit | f21b553eb7903cf4a2d7a01fc222c9dfee8a3253 (patch) | |
tree | 297a7a2d0d89f5909df2a11547cbf995ecee83bc /eval | |
parent | 373531ebe5a36c6215aef12f8dc040cb63fa1685 (diff) |
Implement add() function on vespalib::tensor::Tensor for adding a set of cells to the tensor.
Currently only supported for sparse tensor.
Diffstat (limited to 'eval')
17 files changed, 209 insertions, 19 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index a5bb6fec31e..040a98290d2 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -36,6 +36,7 @@ vespa_define_module( src/tests/tensor/dense_tensor_builder src/tests/tensor/dense_xw_product_function src/tests/tensor/sparse_tensor_builder + src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance src/tests/tensor/tensor_mapper diff --git a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt new file mode 100644 index 00000000000..6c19b13db7e --- /dev/null +++ b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_tensor_add_operation_test_app TEST + SOURCES + tensor_add_operation_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_tensor_add_operation_test_app COMMAND eval_tensor_add_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp b/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp new file mode 100644 index 00000000000..5d017fdcd5c --- /dev/null +++ b/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp @@ -0,0 +1,46 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/sparse/sparse_tensor.h> +#include <vespa/vespalib/testkit/test_kit.h> + +using vespalib::eval::Value; +using vespalib::eval::TensorSpec; +using namespace vespalib::tensor; + +std::unique_ptr<Tensor> +makeTensor(const TensorSpec &spec) +{ + auto value = DefaultTensorEngine::ref().from_spec(spec); + const auto *tensor = dynamic_cast<const Tensor *>(value->as_tensor()); + ASSERT_TRUE(tensor); + value.release(); + return std::unique_ptr<Tensor>(const_cast<Tensor *>(tensor)); +} + +void +assertAdd(const TensorSpec &source, const TensorSpec &arg, const TensorSpec &expected) +{ + auto sourceTensor = makeTensor(source); + auto argTensor = makeTensor(arg); + auto resultTensor = sourceTensor->add(*argTensor); + auto actual = resultTensor->toSpec(); + EXPECT_EQUAL(actual, expected); +} + +TEST("require that cells can be added to a sparse tensor") +{ + assertAdd(TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","b"}}, 2) + .add({{"x","c"},{"y","d"}}, 3), + TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","b"}}, 5) + .add({{"x","e"},{"y","f"}}, 7), + TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","b"}}, 5) + .add({{"x","c"},{"y","d"}}, 3) + .add({{"x","e"},{"y","f"}}, 7)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index f758b0ec915..6243f79a971 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -13,6 +13,9 @@ #include <vespa/eval/eval/operation.h> #include <sstream> +#include <vespa/log/log.h> +LOG_SETUP(".eval.tensor.dense.dense_tensor_view"); + using vespalib::eval::TensorSpec; namespace vespalib::tensor { @@ -290,4 +293,10 @@ DenseTensorView::modify(join_fun_t op, const CellValues &cellValues) const return modifier.build(); } +std::unique_ptr<Tensor> +DenseTensorView::add(const Tensor &) const +{ + LOG_ABORT("should not be reached"); +} + } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index 5aedcf6fb8d..f470e9d374f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -54,6 +54,7 @@ public: Tensor::UP join(join_fun_t function, const Tensor &arg) const override; Tensor::UP reduce(join_fun_t op, const std::vector<vespalib::string> &dimensions) const override; std::unique_ptr<Tensor> modify(join_fun_t op, const CellValues &cellValues) const override; + std::unique_ptr<Tensor> add(const Tensor &arg) const override; bool equals(const Tensor &arg) const override; Tensor::UP clone() const override; eval::TensorSpec toSpec() const override; diff --git a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt index 57855dc7eaa..d50c6d5db10 100644 --- a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt @@ -2,6 +2,8 @@ vespa_add_library(eval_tensor_sparse OBJECT SOURCES sparse_tensor.cpp + sparse_tensor_add.cpp + sparse_tensor_address_builder.cpp sparse_tensor_address_combiner.cpp sparse_tensor_address_padder.cpp sparse_tensor_address_reducer.cpp diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp index dd2befd4df8..e3ee9593d80 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "sparse_tensor.h" +#include "sparse_tensor_add.h" #include "sparse_tensor_address_builder.h" -#include "sparse_tensor_match.h" #include "sparse_tensor_apply.hpp" -#include "sparse_tensor_reduce.hpp" +#include "sparse_tensor_match.h" #include "sparse_tensor_modify.h" +#include "sparse_tensor_reduce.hpp" +#include <vespa/eval/eval/operation.h> #include <vespa/eval/tensor/cell_values.h> #include <vespa/eval/tensor/tensor_address_builder.h> #include <vespa/eval/tensor/tensor_apply.h> #include <vespa/eval/tensor/tensor_visitor.h> -#include <vespa/eval/eval/operation.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> #include <vespa/vespalib/util/array_equal.hpp> @@ -199,6 +200,21 @@ SparseTensor::modify(join_fun_t op, const CellValues &cellValues) const return modifier.build(); } +std::unique_ptr<Tensor> +SparseTensor::add(const Tensor &arg) const +{ + const SparseTensor *rhs = dynamic_cast<const SparseTensor *>(&arg); + if (!rhs) { + return Tensor::UP(); + } + Cells cells; + Stash stash; + copyCells(cells, _cells, stash); + SparseTensorAdd adder(_type, std::move(cells), std::move(stash)); + rhs->accept(adder); + return adder.build(); +} + } VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(vespalib::tensor::SparseTensorAddressRef, double, vespalib::hash<vespalib::tensor::SparseTensorAddressRef>, diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index 36a0c246d25..107cba7a673 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -46,6 +46,7 @@ public: Tensor::UP join(join_fun_t function, const Tensor &arg) const override; Tensor::UP reduce(join_fun_t op, const std::vector<vespalib::string> &dimensions) const override; std::unique_ptr<Tensor> modify(join_fun_t op, const CellValues &cellValues) const override; + std::unique_ptr<Tensor> add(const Tensor &arg) const override; bool equals(const Tensor &arg) const override; Tensor::UP clone() const override; eval::TensorSpec toSpec() const override; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.cpp new file mode 100644 index 00000000000..4503787e00e --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.cpp @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_tensor_add.h" + +namespace vespalib::tensor { + +SparseTensorAdd::SparseTensorAdd(const eval::ValueType &type, Cells &&cells, Stash &&stash) + : _type(type), + _cells(std::move(cells)), + _stash(std::move(stash)), + _addressBuilder() +{ +} + +SparseTensorAdd::~SparseTensorAdd() = default; + +void +SparseTensorAdd::visit(const TensorAddress &address, double value) +{ + _addressBuilder.populate(_type, address); + auto addressRef = _addressBuilder.getAddressRef(); + // Make a persistent copy of the tensor address (owned by _stash) as the cell to insert might not already exist. + auto persistentAddress = SparseTensorAddressRef(addressRef, _stash); + _cells[persistentAddress] = value; +} + +std::unique_ptr<Tensor> +SparseTensorAdd::build() +{ + return std::make_unique<SparseTensor>(std::move(_type), std::move(_cells), std::move(_stash)); +} + +} diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.h new file mode 100644 index 00000000000..8adc95adf35 --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_add.h @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "sparse_tensor.h" +#include "sparse_tensor_address_builder.h" +#include <vespa/eval/tensor/tensor_visitor.h> + +namespace vespalib::tensor { + +/** + * This class handles a tensor add operation on a sparse tensor. + * + * Creates a new tensor by adding the cells of the argument tensor to this tensor. + * Existing cell values are overwritten. + */ +class SparseTensorAdd : public TensorVisitor +{ + using Cells = SparseTensor::Cells; + eval::ValueType _type; + Cells _cells; + Stash _stash; + SparseTensorAddressBuilder _addressBuilder; + +public: + SparseTensorAdd(const eval::ValueType &type, Cells &&cells, Stash &&stash); + ~SparseTensorAdd(); + void visit(const TensorAddress &address, double value) override; + std::unique_ptr<Tensor> build(); +}; + +} diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.cpp new file mode 100644 index 00000000000..3eb7d64f060 --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.cpp @@ -0,0 +1,29 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_tensor_address_builder.h" +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/tensor/tensor_address.h> +#include <vespa/eval/tensor/tensor_address_element_iterator.h> + +namespace vespalib::tensor { + +SparseTensorAddressBuilder::SparseTensorAddressBuilder() + : _address() +{ +} + +void +SparseTensorAddressBuilder::populate(const eval::ValueType &type, const TensorAddress &address) +{ + clear(); + TensorAddressElementIterator itr(address); + for (const auto &dimension : type.dimensions()) { + if (itr.skipToDimension(dimension.name)) { + add(itr.label()); + } else { + addUndefined(); + } + } +} + +} diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h index f74ce257b31..a6f05bb70fb 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h @@ -5,8 +5,12 @@ #include "sparse_tensor_address_ref.h" #include <vespa/vespalib/stllike/string.h> +namespace vespalib::eval { class ValueType; } + namespace vespalib::tensor { +class TensorAddress; + /** * A writer to serialize tensor addresses into a compact representation. @@ -32,7 +36,7 @@ protected: } } public: - SparseTensorAddressBuilder() : _address() {} + SparseTensorAddressBuilder(); void add(vespalib::stringref label) { ensure_room(label.size()+1); append(label); @@ -43,6 +47,7 @@ public: return SparseTensorAddressRef(&_address[0], _address.size()); } bool empty() const { return _address.empty(); } + void populate(const eval::ValueType &type, const TensorAddress &address); }; } diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_modify.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_modify.cpp index 91d0fa3fcdd..0ab8352bfbb 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_modify.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_modify.cpp @@ -19,16 +19,7 @@ SparseTensorModify::~SparseTensorModify() = default; void SparseTensorModify::visit(const TensorAddress &address, double value) { - TensorAddressElementIterator addressElementIterator(address); - - _addressBuilder.clear(); - for (const auto &dimension : _type.dimensions()) { - if (addressElementIterator.skipToDimension(dimension.name)) { - _addressBuilder.add(addressElementIterator.label()); - } else { - _addressBuilder.addUndefined(); - } - } + _addressBuilder.populate(_type, address); auto addressRef = _addressBuilder.getAddressRef(); auto cellItr = _cells.find(addressRef); if (cellItr != _cells.end()) { diff --git a/eval/src/vespa/eval/tensor/tensor.h b/eval/src/vespa/eval/tensor/tensor.h index 8e10af04b18..cdb9d90d3a3 100644 --- a/eval/src/vespa/eval/tensor/tensor.h +++ b/eval/src/vespa/eval/tensor/tensor.h @@ -23,8 +23,9 @@ class CellValues; * Each cell is identified by its address, which consists of a set of dimension -> label pairs, * where both dimension and label is a string on the form of an identifier or integer. */ -struct Tensor : public eval::Tensor +class Tensor : public eval::Tensor { +public: typedef std::unique_ptr<Tensor> UP; typedef std::reference_wrapper<const Tensor> CREF; using join_fun_t = double (*)(double, double); @@ -34,12 +35,20 @@ struct Tensor : public eval::Tensor virtual Tensor::UP apply(const CellFunction &func) const = 0; virtual Tensor::UP join(join_fun_t function, const Tensor &arg) const = 0; virtual Tensor::UP reduce(join_fun_t op, const std::vector<vespalib::string> &dimensions) const = 0; + /* * Creates a new tensor by modifying the underlying cells matching * the given cells applying a join function to determine the new * cell value. */ virtual std::unique_ptr<Tensor> modify(join_fun_t op, const CellValues &cellValues) const = 0; + + /** + * Creates a new tensor by adding the cells of the argument tensor to this tensor. + * Existing cell values are overwritten. + */ + virtual std::unique_ptr<Tensor> add(const Tensor &arg) const = 0; + virtual bool equals(const Tensor &arg) const = 0; // want to remove, but needed by document virtual Tensor::UP clone() const = 0; // want to remove, but needed by document virtual eval::TensorSpec toSpec() const = 0; diff --git a/eval/src/vespa/eval/tensor/tensor_address_element_iterator.h b/eval/src/vespa/eval/tensor/tensor_address_element_iterator.h index 0e8d88b3a1e..e413712362f 100644 --- a/eval/src/vespa/eval/tensor/tensor_address_element_iterator.h +++ b/eval/src/vespa/eval/tensor/tensor_address_element_iterator.h @@ -2,8 +2,9 @@ #pragma once -namespace vespalib { -namespace tensor { +#include <vespa/vespalib/stllike/hash_set.h> + +namespace vespalib::tensor { using DimensionsSet = vespalib::hash_set<vespalib::stringref>; @@ -40,5 +41,4 @@ public: } }; -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 394335d9b67..66fd2978a53 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -83,4 +83,10 @@ WrappedSimpleTensor::modify(join_fun_t, const CellValues &) const LOG_ABORT("should not be reached"); } +std::unique_ptr<Tensor> +WrappedSimpleTensor::add(const Tensor &) const +{ + LOG_ABORT("should not be reached"); +} + } // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h index 511fbc3c795..2d877b6fbbc 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h @@ -39,6 +39,7 @@ public: Tensor::UP join(join_fun_t, const Tensor &) const override; Tensor::UP reduce(join_fun_t, const std::vector<vespalib::string> &) const override; std::unique_ptr<Tensor> modify(join_fun_t, const CellValues &) const override; + std::unique_ptr<Tensor> add(const Tensor &arg) const override; }; } // namespace vespalib::tensor |