diff options
author | Haavard <havardpe@yahoo-inc.com> | 2017-05-22 13:28:55 +0000 |
---|---|---|
committer | Haavard <havardpe@yahoo-inc.com> | 2017-05-23 10:03:29 +0000 |
commit | 31cd122d2c18f2713b40d8b0d55e814fbff7fa81 (patch) | |
tree | dc4c7b440f72e5edc4e69207a2360a28c1befef7 /eval | |
parent | fdfd15fea0fa6be43dfa7015fa5e92bbbae5b708 (diff) |
support tensor mapping for wrapped simple tensors
Diffstat (limited to 'eval')
-rw-r--r-- | eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp | 441 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/tensor_spec.h | 5 | ||||
-rw-r--r-- | eval/src/vespa/eval/tensor/tensor_mapper.cpp | 140 | ||||
-rw-r--r-- | eval/src/vespa/eval/tensor/tensor_mapper.h | 3 | ||||
-rw-r--r-- | eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp | 39 | ||||
-rw-r--r-- | eval/src/vespa/eval/tensor/wrapped_simple_tensor.h | 4 |
6 files changed, 367 insertions, 265 deletions
diff --git a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index 8fede7e3d0b..6572ab1ed92 100644 --- a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -2,244 +2,235 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor_builder.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/tensor_factory.h> #include <vespa/eval/tensor/tensor_mapper.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <ostream> +#include <vespa/eval/tensor/wrapped_simple_tensor.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/simple_tensor.h> using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; +using vespalib::eval::SimpleTensor; using namespace vespalib::tensor; -namespace vespalib { -namespace tensor { +void verify_wrapped(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) { + auto tensor = std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(source)); + auto mapped = TensorMapper::mapToWrapped(*tensor, ValueType::from_spec(type)); + TensorSpec actual = mapped->toSpec(); + EXPECT_EQUAL(actual, expect); +} + +void verify(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) { + auto tensor = DefaultTensorEngine::ref().create(source); + const Tensor *tensor_impl = dynamic_cast<const Tensor *>(tensor.get()); + ASSERT_TRUE(tensor_impl); + TensorMapper mapper(ValueType::from_spec(type)); + auto mapped = mapper.map(*tensor_impl); + TensorSpec actual = mapped->toSpec(); + EXPECT_EQUAL(actual, expect); + TEST_DO(verify_wrapped(source, type, expect)); +} + +TEST("require that sparse tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","1"}}, 1) + .add({{"x","2"},{"y","1"}}, 3) + .add({{"x","1"},{"y","2"}}, 5) + .add({{"x","2"},{"y","2"}}, 7), + "tensor(y{})", + TensorSpec("tensor(y{})") + .add({{"y","1"}}, 4) + .add({{"y","2"}}, 12))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","1"}}, 1) + .add({{"x","2"},{"y","1"}}, 3) + .add({{"x","1"},{"y","2"}}, 5) + .add({{"x","2"},{"y","2"}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","1"}}, 6) + .add({{"x","2"}}, 10))); +} + +TEST("require that sparse tensors can be mapped to dense type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","0"}}, 1) + .add({{"x","2"},{"y","0"}}, 3) + .add({{"x","1"},{"y","1"}}, 5) + .add({{"x","2"},{"y","1"}}, 7), + "tensor(y[3])", + TensorSpec("tensor(y[3])") + .add({{"y",0}}, 4) + .add({{"y",1}}, 12) + .add({{"y",2}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","0x"}}, 1) + .add({{"x","2"},{"y",""}}, 3) + .add({{"x","1"},{"y","1"}}, 5) + .add({{"x","2"},{"y","10"}}, 7), + "tensor(y[3])", + TensorSpec("tensor(y[3])") + .add({{"y",0}}, 3) + .add({{"y",1}}, 5) + .add({{"y",2}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","1"}}, 7), + "tensor(x[2],y[3])", + TensorSpec("tensor(x[2],y[3])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",0},{"y",2}}, 0) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",1},{"y",2}}, 0))); +} -static bool operator==(const Tensor &lhs, const Tensor &rhs) -{ - return lhs.equals(rhs); +TEST("require that sparse tensors can be mapped to abstract dense type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","1"}}, 7), + "tensor(x[2],y[])", + TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","2"},{"y","0"}}, 7), + "tensor(x[],y[])", + TensorSpec("tensor(x[3],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",2},{"y",0}}, 7) + .add({{"x",2},{"y",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","3"}}, 7), + "tensor(x[],y[3])", + TensorSpec("tensor(x[2],y[3])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",0},{"y",2}}, 0) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",1},{"y",2}}, 0))); } +TEST("require that dense tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 3) + .add({{"x",1},{"y",0}}, 5) + .add({{"x",1},{"y",1}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","0"}}, 4) + .add({{"x","1"}}, 12))); } + +TEST("require that mixed tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","0"}}, 4) + .add({{"x","1"}}, 12))); +} + +TEST("require that mixed tensors can be mapped to dense type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(y[])", + TensorSpec("tensor(y[2])") + .add({{"y",0}}, 6) + .add({{"y",1}}, 10))); +} + +TEST("require that mixed tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(x{},y[])", + TensorSpec("tensor(x{},y[2])") + .add({{"x","0"},{"y",0}}, 1) + .add({{"x","0"},{"y",1}}, 3) + .add({{"x","1"},{"y",0}}, 5) + .add({{"x","1"},{"y",1}}, 7))); +} + +TEST("require that dense tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 3) + .add({{"x",1},{"y",0}}, 5) + .add({{"x",1},{"y",1}}, 7), + "tensor(x{},y[])", + TensorSpec("tensor(x{},y[2])") + .add({{"x","0"},{"y",0}}, 1) + .add({{"x","0"},{"y",1}}, 3) + .add({{"x","1"},{"y",0}}, 5) + .add({{"x","1"},{"y",1}}, 7))); } -template <typename BuilderType> -bool defaultBuilder() { return false; } - -template <> -bool defaultBuilder<DefaultTensor::builder>() { return true; } - -template <typename BuilderType> -struct TensorTFromBuilder; - -template <> -struct TensorTFromBuilder<SparseTensorBuilder> { - using TensorT = SparseTensor; -}; - -template <typename BuilderType> -using TensorTFromBuilder_t = typename TensorTFromBuilder<BuilderType>::TensorT; - -struct FixtureBase -{ - Tensor::UP createDenseTensor(const DenseTensorCells &cells) { - return TensorFactory::createDense(cells); - } -}; - -template <typename BuilderType> -struct Fixture : public FixtureBase -{ - BuilderType _builder; - using TensorT = TensorTFromBuilder_t<BuilderType>; - Fixture() : FixtureBase(), _builder() {} - - Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); - } - - void assertSparseMapImpl(const Tensor &exp, - const ValueType &tensorType, - const Tensor &rhs, bool isDefaultBuilder) - { - EXPECT_TRUE(tensorType.is_sparse()); - if (isDefaultBuilder) { - TensorMapper mapper(tensorType); - std::unique_ptr<Tensor> mapped = mapper.map(rhs); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - std::unique_ptr<Tensor> mapped = - TensorMapper::mapToSparse<TensorT>(rhs, tensorType); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - - void assertDenseMapImpl(const Tensor &exp, - const ValueType &tensorType, - const Tensor &rhs) - { - EXPECT_TRUE(tensorType.is_dense()); - TensorMapper mapper(tensorType); - std::unique_ptr<Tensor> mapped = mapper.map(rhs); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - - void - assertSparseMap(const TensorCells &expTensor, - const TensorDimensions &expDimensions, - const vespalib::string &typeSpec, - const TensorCells &rhsTensor, - const TensorDimensions &rhsDimensions) - { - assertSparseMapImpl(*createTensor(expTensor, expDimensions), - ValueType::from_spec(typeSpec), - *createTensor(rhsTensor, rhsDimensions), - defaultBuilder<BuilderType>()); - } - - void - assertDenseMap(const DenseTensorCells &expTensor, - const vespalib::string &typeSpec, - const TensorCells &rhsTensor, - const TensorDimensions &rhsDimensions) - { - assertDenseMapImpl(*createDenseTensor(expTensor), - ValueType::from_spec(typeSpec), - *createTensor(rhsTensor, rhsDimensions)); - } -}; - -using SparseFixture = Fixture<SparseTensorBuilder>; - -template <typename FixtureType> -void -testTensorMapper(FixtureType &f) -{ - TEST_DO(f.assertSparseMap({ - {{{"y","1"}}, 4}, - {{{"y","2"}}, 12} - }, - { "y" }, - "tensor(y{})", - { - {{{"x","1"},{"y","1"}}, 1}, - {{{"x","2"},{"y","1"}}, 3}, - {{{"x","1"},{"y","2"}}, 5}, - {{{"x","2"},{"y","2"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertSparseMap({ - {{{"x","1"}}, 6}, - {{{"x","2"}}, 10} - }, - { "x" }, - "tensor(x{})", - { - {{{"x","1"},{"y","1"}}, 1}, - {{{"x","2"},{"y","1"}}, 3}, - {{{"x","1"},{"y","2"}}, 5}, - {{{"x","2"},{"y","2"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"y",0}}, 4}, - {{{"y",1}}, 12}, - {{{"y",2}}, 0} - }, - "tensor(y[3])", - { - {{{"x","1"},{"y","0"}}, 1}, - {{{"x","2"},{"y","0"}}, 3}, - {{{"x","1"},{"y","1"}}, 5}, - {{{"x","2"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"y",0}}, 3}, - {{{"y",1}}, 5}, - {{{"y",2}}, 0} - }, - "tensor(y[3])", - { - {{{"x","1"},{"y","0x"}}, 1}, - {{{"x","2"},{"y",""}}, 3}, - {{{"x","1"},{"y","1"}}, 5}, - {{{"x","2"},{"y","10"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",0},{"y",2}}, 0}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",1},{"y",2}}, 0} - }, - "tensor(x[2], y[3])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0} - }, - "tensor(x[2], y[])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",2},{"y",0}}, 7}, - {{{"x",2},{"y",1}}, 0} - }, - "tensor(x[], y[])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","2"},{"y","0"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",0},{"y",2}}, 0}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",1},{"y",2}}, 0} - }, - "tensor(x[], y[3])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","3"}}, 7} - }, - { "x", "y" })); +TEST("require that sparse tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","0"},{"y","1"}}, 3) + .add({{"x","1"},{"y","0"}}, 5) + .add({{"x","1"},{"y","1"}}, 7), + "tensor(x[],y{})", + TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7))); } -TEST_F("test tensor mapper for SparseTensor", SparseFixture) -{ - testTensorMapper(f); +TEST("require that missing dimensions are added appropriately") { + TEST_DO(verify(TensorSpec("tensor(x{})") + .add({{"x","foo"}}, 42), + "tensor(x{},y{})", + TensorSpec("tensor(x{},y{})") + .add({{"x","foo"},{"y",""}}, 42))); + + TEST_DO(verify(TensorSpec("tensor(x[1])") + .add({{"x",0}}, 42), + "tensor(x[1],y[],z[2])", + TensorSpec("tensor(x[1],y[1],z[2])") + .add({{"x",0},{"y",0},{"z",0}}, 42) + .add({{"x",0},{"y",0},{"z",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(a{})") + .add({{"a","foo"}}, 42), + "tensor(a{},b[],c{},d[2])", + TensorSpec("tensor(a{},b[1],c{},d[2])") + .add({{"a","foo"},{"b",0},{"c",""},{"d",0}}, 42) + .add({{"a","foo"},{"b",0},{"c",""},{"d",1}}, 0))); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 27ca0060ecb..d076821fa04 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -67,7 +67,10 @@ public: TensorSpec & operator = (const TensorSpec &); ~TensorSpec(); TensorSpec &add(const Address &address, double value) { - _cells.emplace(address, value); + auto res = _cells.emplace(address, value); + if (!res.second) { + res.first->second.value += value; + } return *this; } const vespalib::string &type() const { return _type; } diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.cpp b/eval/src/vespa/eval/tensor/tensor_mapper.cpp index 939e6fdfa26..15dfb1db83a 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.cpp +++ b/eval/src/vespa/eval/tensor/tensor_mapper.cpp @@ -5,16 +5,21 @@ #include "tensor_visitor.h" #include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> #include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/eval/simple_tensor.h> #include "tensor_address_element_iterator.h" #include "default_tensor.h" +#include "wrapped_simple_tensor.h" using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; namespace vespalib { namespace tensor { namespace { +//----------------------------------------------------------------------------- + template <class TensorT> class SparseTensorMapper : public TensorVisitor { @@ -96,6 +101,8 @@ SparseTensorMapper<TensorT>::map(const Tensor &tensor, return mapper.build(); } +//----------------------------------------------------------------------------- + static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t BAD_ADDRESS = std::numeric_limits<uint32_t>::max(); @@ -113,7 +120,9 @@ uint32_t mapLabelToNumber(vespalib::stringref label) { return result; } -class DenseTensorTypeMapper : public TensorVisitor +//----------------------------------------------------------------------------- + +class TensorTypeMapper : public TensorVisitor { ValueType _type; std::vector<ValueType::Dimension> _dimensions; @@ -123,8 +132,8 @@ class DenseTensorTypeMapper : public TensorVisitor virtual void visit(const TensorAddress &address, double value) override; - DenseTensorTypeMapper(const ValueType &type); - ~DenseTensorTypeMapper(); + TensorTypeMapper(const ValueType &type); + ~TensorTypeMapper(); ValueType build(); public: @@ -132,16 +141,18 @@ public: }; bool -DenseTensorTypeMapper::addressOK(const TensorAddress &address) +TensorTypeMapper::addressOK(const TensorAddress &address) { TensorAddressElementIterator<TensorAddress> addressIterator(address); auto dimIterator = _dimensions.begin(); for (const auto &dimension : _type.dimensions()) { if (addressIterator.skipToDimension(dimension.name)) { - uint32_t label = mapLabelToNumber(addressIterator.label()); - if (label == BAD_LABEL || - (dimension.is_bound() && label >= dimIterator->size)) { - return false; + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label == BAD_LABEL || + (dimension.is_bound() && label >= dimIterator->size)) { + return false; + } } addressIterator.next(); } @@ -153,17 +164,19 @@ DenseTensorTypeMapper::addressOK(const TensorAddress &address) void -DenseTensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) +TensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) { TensorAddressElementIterator<TensorAddress> addressIterator(address); auto dimIterator = _dimensions.begin(); for (const auto &dimension : _type.dimensions()) { if (addressIterator.skipToDimension(dimension.name)) { - uint32_t label = mapLabelToNumber(addressIterator.label()); - if (label != BAD_LABEL && - !dimension.is_bound() && - label >= dimIterator->size) { - dimIterator->size = label + 1; + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label != BAD_LABEL && + !dimension.is_bound() && + label >= dimIterator->size) { + dimIterator->size = label + 1; + } } addressIterator.next(); } @@ -173,7 +186,7 @@ DenseTensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) } void -DenseTensorTypeMapper::visit(const TensorAddress &address, double value) +TensorTypeMapper::visit(const TensorAddress &address, double value) { (void) value; if (addressOK(address)) { @@ -181,37 +194,42 @@ DenseTensorTypeMapper::visit(const TensorAddress &address, double value) } } -DenseTensorTypeMapper::DenseTensorTypeMapper(const ValueType &type) +TensorTypeMapper::TensorTypeMapper(const ValueType &type) : _type(type), _dimensions(type.dimensions()) { for (auto &dimension : _dimensions) { - if (!dimension.is_bound()) - dimension.size = 1; + if (dimension.is_indexed()) { + if (!dimension.is_bound()) { + dimension.size = 1; + } + } } } -DenseTensorTypeMapper::~DenseTensorTypeMapper() +TensorTypeMapper::~TensorTypeMapper() { } ValueType -DenseTensorTypeMapper::build() +TensorTypeMapper::build() { return ValueType::tensor_type(std::move(_dimensions)); } ValueType -DenseTensorTypeMapper::map(const Tensor &tensor, const ValueType &type) +TensorTypeMapper::map(const Tensor &tensor, const ValueType &type) { - DenseTensorTypeMapper mapper(type); + TensorTypeMapper mapper(type); tensor.accept(mapper); return mapper.build(); } +//----------------------------------------------------------------------------- + class DenseTensorMapper : public TensorVisitor { - eval::ValueType _type; + ValueType _type; DenseTensor::Cells _cells; uint32_t mapAddressToIndex(const TensorAddress &address); @@ -283,12 +301,77 @@ std::unique_ptr<Tensor> DenseTensorMapper::map(const Tensor &tensor, const ValueType &type) { DenseTensorMapper mapper(type.is_abstract() ? - DenseTensorTypeMapper::map(tensor, type) : + TensorTypeMapper::map(tensor, type) : type); tensor.accept(mapper); return mapper.build(); } +//----------------------------------------------------------------------------- + +class WrappedTensorMapper : public TensorVisitor +{ + using Label = TensorSpec::Label; + + ValueType _type; + TensorSpec _spec; + + WrappedTensorMapper(const ValueType &type) + : _type(type), _spec(type.to_spec()) {} + ~WrappedTensorMapper() {} + + void visit(const TensorAddress &address, double value) override; + + std::unique_ptr<Tensor> build() { + auto tensor = eval::SimpleTensor::create(_spec); + return std::make_unique<WrappedSimpleTensor>(std::move(tensor)); + } + +public: + static std::unique_ptr<Tensor> + map(const Tensor &tensor, const ValueType &type); +}; + +void +WrappedTensorMapper::visit(const TensorAddress &address, double value) +{ + TensorSpec::Address addr; + TensorAddressElementIterator<TensorAddress> addressIterator(address); + for (const auto &dimension: _type.dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if ((label == BAD_LABEL) || (label >= dimension.size)) { + return; // bad address; ignore cell + } + addr.emplace(dimension.name, label); + } else { + addr.emplace(dimension.name, addressIterator.label()); + } + addressIterator.next(); + } else { + if (dimension.is_indexed()) { + addr.emplace(dimension.name, size_t(0)); + } else { + addr.emplace(dimension.name, vespalib::string()); + } + } + } + _spec.add(addr, value); +} + +std::unique_ptr<Tensor> +WrappedTensorMapper::map(const Tensor &tensor, const ValueType &type) +{ + WrappedTensorMapper mapper(type.is_abstract() ? + TensorTypeMapper::map(tensor, type) : + type); + tensor.accept(mapper); + return mapper.build(); +} + +//----------------------------------------------------------------------------- + } // namespace vespalib::tensor::<anonymous> TensorMapper::TensorMapper(const ValueType &type) @@ -316,6 +399,13 @@ TensorMapper::mapToDense(const Tensor &tensor, const ValueType &type) } std::unique_ptr<Tensor> +TensorMapper::mapToWrapped(const Tensor &tensor, const ValueType &type) +{ + assert(!type.dimensions().empty()); + return WrappedTensorMapper::map(tensor, type); +} + +std::unique_ptr<Tensor> TensorMapper::map(const Tensor &tensor) const { if (_type.is_sparse()) { @@ -323,7 +413,7 @@ TensorMapper::map(const Tensor &tensor) const } else if (_type.is_dense()) { return mapToDense(tensor, _type); } else { - return std::unique_ptr<Tensor>(); + return mapToWrapped(tensor, _type); } } diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.h b/eval/src/vespa/eval/tensor/tensor_mapper.h index 09d8a6208a3..02b0a30e185 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.h +++ b/eval/src/vespa/eval/tensor/tensor_mapper.h @@ -35,6 +35,9 @@ public: static std::unique_ptr<Tensor> mapToDense(const Tensor &tensor, const eval::ValueType &type); + static std::unique_ptr<Tensor> + mapToWrapped(const Tensor &tensor, const eval::ValueType &type); + std::unique_ptr<Tensor> map(const Tensor &tensor) const; }; diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index e6c22fcb1db..212960f9af6 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "wrapped_simple_tensor.h" +#include "tensor_address_builder.h" +#include "tensor_visitor.h" #include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib::tensor { @@ -36,6 +39,30 @@ WrappedSimpleTensor::sum() const return result; } +void +WrappedSimpleTensor::accept(TensorVisitor &visitor) const +{ + TensorAddressBuilder addr; + const auto &dimensions = _tensor.type().dimensions(); + for (const auto &cell: _tensor.cells()) { + addr.clear(); + for (size_t i = 0; i < dimensions.size(); ++i) { + if (dimensions[i].is_indexed()) { + addr.add(dimensions[i].name, make_string("%zu", cell.address[i].index)); + } else { + addr.add(dimensions[i].name, cell.address[i].name); + } + } + visitor.visit(addr.build(), cell.value); + } +} + +void +WrappedSimpleTensor::print(std::ostream &out) const +{ + out << toString(); +} + //----------------------------------------------------------------------------- Tensor::UP @@ -108,12 +135,6 @@ WrappedSimpleTensor::reduce(const eval::BinaryOperation &, const std::vector<ves return Tensor::UP(); } -void -WrappedSimpleTensor::print(std::ostream &) const -{ - abort(); -} - Tensor::UP WrappedSimpleTensor::clone() const { @@ -121,10 +142,4 @@ WrappedSimpleTensor::clone() const return Tensor::UP(); } -void -WrappedSimpleTensor::accept(TensorVisitor &) const -{ - abort(); -} - } // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h index 7ea82946e96..6de501438ed 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h @@ -33,6 +33,8 @@ public: vespalib::string toString() const override; eval::TensorSpec toSpec() const override; double sum() const override; + void accept(TensorVisitor &visitor) const override; + void print(std::ostream &out) const override; // functions below should not be used for this implementation Tensor::UP add(const Tensor &) const override; Tensor::UP subtract(const Tensor &) const override; @@ -44,9 +46,7 @@ public: Tensor::UP sum(const vespalib::string &) const override; Tensor::UP apply(const eval::BinaryOperation &, const Tensor &) const override; Tensor::UP reduce(const eval::BinaryOperation &, const std::vector<vespalib::string> &) const override; - void print(std::ostream &out) const override; Tensor::UP clone() const override; - void accept(TensorVisitor &visitor) const override; }; } // namespace vespalib::tensor |