diff options
author | Håvard Pettersen <havardpe@oath.com> | 2019-05-27 12:24:06 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2019-05-31 11:49:25 +0000 |
commit | b74a5049bb1bc8b7dce4167f4b639dfb3761c411 (patch) | |
tree | 5b405602554eda05c541e2cce8f01c259f6fa67a /eval | |
parent | ad59a2b76aa92da991805758dfcb15ad7d9bc0ea (diff) |
remove tensor factory
Diffstat (limited to 'eval')
12 files changed, 170 insertions, 660 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index d15f1c37d2d..4468ac46f61 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -44,7 +44,6 @@ vespa_define_module( src/tests/tensor/tensor_performance src/tests/tensor/tensor_remove_operation src/tests/tensor/tensor_serialization - src/tests/tensor/tensor_slime_serialization src/tests/tensor/typed_cells src/tests/tensor/vector_from_doubles_function diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index b7aa988775d..2c113e814e9 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -4,8 +4,7 @@ #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> #include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <vespa/eval/tensor/tensor_factory.h> +#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/eval/tensor/serialization/sparse_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> @@ -14,6 +13,7 @@ #include <vespa/eval/tensor/dense/dense_tensor_view.h> using namespace vespalib::tensor; +using vespalib::eval::TensorSpec; using vespalib::nbostream; using ExpBuffer = std::vector<uint8_t>; @@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs) } -namespace vespalib::tensor { - -static bool operator==(const Tensor &lhs, const Tensor &rhs) -{ - return lhs.equals(rhs); -} - -} - -template <class BuilderType> -void -checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs) -{ - (void) stream; - (void) rhs; -} - -template <> -void -checkDeserialize<DefaultTensor::builder>(nbostream &stream, const Tensor &rhs) -{ - nbostream wrapStream(stream.peek(), stream.size()); - auto chk = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_EQUAL(0u, wrapStream.size()); - EXPECT_EQUAL(*chk, rhs); -} - -template <typename BuilderType> -struct Fixture -{ - BuilderType _builder; - Fixture() : _builder() {} - - Tensor::UP createTensor(const TensorCells &cells) { - return TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); +//----------------------------------------------------------------------------- + +template <typename T> +void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { + nbostream input(&exp[0], exp.size()); + std::vector<T> cells; + TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells); + ASSERT_EQUAL(cells.size(), spec.cells().size()); + size_t i = 0; + for (const auto &cell: spec.cells()) { + EXPECT_EQUAL(cells[i++], cell.second.value); } + ASSERT_EQUAL(i, cells.size()); +} - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - Tensor::UP deserialize(nbostream &stream) { - BuilderType builder; - nbostream wrapStream(stream.peek(), stream.size()); - auto formatId = wrapStream.getInt1_4Bytes(); - ASSERT_EQUAL(formatId, 1u); // sparse format - SparseBinaryFormat::deserialize(wrapStream, builder); - EXPECT_TRUE(wrapStream.empty()); - auto ret = builder.build(); - checkDeserialize<BuilderType>(stream, *ret); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs, - const TensorDimensions &rhsDimensions) { - Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); - nbostream rhsStream; - serialize(rhsStream, *rhsTensor); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); +void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { + auto &engine = DefaultTensorEngine::ref(); + auto value = engine.from_spec(spec); + auto value_spec = engine.to_spec(*value); + nbostream actual; + engine.encode(*value, actual); + EXPECT_EQUAL(exp, actual); + auto decoded = engine.decode(actual); + auto decoded_spec = engine.to_spec(*decoded); + EXPECT_EQUAL(0u, actual.size()); + EXPECT_EQUAL(value_spec, decoded_spec); + if (value->type().is_dense()) { + TEST_DO(verify_cells_only<float>(exp, value_spec)); + TEST_DO(verify_cells_only<double>(exp, value_spec)); } -}; - -using SparseFixture = Fixture<SparseTensorBuilder>; - +} -template <typename FixtureType> -void -testTensorSerialization(FixtureType &f) -{ - TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {})); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, - {}, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, - {}, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, +//----------------------------------------------------------------------------- + +TEST("test tensor serialization for SparseTensor") { + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, + TensorSpec("tensor(x{})"))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, + TensorSpec("tensor(x{},y{})"))); + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + TensorSpec("tensor(x{})") + .add({{"x", "1"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, - 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, - 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"y","3"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, + 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, + TensorSpec("tensor(x{},y{})") + .add({{"x", "2"}, {"y", "4"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, {"x", "y"})); + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); } -TEST_F("test tensor serialization for SparseTensor", SparseFixture) -{ - testTensorSerialization(f); +TEST("test tensor serialization for DenseTensor") { + TEST_DO(verify_serialized({0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("double"))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1])") + .add({{"x", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2])") + .add({{"x", 1}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2],y[1])") + .add({{"x", 1}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[4])") + .add({{"x", 0}, {"y", 3}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - -struct DenseFixture -{ - Tensor::UP createTensor(const DenseTensorCells &cells) { - return TensorFactory::createDense(cells); - } - - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - - Tensor::UP deserialize(nbostream &stream) { - nbostream wrapStream(stream.peek(), stream.size()); - auto ret = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_TRUE(wrapStream.size() == 0); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) { - assertSerialized(exp, SerializeFormat::DOUBLE, rhs); - } - template <typename T> - void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) { - nbostream a(&exp[0], exp.size()); - std::vector<T> v; - TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v); - EXPECT_EQUAL(v.size(), rhs.cellsRef().size()); - for (size_t i(0); i < v.size(); i++) { - EXPECT_EQUAL(v[i], rhs.cellsRef()[i]); - } - } - void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) { - Tensor::UP rhsTensor(createTensor(rhs)); - nbostream rhsStream; - TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); - - assertCellsOnly<float>(exp, dynamic_cast<const DenseTensorView &>(*rhs2)); - assertCellsOnly<double>(exp, dynamic_cast<const DenseTensorView &>(*rhs2)); - } -}; - - -TEST_F("test tensor serialization for DenseTensor", DenseFixture) { - TEST_DO(f.assertSerialized({0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x04, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 3}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 2}, {"y", 4}}, 3}})); -} - -TEST_F("test 'float' cells", DenseFixture) { - TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x40, 0x00, 0x00 }, - SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} })); +TEST("test 'float' cells") { + TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x40, 0x00, 0x00 }, + TensorSpec("tensor<float>(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore b/eval/src/tests/tensor/tensor_slime_serialization/.gitignore deleted file mode 100644 index 9cb3b664d58..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_slime_serialization_test_app diff --git a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt b/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt deleted file mode 100644 index 9a9b8b7a436..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_slime_serialization_test_app TEST - SOURCES - tensor_slime_serialization_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_tensor_slime_serialization_test_app COMMAND eval_tensor_slime_serialization_test_app) diff --git a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp deleted file mode 100644 index e6e6c7de686..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <vespa/eval/tensor/tensor_factory.h> -#include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/eval/tensor/serialization/slime_binary_format.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <iostream> - -using namespace vespalib::tensor; - -template <typename BuilderType> -struct Fixture -{ - BuilderType _builder; - Fixture() : _builder() {} - - Tensor::UP createTensor(const TensorCells &cells) { - return vespalib::tensor::TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); - } - - static inline uint32_t getTensorTypeId(); - - void assertSerialized(const vespalib::string &exp, const TensorCells &rhs, - const TensorDimensions &rhsDimensions) { - Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); - auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::Memory memory_exp(exp); - vespalib::Slime expSlime; - size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); - EXPECT_TRUE(used > 0); - EXPECT_EQUAL(expSlime, *slime); - } -}; - -template <> -uint32_t -Fixture<SparseTensorBuilder>::getTensorTypeId() { return 2u; } - - -using SparseFixture = Fixture<SparseTensorBuilder>; - - -namespace { -vespalib::string twoCellsJson[3] = -{ - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { y:'3'}, value: 4.0 }," - "{ address: { x:'1'}, value: 3.0 }" - "] }", - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }," - "{ address: { y:'3'}, value: 4.0 }" - "] }", - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }," - "{ address: { y:'3'}, value: 4.0 }" - "] }", -}; -} - - -template <typename FixtureType> -void -testTensorSlimeSerialization(FixtureType &f) -{ - TEST_DO(f.assertSerialized("{ dimensions: [], cells: [] }", {}, {})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [] }", - {}, { "x" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [] }", - {}, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ]," - "cells: [" - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, { "x" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { }, value: 3.0 }" - "] }", - { {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { y: '3' }, value: 3.0 }" - "] }", - { {{{"y","3"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'2', y:'4'}, value: 3.0 }" - "] }", - { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, {"x", "y"})); - TEST_DO(f.assertSerialized(twoCellsJson[FixtureType::getTensorTypeId()], - { {{{"x","1"}}, 3}, {{{"y","3"}}, 4} }, - {"x", "y"})); -} - -TEST_F("test tensor slime serialization for SparseTensor", SparseFixture) -{ - testTensorSlimeSerialization(f); -} - - -struct DenseFixture -{ - DenseFixture() {} - - Tensor::UP createTensor(const DenseTensorCells &cells) { - return vespalib::tensor::TensorFactory::createDense(cells); - } - - void assertSerialized(const vespalib::string &exp, - const DenseTensorCells &rhs) { - Tensor::UP rhsTensor(createTensor(rhs)); - auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::Memory memory_exp(exp); - vespalib::Slime expSlime; - size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); - EXPECT_TRUE(used > 0); - EXPECT_EQUAL(expSlime, *slime); - } -}; - - -TEST_F("test tensor slime serialization for DenseTensor", DenseFixture) -{ - TEST_DO(f.assertSerialized("{ dimensions: [], cells: [" - "{ address: { }, value: 0.0 }" - "] }", {})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [" - "{ address: { x: '0' }, value: 0.0 }" - "] }", - { {{{"x",0}}, 0} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }" - "] }", - { {{{"x",0},{"y",0}}, 0} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ]," - "cells: [" - "{ address: { x: '0' }, value: 0.0 }," - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x",1}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 3.0 }" - "] }", - { {{{"x",0},{"y",0}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }," - "{ address: { x: '1', y: '0' }, value: 3.0 }" - "] }", - { {{{"x",1},{"y", 0}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }," - "{ address: { x: '0', y: '1' }, value: 0.0 }," - "{ address: { x: '0', y: '2' }, value: 0.0 }," - "{ address: { x: '0', y: '3' }, value: 3.0 }" - "] }", - { {{{"x",0},{"y",3}}, 3} })); -} - - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index ca4d275ea44..4e9940d3c0a 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(eval_tensor OBJECT tensor.cpp tensor_address.cpp tensor_apply.cpp - tensor_factory.cpp tensor_mapper.cpp wrapped_simple_tensor.cpp ) diff --git a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt index cd3cd455e55..fc9ac64ea68 100644 --- a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt @@ -3,6 +3,5 @@ vespa_add_library(eval_tensor_serialization OBJECT SOURCES sparse_binary_format.cpp dense_binary_format.cpp - slime_binary_format.cpp typed_binary_format.cpp ) diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp deleted file mode 100644 index ece3c2e4a07..00000000000 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "slime_binary_format.h" -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/tensor_builder.h> -#include <vespa/eval/tensor/tensor_visitor.h> -#include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/vespalib/data/slime/cursor.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/memory.h> - -namespace vespalib::tensor { - - -using slime::Inserter; -using slime::SlimeInserter; -using slime::Cursor; -using slime::ObjectInserter; - -namespace { - -Memory memory_address("address"); -Memory memory_cells("cells"); -Memory memory_dimensions("dimensions"); -Memory memory_value("value"); - -void writeTensorAddress(Cursor &cursor, const TensorAddress &value) { - ObjectInserter addressInserter(cursor, memory_address); - Cursor &addressCursor = addressInserter.insertObject(); - for (const auto &elem : value.elements()) { - Memory dimension(elem.dimension()); - Memory label(elem.label()); - addressCursor.setString(dimension, label); - } -} - -} - -class SlimeBinaryFormatSerializer : public TensorVisitor -{ - Cursor &_tensor; // cursor for whole tensor - Cursor &_dimensions; // cursor for dimensions array - Cursor &_cells; // cursor for cells array -public: - SlimeBinaryFormatSerializer(Inserter &inserter); - virtual ~SlimeBinaryFormatSerializer() override; - virtual void visit(const TensorAddress &address, double value) override; - void serialize(const Tensor &tensor); -}; - -SlimeBinaryFormatSerializer::SlimeBinaryFormatSerializer(Inserter &inserter) - : _tensor(inserter.insertObject()), - _dimensions(_tensor.setArray(memory_dimensions)), - _cells(_tensor.setArray(memory_cells)) -{ -} - - -SlimeBinaryFormatSerializer::~SlimeBinaryFormatSerializer() = default; - -void -SlimeBinaryFormatSerializer::visit(const TensorAddress &address, double value) -{ - Cursor &cellCursor = _cells.addObject(); - writeTensorAddress(cellCursor, address); - cellCursor.setDouble(memory_value, value); -} - - -void -SlimeBinaryFormatSerializer::serialize(const Tensor &tensor) -{ - eval::ValueType type(tensor.type()); - for (const auto & dimension : type.dimensions()) { - _dimensions.addString(Memory(dimension.name)); - } - tensor.accept(*this); -} - - -void -SlimeBinaryFormat::serialize(Inserter &inserter, const Tensor &tensor) -{ - SlimeBinaryFormatSerializer serializer(inserter); - serializer.serialize(tensor); -} - - -std::unique_ptr<Slime> -SlimeBinaryFormat::serialize(const Tensor &tensor) -{ - auto slime = std::make_unique<Slime>(); - SlimeInserter inserter(*slime); - serialize(inserter, tensor); - return slime; -} - - -} diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h deleted file mode 100644 index c9e9ff2c3e9..00000000000 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace vespalib { class Slime; } - -namespace vespalib::slime { struct Inserter; } - -namespace vespalib::tensor { - -class Tensor; -class TensorBuilder; - -/** - * Class for serializing a tensor into a slime object. - */ -class SlimeBinaryFormat -{ -public: - static void serialize(slime::Inserter &inserter, const Tensor &tensor); - static std::unique_ptr<Slime> serialize(const Tensor &tensor); -}; - -} diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index 4ca037e82a4..7be4db46372 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -25,9 +25,9 @@ namespace { constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u; constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u; constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u; -constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; //Future +constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; constexpr uint32_t DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6u; -constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; //Future +constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; constexpr uint32_t DOUBLE_VALUE_TYPE = 0; constexpr uint32_t FLOAT_VALUE_TYPE = 1; @@ -91,10 +91,11 @@ TypedBinaryFormat::deserialize(nbostream &stream) if (formatId == DENSE_BINARY_FORMAT_TYPE) { return DenseBinaryFormat(SerializeFormat::DOUBLE).deserialize(stream); } - if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) { - return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserialize(stream); - } - if (formatId == MIXED_BINARY_FORMAT_TYPE) { + if ((formatId == SPARSE_BINARY_FORMAT_WITH_CELLTYPE) || + (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) || + (formatId == MIXED_BINARY_FORMAT_TYPE) || + (formatId == MIXED_BINARY_FORMAT_WITH_CELLTYPE)) + { stream.adjustReadPos(read_pos - stream.rp()); return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::decode(stream)); } diff --git a/eval/src/vespa/eval/tensor/tensor_factory.cpp b/eval/src/vespa/eval/tensor/tensor_factory.cpp deleted file mode 100644 index 0b7fa3b9c2e..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_factory.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor.h" -#include "tensor_factory.h" -#include "tensor_builder.h" -#include <vespa/eval/tensor/dense/dense_tensor_builder.h> - -namespace vespalib::tensor { - -std::unique_ptr<Tensor> -TensorFactory::create(const TensorCells &cells, TensorBuilder &builder) { - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - builder.define_dimension(dimension); - } - } - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - const auto &label = addressElem.second; - builder.add_label(builder.define_dimension(dimension), label); - } - builder.add_cell(cell.second); - } - return builder.build(); -} - - -std::unique_ptr<Tensor> -TensorFactory::create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder) { - for (const auto &dimension : dimensions) { - builder.define_dimension(dimension); - } - return create(cells, builder); -} - - -std::unique_ptr<Tensor> -TensorFactory::createDense(const DenseTensorCells &cells) -{ - std::map<std::string, size_t> dimensionSizes; - DenseTensorBuilder builder; - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - dimensionSizes[addressElem.first] = std::max(dimensionSizes[addressElem.first], (addressElem.second + 1)); - } - } - std::map<std::string, typename DenseTensorBuilder::Dimension> dimensionEnums; - for (const auto &dimensionElem : dimensionSizes) { - dimensionEnums[dimensionElem.first] = builder.defineDimension(dimensionElem.first, dimensionElem.second); - } - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - size_t label = addressElem.second; - builder.addLabel(dimensionEnums[dimension], label); - } - builder.addCell(cell.second); - } - return builder.build(); -} - - -} diff --git a/eval/src/vespa/eval/tensor/tensor_factory.h b/eval/src/vespa/eval/tensor/tensor_factory.h deleted file mode 100644 index 5364c28c8ff..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_factory.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "types.h" - -namespace vespalib::tensor { - - -class Tensor; -class TensorBuilder; - -/** - * A factory for creating tensors based on stl structures (TensorCells and - * TensorDimensions) in unit tests. - */ -class TensorFactory { -public: - static std::unique_ptr<Tensor> - create(const TensorCells &cells, TensorBuilder &builder); - static std::unique_ptr<Tensor> - create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder); - static std::unique_ptr<Tensor> - createDense(const DenseTensorCells &cells); -}; - -} |