diff options
Diffstat (limited to 'eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp')
-rw-r--r-- | eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp | 402 |
1 files changed, 163 insertions, 239 deletions
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index b7aa988775d..2c113e814e9 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -4,8 +4,7 @@ #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> #include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <vespa/eval/tensor/tensor_factory.h> +#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/eval/tensor/serialization/sparse_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> @@ -14,6 +13,7 @@ #include <vespa/eval/tensor/dense/dense_tensor_view.h> using namespace vespalib::tensor; +using vespalib::eval::TensorSpec; using vespalib::nbostream; using ExpBuffer = std::vector<uint8_t>; @@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs) } -namespace vespalib::tensor { - -static bool operator==(const Tensor &lhs, const Tensor &rhs) -{ - return lhs.equals(rhs); -} - -} - -template <class BuilderType> -void -checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs) -{ - (void) stream; - (void) rhs; -} - -template <> -void -checkDeserialize<DefaultTensor::builder>(nbostream &stream, const Tensor &rhs) -{ - nbostream wrapStream(stream.peek(), stream.size()); - auto chk = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_EQUAL(0u, wrapStream.size()); - EXPECT_EQUAL(*chk, rhs); -} - -template <typename BuilderType> -struct Fixture -{ - BuilderType _builder; - Fixture() : _builder() {} - - Tensor::UP createTensor(const TensorCells &cells) { - return TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); +//----------------------------------------------------------------------------- + +template <typename T> +void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { + nbostream input(&exp[0], exp.size()); + std::vector<T> cells; + TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells); + ASSERT_EQUAL(cells.size(), spec.cells().size()); + size_t i = 0; + for (const auto &cell: spec.cells()) { + EXPECT_EQUAL(cells[i++], cell.second.value); } + ASSERT_EQUAL(i, cells.size()); +} - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - Tensor::UP deserialize(nbostream &stream) { - BuilderType builder; - nbostream wrapStream(stream.peek(), stream.size()); - auto formatId = wrapStream.getInt1_4Bytes(); - ASSERT_EQUAL(formatId, 1u); // sparse format - SparseBinaryFormat::deserialize(wrapStream, builder); - EXPECT_TRUE(wrapStream.empty()); - auto ret = builder.build(); - checkDeserialize<BuilderType>(stream, *ret); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs, - const TensorDimensions &rhsDimensions) { - Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); - nbostream rhsStream; - serialize(rhsStream, *rhsTensor); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); +void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { + auto &engine = DefaultTensorEngine::ref(); + auto value = engine.from_spec(spec); + auto value_spec = engine.to_spec(*value); + nbostream actual; + engine.encode(*value, actual); + EXPECT_EQUAL(exp, actual); + auto decoded = engine.decode(actual); + auto decoded_spec = engine.to_spec(*decoded); + EXPECT_EQUAL(0u, actual.size()); + EXPECT_EQUAL(value_spec, decoded_spec); + if (value->type().is_dense()) { + TEST_DO(verify_cells_only<float>(exp, value_spec)); + TEST_DO(verify_cells_only<double>(exp, value_spec)); } -}; - -using SparseFixture = Fixture<SparseTensorBuilder>; - +} -template <typename FixtureType> -void -testTensorSerialization(FixtureType &f) -{ - TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {})); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, - {}, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, - {}, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, +//----------------------------------------------------------------------------- + +TEST("test tensor serialization for SparseTensor") { + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, + TensorSpec("tensor(x{})"))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, + TensorSpec("tensor(x{},y{})"))); + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + TensorSpec("tensor(x{})") + .add({{"x", "1"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, - 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, - 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"y","3"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, + 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, + TensorSpec("tensor(x{},y{})") + .add({{"x", "2"}, {"y", "4"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, {"x", "y"})); + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); } -TEST_F("test tensor serialization for SparseTensor", SparseFixture) -{ - testTensorSerialization(f); +TEST("test tensor serialization for DenseTensor") { + TEST_DO(verify_serialized({0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("double"))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1])") + .add({{"x", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2])") + .add({{"x", 1}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2],y[1])") + .add({{"x", 1}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[4])") + .add({{"x", 0}, {"y", 3}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - -struct DenseFixture -{ - Tensor::UP createTensor(const DenseTensorCells &cells) { - return TensorFactory::createDense(cells); - } - - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - - Tensor::UP deserialize(nbostream &stream) { - nbostream wrapStream(stream.peek(), stream.size()); - auto ret = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_TRUE(wrapStream.size() == 0); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) { - assertSerialized(exp, SerializeFormat::DOUBLE, rhs); - } - template <typename T> - void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) { - nbostream a(&exp[0], exp.size()); - std::vector<T> v; - TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v); - EXPECT_EQUAL(v.size(), rhs.cellsRef().size()); - for (size_t i(0); i < v.size(); i++) { - EXPECT_EQUAL(v[i], rhs.cellsRef()[i]); - } - } - void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) { - Tensor::UP rhsTensor(createTensor(rhs)); - nbostream rhsStream; - TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); - - assertCellsOnly<float>(exp, dynamic_cast<const DenseTensorView &>(*rhs2)); - assertCellsOnly<double>(exp, dynamic_cast<const DenseTensorView &>(*rhs2)); - } -}; - - -TEST_F("test tensor serialization for DenseTensor", DenseFixture) { - TEST_DO(f.assertSerialized({0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x04, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 3}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 2}, {"y", 4}}, 3}})); -} - -TEST_F("test 'float' cells", DenseFixture) { - TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x40, 0x00, 0x00 }, - SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} })); +TEST("test 'float' cells") { + TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x40, 0x00, 0x00 }, + TensorSpec("tensor<float>(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - TEST_MAIN() { TEST_RUN_ALL(); } |