summaryrefslogtreecommitdiffstats
path: root/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp')
-rw-r--r--eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp402
1 files changed, 163 insertions, 239 deletions
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
index b7aa988775d..2c113e814e9 100644
--- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
+++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
@@ -4,8 +4,7 @@
#include <vespa/eval/tensor/sparse/sparse_tensor.h>
#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h>
#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/eval/tensor/serialization/sparse_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -14,6 +13,7 @@
#include <vespa/eval/tensor/dense/dense_tensor_view.h>
using namespace vespalib::tensor;
+using vespalib::eval::TensorSpec;
using vespalib::nbostream;
using ExpBuffer = std::vector<uint8_t>;
@@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs)
}
-namespace vespalib::tensor {
-
-static bool operator==(const Tensor &lhs, const Tensor &rhs)
-{
- return lhs.equals(rhs);
-}
-
-}
-
-template <class BuilderType>
-void
-checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs)
-{
- (void) stream;
- (void) rhs;
-}
-
-template <>
-void
-checkDeserialize<DefaultTensor::builder>(nbostream &stream, const Tensor &rhs)
-{
- nbostream wrapStream(stream.peek(), stream.size());
- auto chk = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_EQUAL(0u, wrapStream.size());
- EXPECT_EQUAL(*chk, rhs);
-}
-
-template <typename BuilderType>
-struct Fixture
-{
- BuilderType _builder;
- Fixture() : _builder() {}
-
- Tensor::UP createTensor(const TensorCells &cells) {
- return TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
+//-----------------------------------------------------------------------------
+
+template <typename T>
+void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) {
+ nbostream input(&exp[0], exp.size());
+ std::vector<T> cells;
+ TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells);
+ ASSERT_EQUAL(cells.size(), spec.cells().size());
+ size_t i = 0;
+ for (const auto &cell: spec.cells()) {
+ EXPECT_EQUAL(cells[i++], cell.second.value);
}
+ ASSERT_EQUAL(i, cells.size());
+}
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
- Tensor::UP deserialize(nbostream &stream) {
- BuilderType builder;
- nbostream wrapStream(stream.peek(), stream.size());
- auto formatId = wrapStream.getInt1_4Bytes();
- ASSERT_EQUAL(formatId, 1u); // sparse format
- SparseBinaryFormat::deserialize(wrapStream, builder);
- EXPECT_TRUE(wrapStream.empty());
- auto ret = builder.build();
- checkDeserialize<BuilderType>(stream, *ret);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs,
- const TensorDimensions &rhsDimensions) {
- Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions));
- nbostream rhsStream;
- serialize(rhsStream, *rhsTensor);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
+void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
+ auto &engine = DefaultTensorEngine::ref();
+ auto value = engine.from_spec(spec);
+ auto value_spec = engine.to_spec(*value);
+ nbostream actual;
+ engine.encode(*value, actual);
+ EXPECT_EQUAL(exp, actual);
+ auto decoded = engine.decode(actual);
+ auto decoded_spec = engine.to_spec(*decoded);
+ EXPECT_EQUAL(0u, actual.size());
+ EXPECT_EQUAL(value_spec, decoded_spec);
+ if (value->type().is_dense()) {
+ TEST_DO(verify_cells_only<float>(exp, value_spec));
+ TEST_DO(verify_cells_only<double>(exp, value_spec));
}
-};
-
-using SparseFixture = Fixture<SparseTensorBuilder>;
-
+}
-template <typename FixtureType>
-void
-testTensorSerialization(FixtureType &f)
-{
- TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {}));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
- {}, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
- {}, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
- 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+//-----------------------------------------------------------------------------
+
+TEST("test tensor serialization for SparseTensor") {
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
+ TensorSpec("tensor(x{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
+ TensorSpec("tensor(x{},y{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+ TensorSpec("tensor(x{})")
+ .add({{"x", "1"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00 },
- { {{}, 3} }, { "x", "y"}));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
- 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
- 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"y","3"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+ 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00 },
- { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "2"}, {"y", "4"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
0x01, 0x01, 0x31, 0x00, 0x40, 0x08,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, {"x", "y"}));
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
}
-TEST_F("test tensor serialization for SparseTensor", SparseFixture)
-{
- testTensorSerialization(f);
+TEST("test tensor serialization for DenseTensor") {
+ TEST_DO(verify_serialized({0x02, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("double")));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1])")
+ .add({{"x", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2])")
+ .add({{"x", 1}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2],y[1])")
+ .add({{"x", 1}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x04,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[4])")
+ .add({{"x", 0}, {"y", 3}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
-struct DenseFixture
-{
- Tensor::UP createTensor(const DenseTensorCells &cells) {
- return TensorFactory::createDense(cells);
- }
-
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
-
- Tensor::UP deserialize(nbostream &stream) {
- nbostream wrapStream(stream.peek(), stream.size());
- auto ret = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_TRUE(wrapStream.size() == 0);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) {
- assertSerialized(exp, SerializeFormat::DOUBLE, rhs);
- }
- template <typename T>
- void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) {
- nbostream a(&exp[0], exp.size());
- std::vector<T> v;
- TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v);
- EXPECT_EQUAL(v.size(), rhs.cellsRef().size());
- for (size_t i(0); i < v.size(); i++) {
- EXPECT_EQUAL(v[i], rhs.cellsRef()[i]);
- }
- }
- void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) {
- Tensor::UP rhsTensor(createTensor(rhs));
- nbostream rhsStream;
- TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
-
- assertCellsOnly<float>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- assertCellsOnly<double>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- }
-};
-
-
-TEST_F("test tensor serialization for DenseTensor", DenseFixture) {
- TEST_DO(f.assertSerialized({0x02, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x04,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 3}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 2}, {"y", 4}}, 3}}));
-}
-
-TEST_F("test 'float' cells", DenseFixture) {
- TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x40, 0x00, 0x00 },
- SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} }));
+TEST("test 'float' cells") {
+ TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x40, 0x00, 0x00 },
+ TensorSpec("tensor<float>(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
TEST_MAIN() { TEST_RUN_ALL(); }