summaryrefslogtreecommitdiffstats
path: root/eval
diff options
context:
space:
mode:
authorHåvard Pettersen <havardpe@oath.com>2019-05-27 12:24:06 +0000
committerHåvard Pettersen <havardpe@oath.com>2019-05-31 11:49:25 +0000
commitb74a5049bb1bc8b7dce4167f4b639dfb3761c411 (patch)
tree5b405602554eda05c541e2cce8f01c259f6fa67a /eval
parentad59a2b76aa92da991805758dfcb15ad7d9bc0ea (diff)
remove tensor factory
Diffstat (limited to 'eval')
-rw-r--r--eval/CMakeLists.txt1
-rw-r--r--eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp402
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/.gitignore1
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt8
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp185
-rw-r--r--eval/src/vespa/eval/tensor/CMakeLists.txt1
-rw-r--r--eval/src/vespa/eval/tensor/serialization/CMakeLists.txt1
-rw-r--r--eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp100
-rw-r--r--eval/src/vespa/eval/tensor/serialization/slime_binary_format.h26
-rw-r--r--eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp13
-rw-r--r--eval/src/vespa/eval/tensor/tensor_factory.cpp65
-rw-r--r--eval/src/vespa/eval/tensor/tensor_factory.h27
12 files changed, 170 insertions, 660 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index d15f1c37d2d..4468ac46f61 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -44,7 +44,6 @@ vespa_define_module(
src/tests/tensor/tensor_performance
src/tests/tensor/tensor_remove_operation
src/tests/tensor/tensor_serialization
- src/tests/tensor/tensor_slime_serialization
src/tests/tensor/typed_cells
src/tests/tensor/vector_from_doubles_function
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
index b7aa988775d..2c113e814e9 100644
--- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
+++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
@@ -4,8 +4,7 @@
#include <vespa/eval/tensor/sparse/sparse_tensor.h>
#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h>
#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/eval/tensor/serialization/sparse_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -14,6 +13,7 @@
#include <vespa/eval/tensor/dense/dense_tensor_view.h>
using namespace vespalib::tensor;
+using vespalib::eval::TensorSpec;
using vespalib::nbostream;
using ExpBuffer = std::vector<uint8_t>;
@@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs)
}
-namespace vespalib::tensor {
-
-static bool operator==(const Tensor &lhs, const Tensor &rhs)
-{
- return lhs.equals(rhs);
-}
-
-}
-
-template <class BuilderType>
-void
-checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs)
-{
- (void) stream;
- (void) rhs;
-}
-
-template <>
-void
-checkDeserialize<DefaultTensor::builder>(nbostream &stream, const Tensor &rhs)
-{
- nbostream wrapStream(stream.peek(), stream.size());
- auto chk = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_EQUAL(0u, wrapStream.size());
- EXPECT_EQUAL(*chk, rhs);
-}
-
-template <typename BuilderType>
-struct Fixture
-{
- BuilderType _builder;
- Fixture() : _builder() {}
-
- Tensor::UP createTensor(const TensorCells &cells) {
- return TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
+//-----------------------------------------------------------------------------
+
+template <typename T>
+void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) {
+ nbostream input(&exp[0], exp.size());
+ std::vector<T> cells;
+ TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells);
+ ASSERT_EQUAL(cells.size(), spec.cells().size());
+ size_t i = 0;
+ for (const auto &cell: spec.cells()) {
+ EXPECT_EQUAL(cells[i++], cell.second.value);
}
+ ASSERT_EQUAL(i, cells.size());
+}
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
- Tensor::UP deserialize(nbostream &stream) {
- BuilderType builder;
- nbostream wrapStream(stream.peek(), stream.size());
- auto formatId = wrapStream.getInt1_4Bytes();
- ASSERT_EQUAL(formatId, 1u); // sparse format
- SparseBinaryFormat::deserialize(wrapStream, builder);
- EXPECT_TRUE(wrapStream.empty());
- auto ret = builder.build();
- checkDeserialize<BuilderType>(stream, *ret);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs,
- const TensorDimensions &rhsDimensions) {
- Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions));
- nbostream rhsStream;
- serialize(rhsStream, *rhsTensor);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
+void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
+ auto &engine = DefaultTensorEngine::ref();
+ auto value = engine.from_spec(spec);
+ auto value_spec = engine.to_spec(*value);
+ nbostream actual;
+ engine.encode(*value, actual);
+ EXPECT_EQUAL(exp, actual);
+ auto decoded = engine.decode(actual);
+ auto decoded_spec = engine.to_spec(*decoded);
+ EXPECT_EQUAL(0u, actual.size());
+ EXPECT_EQUAL(value_spec, decoded_spec);
+ if (value->type().is_dense()) {
+ TEST_DO(verify_cells_only<float>(exp, value_spec));
+ TEST_DO(verify_cells_only<double>(exp, value_spec));
}
-};
-
-using SparseFixture = Fixture<SparseTensorBuilder>;
-
+}
-template <typename FixtureType>
-void
-testTensorSerialization(FixtureType &f)
-{
- TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {}));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
- {}, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
- {}, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
- 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+//-----------------------------------------------------------------------------
+
+TEST("test tensor serialization for SparseTensor") {
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
+ TensorSpec("tensor(x{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
+ TensorSpec("tensor(x{},y{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+ TensorSpec("tensor(x{})")
+ .add({{"x", "1"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00 },
- { {{}, 3} }, { "x", "y"}));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
- 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
- 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"y","3"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+ 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00 },
- { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "2"}, {"y", "4"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
0x01, 0x01, 0x31, 0x00, 0x40, 0x08,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, {"x", "y"}));
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
}
-TEST_F("test tensor serialization for SparseTensor", SparseFixture)
-{
- testTensorSerialization(f);
+TEST("test tensor serialization for DenseTensor") {
+ TEST_DO(verify_serialized({0x02, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("double")));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1])")
+ .add({{"x", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2])")
+ .add({{"x", 1}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2],y[1])")
+ .add({{"x", 1}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x04,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[4])")
+ .add({{"x", 0}, {"y", 3}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
-struct DenseFixture
-{
- Tensor::UP createTensor(const DenseTensorCells &cells) {
- return TensorFactory::createDense(cells);
- }
-
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
-
- Tensor::UP deserialize(nbostream &stream) {
- nbostream wrapStream(stream.peek(), stream.size());
- auto ret = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_TRUE(wrapStream.size() == 0);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) {
- assertSerialized(exp, SerializeFormat::DOUBLE, rhs);
- }
- template <typename T>
- void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) {
- nbostream a(&exp[0], exp.size());
- std::vector<T> v;
- TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v);
- EXPECT_EQUAL(v.size(), rhs.cellsRef().size());
- for (size_t i(0); i < v.size(); i++) {
- EXPECT_EQUAL(v[i], rhs.cellsRef()[i]);
- }
- }
- void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) {
- Tensor::UP rhsTensor(createTensor(rhs));
- nbostream rhsStream;
- TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
-
- assertCellsOnly<float>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- assertCellsOnly<double>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- }
-};
-
-
-TEST_F("test tensor serialization for DenseTensor", DenseFixture) {
- TEST_DO(f.assertSerialized({0x02, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x04,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 3}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 2}, {"y", 4}}, 3}}));
-}
-
-TEST_F("test 'float' cells", DenseFixture) {
- TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x40, 0x00, 0x00 },
- SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} }));
+TEST("test 'float' cells") {
+ TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x40, 0x00, 0x00 },
+ TensorSpec("tensor<float>(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore b/eval/src/tests/tensor/tensor_slime_serialization/.gitignore
deleted file mode 100644
index 9cb3b664d58..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-vespalib_tensor_slime_serialization_test_app
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt b/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt
deleted file mode 100644
index 9a9b8b7a436..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(eval_tensor_slime_serialization_test_app TEST
- SOURCES
- tensor_slime_serialization_test.cpp
- DEPENDS
- vespaeval
-)
-vespa_add_test(NAME eval_tensor_slime_serialization_test_app COMMAND eval_tensor_slime_serialization_test_app)
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp
deleted file mode 100644
index e6e6c7de686..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp
+++ /dev/null
@@ -1,185 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/eval/tensor/sparse/sparse_tensor.h>
-#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h>
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
-#include <vespa/eval/tensor/serialization/typed_binary_format.h>
-#include <vespa/eval/tensor/serialization/slime_binary_format.h>
-#include <vespa/vespalib/data/slime/slime.h>
-#include <iostream>
-
-using namespace vespalib::tensor;
-
-template <typename BuilderType>
-struct Fixture
-{
- BuilderType _builder;
- Fixture() : _builder() {}
-
- Tensor::UP createTensor(const TensorCells &cells) {
- return vespalib::tensor::TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
- }
-
- static inline uint32_t getTensorTypeId();
-
- void assertSerialized(const vespalib::string &exp, const TensorCells &rhs,
- const TensorDimensions &rhsDimensions) {
- Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions));
- auto slime = SlimeBinaryFormat::serialize(*rhsTensor);
- vespalib::Memory memory_exp(exp);
- vespalib::Slime expSlime;
- size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime);
- EXPECT_TRUE(used > 0);
- EXPECT_EQUAL(expSlime, *slime);
- }
-};
-
-template <>
-uint32_t
-Fixture<SparseTensorBuilder>::getTensorTypeId() { return 2u; }
-
-
-using SparseFixture = Fixture<SparseTensorBuilder>;
-
-
-namespace {
-vespalib::string twoCellsJson[3] =
-{
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { y:'3'}, value: 4.0 },"
- "{ address: { x:'1'}, value: 3.0 }"
- "] }",
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 },"
- "{ address: { y:'3'}, value: 4.0 }"
- "] }",
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 },"
- "{ address: { y:'3'}, value: 4.0 }"
- "] }",
-};
-}
-
-
-template <typename FixtureType>
-void
-testTensorSlimeSerialization(FixtureType &f)
-{
- TEST_DO(f.assertSerialized("{ dimensions: [], cells: [] }", {}, {}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [] }",
- {}, { "x" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [] }",
- {}, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ],"
- "cells: ["
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, { "x" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { }, value: 3.0 }"
- "] }",
- { {{}, 3} }, { "x", "y"}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { y: '3' }, value: 3.0 }"
- "] }",
- { {{{"y","3"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'2', y:'4'}, value: 3.0 }"
- "] }",
- { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, {"x", "y"}));
- TEST_DO(f.assertSerialized(twoCellsJson[FixtureType::getTensorTypeId()],
- { {{{"x","1"}}, 3}, {{{"y","3"}}, 4} },
- {"x", "y"}));
-}
-
-TEST_F("test tensor slime serialization for SparseTensor", SparseFixture)
-{
- testTensorSlimeSerialization(f);
-}
-
-
-struct DenseFixture
-{
- DenseFixture() {}
-
- Tensor::UP createTensor(const DenseTensorCells &cells) {
- return vespalib::tensor::TensorFactory::createDense(cells);
- }
-
- void assertSerialized(const vespalib::string &exp,
- const DenseTensorCells &rhs) {
- Tensor::UP rhsTensor(createTensor(rhs));
- auto slime = SlimeBinaryFormat::serialize(*rhsTensor);
- vespalib::Memory memory_exp(exp);
- vespalib::Slime expSlime;
- size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime);
- EXPECT_TRUE(used > 0);
- EXPECT_EQUAL(expSlime, *slime);
- }
-};
-
-
-TEST_F("test tensor slime serialization for DenseTensor", DenseFixture)
-{
- TEST_DO(f.assertSerialized("{ dimensions: [], cells: ["
- "{ address: { }, value: 0.0 }"
- "] }", {}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: ["
- "{ address: { x: '0' }, value: 0.0 }"
- "] }",
- { {{{"x",0}}, 0} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 }"
- "] }",
- { {{{"x",0},{"y",0}}, 0} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ],"
- "cells: ["
- "{ address: { x: '0' }, value: 0.0 },"
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x",1}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 3.0 }"
- "] }",
- { {{{"x",0},{"y",0}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 },"
- "{ address: { x: '1', y: '0' }, value: 3.0 }"
- "] }",
- { {{{"x",1},{"y", 0}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 },"
- "{ address: { x: '0', y: '1' }, value: 0.0 },"
- "{ address: { x: '0', y: '2' }, value: 0.0 },"
- "{ address: { x: '0', y: '3' }, value: 3.0 }"
- "] }",
- { {{{"x",0},{"y",3}}, 3} }));
-}
-
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt
index ca4d275ea44..4e9940d3c0a 100644
--- a/eval/src/vespa/eval/tensor/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_library(eval_tensor OBJECT
tensor.cpp
tensor_address.cpp
tensor_apply.cpp
- tensor_factory.cpp
tensor_mapper.cpp
wrapped_simple_tensor.cpp
)
diff --git a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
index cd3cd455e55..fc9ac64ea68 100644
--- a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
@@ -3,6 +3,5 @@ vespa_add_library(eval_tensor_serialization OBJECT
SOURCES
sparse_binary_format.cpp
dense_binary_format.cpp
- slime_binary_format.cpp
typed_binary_format.cpp
)
diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp
deleted file mode 100644
index ece3c2e4a07..00000000000
--- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "slime_binary_format.h"
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/tensor_builder.h>
-#include <vespa/eval/tensor/tensor_visitor.h>
-#include <vespa/vespalib/data/slime/inserter.h>
-#include <vespa/vespalib/data/slime/cursor.h>
-#include <vespa/vespalib/data/slime/slime.h>
-#include <vespa/vespalib/data/memory.h>
-
-namespace vespalib::tensor {
-
-
-using slime::Inserter;
-using slime::SlimeInserter;
-using slime::Cursor;
-using slime::ObjectInserter;
-
-namespace {
-
-Memory memory_address("address");
-Memory memory_cells("cells");
-Memory memory_dimensions("dimensions");
-Memory memory_value("value");
-
-void writeTensorAddress(Cursor &cursor, const TensorAddress &value) {
- ObjectInserter addressInserter(cursor, memory_address);
- Cursor &addressCursor = addressInserter.insertObject();
- for (const auto &elem : value.elements()) {
- Memory dimension(elem.dimension());
- Memory label(elem.label());
- addressCursor.setString(dimension, label);
- }
-}
-
-}
-
-class SlimeBinaryFormatSerializer : public TensorVisitor
-{
- Cursor &_tensor; // cursor for whole tensor
- Cursor &_dimensions; // cursor for dimensions array
- Cursor &_cells; // cursor for cells array
-public:
- SlimeBinaryFormatSerializer(Inserter &inserter);
- virtual ~SlimeBinaryFormatSerializer() override;
- virtual void visit(const TensorAddress &address, double value) override;
- void serialize(const Tensor &tensor);
-};
-
-SlimeBinaryFormatSerializer::SlimeBinaryFormatSerializer(Inserter &inserter)
- : _tensor(inserter.insertObject()),
- _dimensions(_tensor.setArray(memory_dimensions)),
- _cells(_tensor.setArray(memory_cells))
-{
-}
-
-
-SlimeBinaryFormatSerializer::~SlimeBinaryFormatSerializer() = default;
-
-void
-SlimeBinaryFormatSerializer::visit(const TensorAddress &address, double value)
-{
- Cursor &cellCursor = _cells.addObject();
- writeTensorAddress(cellCursor, address);
- cellCursor.setDouble(memory_value, value);
-}
-
-
-void
-SlimeBinaryFormatSerializer::serialize(const Tensor &tensor)
-{
- eval::ValueType type(tensor.type());
- for (const auto & dimension : type.dimensions()) {
- _dimensions.addString(Memory(dimension.name));
- }
- tensor.accept(*this);
-}
-
-
-void
-SlimeBinaryFormat::serialize(Inserter &inserter, const Tensor &tensor)
-{
- SlimeBinaryFormatSerializer serializer(inserter);
- serializer.serialize(tensor);
-}
-
-
-std::unique_ptr<Slime>
-SlimeBinaryFormat::serialize(const Tensor &tensor)
-{
- auto slime = std::make_unique<Slime>();
- SlimeInserter inserter(*slime);
- serialize(inserter, tensor);
- return slime;
-}
-
-
-}
diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h
deleted file mode 100644
index c9e9ff2c3e9..00000000000
--- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <memory>
-
-namespace vespalib { class Slime; }
-
-namespace vespalib::slime { struct Inserter; }
-
-namespace vespalib::tensor {
-
-class Tensor;
-class TensorBuilder;
-
-/**
- * Class for serializing a tensor into a slime object.
- */
-class SlimeBinaryFormat
-{
-public:
- static void serialize(slime::Inserter &inserter, const Tensor &tensor);
- static std::unique_ptr<Slime> serialize(const Tensor &tensor);
-};
-
-}
diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
index 4ca037e82a4..7be4db46372 100644
--- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
+++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
@@ -25,9 +25,9 @@ namespace {
constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u;
constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u;
constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u;
-constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; //Future
+constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u;
constexpr uint32_t DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6u;
-constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; //Future
+constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u;
constexpr uint32_t DOUBLE_VALUE_TYPE = 0;
constexpr uint32_t FLOAT_VALUE_TYPE = 1;
@@ -91,10 +91,11 @@ TypedBinaryFormat::deserialize(nbostream &stream)
if (formatId == DENSE_BINARY_FORMAT_TYPE) {
return DenseBinaryFormat(SerializeFormat::DOUBLE).deserialize(stream);
}
- if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) {
- return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserialize(stream);
- }
- if (formatId == MIXED_BINARY_FORMAT_TYPE) {
+ if ((formatId == SPARSE_BINARY_FORMAT_WITH_CELLTYPE) ||
+ (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) ||
+ (formatId == MIXED_BINARY_FORMAT_TYPE) ||
+ (formatId == MIXED_BINARY_FORMAT_WITH_CELLTYPE))
+ {
stream.adjustReadPos(read_pos - stream.rp());
return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::decode(stream));
}
diff --git a/eval/src/vespa/eval/tensor/tensor_factory.cpp b/eval/src/vespa/eval/tensor/tensor_factory.cpp
deleted file mode 100644
index 0b7fa3b9c2e..00000000000
--- a/eval/src/vespa/eval/tensor/tensor_factory.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "tensor.h"
-#include "tensor_factory.h"
-#include "tensor_builder.h"
-#include <vespa/eval/tensor/dense/dense_tensor_builder.h>
-
-namespace vespalib::tensor {
-
-std::unique_ptr<Tensor>
-TensorFactory::create(const TensorCells &cells, TensorBuilder &builder) {
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- builder.define_dimension(dimension);
- }
- }
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- const auto &label = addressElem.second;
- builder.add_label(builder.define_dimension(dimension), label);
- }
- builder.add_cell(cell.second);
- }
- return builder.build();
-}
-
-
-std::unique_ptr<Tensor>
-TensorFactory::create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder) {
- for (const auto &dimension : dimensions) {
- builder.define_dimension(dimension);
- }
- return create(cells, builder);
-}
-
-
-std::unique_ptr<Tensor>
-TensorFactory::createDense(const DenseTensorCells &cells)
-{
- std::map<std::string, size_t> dimensionSizes;
- DenseTensorBuilder builder;
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- dimensionSizes[addressElem.first] = std::max(dimensionSizes[addressElem.first], (addressElem.second + 1));
- }
- }
- std::map<std::string, typename DenseTensorBuilder::Dimension> dimensionEnums;
- for (const auto &dimensionElem : dimensionSizes) {
- dimensionEnums[dimensionElem.first] = builder.defineDimension(dimensionElem.first, dimensionElem.second);
- }
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- size_t label = addressElem.second;
- builder.addLabel(dimensionEnums[dimension], label);
- }
- builder.addCell(cell.second);
- }
- return builder.build();
-}
-
-
-}
diff --git a/eval/src/vespa/eval/tensor/tensor_factory.h b/eval/src/vespa/eval/tensor/tensor_factory.h
deleted file mode 100644
index 5364c28c8ff..00000000000
--- a/eval/src/vespa/eval/tensor/tensor_factory.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "types.h"
-
-namespace vespalib::tensor {
-
-
-class Tensor;
-class TensorBuilder;
-
-/**
- * A factory for creating tensors based on stl structures (TensorCells and
- * TensorDimensions) in unit tests.
- */
-class TensorFactory {
-public:
- static std::unique_ptr<Tensor>
- create(const TensorCells &cells, TensorBuilder &builder);
- static std::unique_ptr<Tensor>
- create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder);
- static std::unique_ptr<Tensor>
- createDense(const DenseTensorCells &cells);
-};
-
-}