aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHåvard Pettersen <havardpe@oath.com>2019-05-27 12:24:06 +0000
committerHåvard Pettersen <havardpe@oath.com>2019-05-31 11:49:25 +0000
commitb74a5049bb1bc8b7dce4167f4b639dfb3761c411 (patch)
tree5b405602554eda05c541e2cce8f01c259f6fa67a
parentad59a2b76aa92da991805758dfcb15ad7d9bc0ea (diff)
remove tensor factory
-rw-r--r--document/src/tests/serialization/vespadocumentserializer_test.cpp54
-rw-r--r--document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp33
-rw-r--r--document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp14
-rw-r--r--eval/CMakeLists.txt1
-rw-r--r--eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp402
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/.gitignore1
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt8
-rw-r--r--eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp185
-rw-r--r--eval/src/vespa/eval/tensor/CMakeLists.txt1
-rw-r--r--eval/src/vespa/eval/tensor/serialization/CMakeLists.txt1
-rw-r--r--eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp100
-rw-r--r--eval/src/vespa/eval/tensor/serialization/slime_binary_format.h26
-rw-r--r--eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp13
-rw-r--r--eval/src/vespa/eval/tensor/tensor_factory.cpp65
-rw-r--r--eval/src/vespa/eval/tensor/tensor_factory.h27
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_test.cpp27
-rw-r--r--searchcore/src/tests/proton/docsummary/docsummary.cpp28
-rw-r--r--searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp23
-rw-r--r--searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp34
-rw-r--r--searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp137
-rw-r--r--searchlib/src/tests/features/constant/constant_test.cpp26
-rw-r--r--searchlib/src/tests/features/tensor/tensor_test.cpp2
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp1
23 files changed, 321 insertions, 888 deletions
diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp
index c573eef6147..4b5c3e6f195 100644
--- a/document/src/tests/serialization/vespadocumentserializer_test.cpp
+++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp
@@ -40,8 +40,7 @@
#include <vespa/eval/tensor/types.h>
#include <vespa/eval/tensor/tensor_builder.h>
#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/vespalib/io/fileutil.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/testkit/testapp.h>
@@ -53,10 +52,9 @@ using vespalib::Slime;
using vespalib::nbostream;
using vespalib::nbostream_longlivedbuf;
using vespalib::slime::Cursor;
+using vespalib::eval::TensorSpec;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorBuilder;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
+using vespalib::tensor::DefaultTensorEngine;
using vespalib::compression::CompressionConfig;
using namespace document;
using std::string;
@@ -828,13 +826,14 @@ TEST("Require that predicate deserialization matches Java") {
namespace
{
-Tensor::UP
-createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+Tensor::UP createTensor(const TensorSpec &spec) {
+ auto value = DefaultTensorEngine::ref().from_spec(spec);
+ Tensor *tensor = dynamic_cast<Tensor*>(value.get());
+ ASSERT_TRUE(tensor != nullptr);
+ value.release();
+ return Tensor::UP(tensor);
}
-
}
TEST("Require that tensors can be serialized")
@@ -846,12 +845,12 @@ TEST("Require that tensors can be serialized")
nbostream stream;
serializeAndDeserialize(noTensorValue, stream);
stream.clear();
- emptyTensorValue = createTensor({}, {"x", "y"});
+ emptyTensorValue = createTensor(TensorSpec("tensor(x{},y{})"));
serializeAndDeserialize(emptyTensorValue, stream);
stream.clear();
- twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3},
- {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"});
+ twoCellsTwoDimsValue = createTensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)
+ .add({{"x", "4"}, {"y", "5"}}, 7));
serializeAndDeserialize(twoCellsTwoDimsValue, stream);
EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue);
EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue);
@@ -859,6 +858,7 @@ TEST("Require that tensors can be serialized")
}
+
const int tensor_doc_type_id = 321;
const string tensor_field_name = "my_tensor";
@@ -904,15 +904,15 @@ void checkDeserialization(const string &name, std::unique_ptr<Tensor> tensor) {
deserializeAndCheck(data_dir + name + "__java", value);
}
+
TEST("Require that tensor deserialization matches Java") {
checkDeserialization("non_existing_tensor", std::unique_ptr<Tensor>());
- checkDeserialization("empty_tensor", createTensor({}, {"dimX", "dimY"}));
+ checkDeserialization("empty_tensor", createTensor(TensorSpec("tensor(dimX{},dimY{})")));
checkDeserialization("multi_cell_tensor",
- createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 },
- {{{"dimX", "ccc"},
- {"dimY", "dddd"}}, 3.0},
- {{{"dimX", "e"},{"dimY","ff"}}, 5.0} },
- { "dimX", "dimY" }));
+ createTensor(TensorSpec("tensor(dimX{},dimY{})")
+ .add({{"dimX", "a"}, {"dimY", "bb"}}, 2.0)
+ .add({{"dimX", "ccc"}, {"dimY", "dddd"}}, 3.0)
+ .add({{"dimX", "e"}, {"dimY", "ff"}}, 5.0)));
}
struct TensorDocFixture {
@@ -980,14 +980,14 @@ DeserializedTensorDoc::getTensor() const
TEST("Require that wrong tensor type hides tensor")
{
TensorDocFixture f(tensor_doc_repo,
- createTensor({ {{{"dimX", "a"},{"dimY", "bb"}}, 2.0 },
- {{{"dimX", "ccc"},{"dimY", "dddd"}}, 3.0},
- {{{"dimX", "e"},{"dimY","ff"}}, 5.0} },
- { "dimX", "dimY" }));
+ createTensor(TensorSpec("tensor(dimX{},dimY{})")
+ .add({{"dimX", "a"}, {"dimY", "bb"}}, 2.0)
+ .add({{"dimX", "ccc"}, {"dimY", "dddd"}}, 3.0)
+ .add({{"dimX", "e"}, {"dimY", "ff"}}, 5.0)));
TensorDocFixture f1(tensor_doc_repo1,
- createTensor({ {{{"dimX", "a"}}, 20.0 },
- {{{"dimX", "ccc"}}, 30.0} },
- { "dimX" }));
+ createTensor(TensorSpec("tensor(dimX{})")
+ .add({{"dimX", "a"}}, 20.0)
+ .add({{"dimX", "ccc"}}, 30.0)));
DeserializedTensorDoc doc;
doc.setup(tensor_doc_repo, f._blob);
EXPECT_TRUE(doc.getTensor() != nullptr);
diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp
index dc91df0b5a0..9d2da9c983a 100644
--- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp
@@ -9,8 +9,7 @@ LOG_SETUP("fieldvalue_test");
#include <vespa/document/fieldvalue/tensorfieldvalue.h>
#include <vespa/eval/tensor/tensor.h>
#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/test/test_utils.h>
#include <vespa/vespalib/testkit/testapp.h>
@@ -19,6 +18,7 @@ using namespace document;
using namespace vespalib::tensor;
using vespalib::eval::TensorSpec;
using vespalib::eval::ValueType;
+using vespalib::tensor::DefaultTensorEngine;
using vespalib::tensor::test::makeTensor;
namespace
@@ -27,10 +27,12 @@ namespace
TensorDataType xSparseTensorDataType(ValueType::from_spec("tensor(x{})"));
TensorDataType xySparseTensorDataType(ValueType::from_spec("tensor(x{},y{})"));
-Tensor::UP
-createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+Tensor::UP createTensor(const TensorSpec &spec) {
+ auto value = DefaultTensorEngine::ref().from_spec(spec);
+ Tensor *tensor = dynamic_cast<Tensor*>(value.get());
+ ASSERT_TRUE(tensor != nullptr);
+ value.release();
+ return Tensor::UP(tensor);
}
std::unique_ptr<Tensor>
@@ -54,10 +56,10 @@ TEST("require that TensorFieldValue can be assigned tensors and cloned") {
TensorFieldValue noTensorValue(xySparseTensorDataType);
TensorFieldValue emptyTensorValue(xySparseTensorDataType);
TensorFieldValue twoCellsTwoDimsValue(xySparseTensorDataType);
- emptyTensorValue = createTensor({}, {"x", "y"});
- twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3},
- {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"});
+ emptyTensorValue = createTensor(TensorSpec("tensor(x{},y{})"));
+ twoCellsTwoDimsValue = createTensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)
+ .add({{"x", "4"}, {"y", "5"}}, 7));
EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue);
EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue);
EXPECT_NOT_EQUAL(emptyTensorValue, noTensorValue);
@@ -74,10 +76,9 @@ TEST("require that TensorFieldValue can be assigned tensors and cloned") {
EXPECT_NOT_EQUAL(*twoClone, *noneClone);
EXPECT_NOT_EQUAL(*twoClone, *emptyClone);
TensorFieldValue twoCellsTwoDimsValue2(xySparseTensorDataType);
- twoCellsTwoDimsValue2 =
- createTensor({ {{{"y", "3"}}, 3},
- {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"});
+ twoCellsTwoDimsValue2 = createTensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)
+ .add({{"x", "4"}, {"y", "5"}}, 7));
EXPECT_NOT_EQUAL(*noneClone, twoCellsTwoDimsValue2);
EXPECT_NOT_EQUAL(*emptyClone, twoCellsTwoDimsValue2);
EXPECT_EQUAL(*twoClone, twoCellsTwoDimsValue2);
@@ -87,8 +88,8 @@ TEST("require that TensorFieldValue::toString works")
{
TensorFieldValue tensorFieldValue(xSparseTensorDataType);
EXPECT_EQUAL("{TensorFieldValue: null}", tensorFieldValue.toString());
- tensorFieldValue = createTensor({{{{"x","a"}}, 3}}, {"x"});
- EXPECT_EQUAL("{TensorFieldValue: {\"dimensions\":[\"x\"],\"cells\":[{\"address\":{\"x\":\"a\"},\"value\":3}]}}", tensorFieldValue.toString());
+ tensorFieldValue = createTensor(TensorSpec("tensor(x{})").add({{"x", "a"}}, 3));
+ EXPECT_EQUAL("{TensorFieldValue: spec(tensor(x{})) {\n [a]: 3\n}}", tensorFieldValue.toString());
}
TEST("require that wrong tensor type for special case assign throws exception")
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
index f209358c901..9b318a39f0a 100644
--- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
@@ -4,16 +4,16 @@
#include <vespa/document/base/exceptions.h>
#include <vespa/document/datatype/tensor_data_type.h>
#include <vespa/vespalib/util/xmlstream.h>
+#include <vespa/eval/eval/tensor_spec.h>
#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/serialization/slime_binary_format.h>
-#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <ostream>
#include <cassert>
-using vespalib::slime::JsonFormat;
using vespalib::tensor::Tensor;
-using vespalib::tensor::SlimeBinaryFormat;
+using vespalib::eval::TensorSpec;
using vespalib::eval::ValueType;
+using Engine = vespalib::tensor::DefaultTensorEngine;
using namespace vespalib::xml;
namespace document {
@@ -148,11 +148,7 @@ TensorFieldValue::print(std::ostream& out, bool verbose,
(void) indent;
out << "{TensorFieldValue: ";
if (_tensor) {
- auto slime = SlimeBinaryFormat::serialize(*_tensor);
- vespalib::SimpleBuffer buf;
- JsonFormat::encode(*slime, buf, true);
- auto json = buf.get().make_string();
- out << json;
+ out << Engine::ref().to_spec(*_tensor).to_string();
} else {
out << "null";
}
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index d15f1c37d2d..4468ac46f61 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -44,7 +44,6 @@ vespa_define_module(
src/tests/tensor/tensor_performance
src/tests/tensor/tensor_remove_operation
src/tests/tensor/tensor_serialization
- src/tests/tensor/tensor_slime_serialization
src/tests/tensor/typed_cells
src/tests/tensor/vector_from_doubles_function
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
index b7aa988775d..2c113e814e9 100644
--- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
+++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
@@ -4,8 +4,7 @@
#include <vespa/eval/tensor/sparse/sparse_tensor.h>
#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h>
#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/eval/tensor/serialization/sparse_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -14,6 +13,7 @@
#include <vespa/eval/tensor/dense/dense_tensor_view.h>
using namespace vespalib::tensor;
+using vespalib::eval::TensorSpec;
using vespalib::nbostream;
using ExpBuffer = std::vector<uint8_t>;
@@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector<uint8_t> &rhs)
}
-namespace vespalib::tensor {
-
-static bool operator==(const Tensor &lhs, const Tensor &rhs)
-{
- return lhs.equals(rhs);
-}
-
-}
-
-template <class BuilderType>
-void
-checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs)
-{
- (void) stream;
- (void) rhs;
-}
-
-template <>
-void
-checkDeserialize<DefaultTensor::builder>(nbostream &stream, const Tensor &rhs)
-{
- nbostream wrapStream(stream.peek(), stream.size());
- auto chk = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_EQUAL(0u, wrapStream.size());
- EXPECT_EQUAL(*chk, rhs);
-}
-
-template <typename BuilderType>
-struct Fixture
-{
- BuilderType _builder;
- Fixture() : _builder() {}
-
- Tensor::UP createTensor(const TensorCells &cells) {
- return TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
+//-----------------------------------------------------------------------------
+
+template <typename T>
+void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) {
+ nbostream input(&exp[0], exp.size());
+ std::vector<T> cells;
+ TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells);
+ ASSERT_EQUAL(cells.size(), spec.cells().size());
+ size_t i = 0;
+ for (const auto &cell: spec.cells()) {
+ EXPECT_EQUAL(cells[i++], cell.second.value);
}
+ ASSERT_EQUAL(i, cells.size());
+}
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
- Tensor::UP deserialize(nbostream &stream) {
- BuilderType builder;
- nbostream wrapStream(stream.peek(), stream.size());
- auto formatId = wrapStream.getInt1_4Bytes();
- ASSERT_EQUAL(formatId, 1u); // sparse format
- SparseBinaryFormat::deserialize(wrapStream, builder);
- EXPECT_TRUE(wrapStream.empty());
- auto ret = builder.build();
- checkDeserialize<BuilderType>(stream, *ret);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs,
- const TensorDimensions &rhsDimensions) {
- Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions));
- nbostream rhsStream;
- serialize(rhsStream, *rhsTensor);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
+void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
+ auto &engine = DefaultTensorEngine::ref();
+ auto value = engine.from_spec(spec);
+ auto value_spec = engine.to_spec(*value);
+ nbostream actual;
+ engine.encode(*value, actual);
+ EXPECT_EQUAL(exp, actual);
+ auto decoded = engine.decode(actual);
+ auto decoded_spec = engine.to_spec(*decoded);
+ EXPECT_EQUAL(0u, actual.size());
+ EXPECT_EQUAL(value_spec, decoded_spec);
+ if (value->type().is_dense()) {
+ TEST_DO(verify_cells_only<float>(exp, value_spec));
+ TEST_DO(verify_cells_only<double>(exp, value_spec));
}
-};
-
-using SparseFixture = Fixture<SparseTensorBuilder>;
-
+}
-template <typename FixtureType>
-void
-testTensorSerialization(FixtureType &f)
-{
- TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {}));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
- {}, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
- {}, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
- 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+//-----------------------------------------------------------------------------
+
+TEST("test tensor serialization for SparseTensor") {
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 },
+ TensorSpec("tensor(x{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 },
+ TensorSpec("tensor(x{},y{})")));
+ TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+ TensorSpec("tensor(x{})")
+ .add({{"x", "1"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00 },
- { {{}, 3} }, { "x", "y"}));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
- 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
- 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00 },
- { {{{"y","3"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
+ 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00,
+ 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00 },
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", ""}, {"y", "3"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01,
0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00 },
- { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "2"}, {"y", "4"}}, 3)));
+ TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79,
0x01, 0x01, 0x31, 0x00, 0x40, 0x08,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
- { {{{"x","1"}}, 3} }, {"x", "y"}));
+ TensorSpec("tensor(x{},y{})")
+ .add({{"x", "1"}, {"y", ""}}, 3)));
}
-TEST_F("test tensor serialization for SparseTensor", SparseFixture)
-{
- testTensorSerialization(f);
+TEST("test tensor serialization for DenseTensor") {
+ TEST_DO(verify_serialized({0x02, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("double")));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1])")
+ .add({{"x", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 0)));
+ TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2])")
+ .add({{"x", 1}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x01,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[1])")
+ .add({{"x", 0}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02,
+ 0x01, 0x79, 0x01,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[2],y[1])")
+ .add({{"x", 1}, {"y", 0}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01,
+ 0x01, 0x79, 0x04,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[1],y[4])")
+ .add({{"x", 0}, {"y", 3}}, 3)));
+ TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x08, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00},
+ TensorSpec("tensor(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
-struct DenseFixture
-{
- Tensor::UP createTensor(const DenseTensorCells &cells) {
- return TensorFactory::createDense(cells);
- }
-
- void serialize(nbostream &stream, const Tensor &tensor) {
- TypedBinaryFormat::serialize(stream, tensor);
- }
-
- Tensor::UP deserialize(nbostream &stream) {
- nbostream wrapStream(stream.peek(), stream.size());
- auto ret = TypedBinaryFormat::deserialize(wrapStream);
- EXPECT_TRUE(wrapStream.size() == 0);
- stream.adjustReadPos(stream.size());
- return ret;
- }
- void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) {
- assertSerialized(exp, SerializeFormat::DOUBLE, rhs);
- }
- template <typename T>
- void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) {
- nbostream a(&exp[0], exp.size());
- std::vector<T> v;
- TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v);
- EXPECT_EQUAL(v.size(), rhs.cellsRef().size());
- for (size_t i(0); i < v.size(); i++) {
- EXPECT_EQUAL(v[i], rhs.cellsRef()[i]);
- }
- }
- void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) {
- Tensor::UP rhsTensor(createTensor(rhs));
- nbostream rhsStream;
- TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType);
- EXPECT_EQUAL(exp, rhsStream);
- auto rhs2 = deserialize(rhsStream);
- EXPECT_EQUAL(*rhs2, *rhsTensor);
-
- assertCellsOnly<float>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- assertCellsOnly<double>(exp, dynamic_cast<const DenseTensorView &>(*rhs2));
- }
-};
-
-
-TEST_F("test tensor serialization for DenseTensor", DenseFixture) {
- TEST_DO(f.assertSerialized({0x02, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 0}}));
- TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x01,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02,
- 0x01, 0x79, 0x01,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 1}, {"y", 0}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01,
- 0x01, 0x79, 0x04,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 0}, {"y", 3}}, 3}}));
- TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x08, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00},
- {{{{"x", 2}, {"y", 4}}, 3}}));
-}
-
-TEST_F("test 'float' cells", DenseFixture) {
- TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
- 0x01, 0x79, 0x05,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x40, 0x00, 0x00 },
- SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} }));
+TEST("test 'float' cells") {
+ TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03,
+ 0x01, 0x79, 0x05,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x40, 0x00, 0x00 },
+ TensorSpec("tensor<float>(x[3],y[5])")
+ .add({{"x", 2}, {"y", 4}}, 3)));
}
-
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore b/eval/src/tests/tensor/tensor_slime_serialization/.gitignore
deleted file mode 100644
index 9cb3b664d58..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-vespalib_tensor_slime_serialization_test_app
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt b/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt
deleted file mode 100644
index 9a9b8b7a436..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(eval_tensor_slime_serialization_test_app TEST
- SOURCES
- tensor_slime_serialization_test.cpp
- DEPENDS
- vespaeval
-)
-vespa_add_test(NAME eval_tensor_slime_serialization_test_app COMMAND eval_tensor_slime_serialization_test_app)
diff --git a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp
deleted file mode 100644
index e6e6c7de686..00000000000
--- a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp
+++ /dev/null
@@ -1,185 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/eval/tensor/sparse/sparse_tensor.h>
-#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h>
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
-#include <vespa/eval/tensor/serialization/typed_binary_format.h>
-#include <vespa/eval/tensor/serialization/slime_binary_format.h>
-#include <vespa/vespalib/data/slime/slime.h>
-#include <iostream>
-
-using namespace vespalib::tensor;
-
-template <typename BuilderType>
-struct Fixture
-{
- BuilderType _builder;
- Fixture() : _builder() {}
-
- Tensor::UP createTensor(const TensorCells &cells) {
- return vespalib::tensor::TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
- }
-
- static inline uint32_t getTensorTypeId();
-
- void assertSerialized(const vespalib::string &exp, const TensorCells &rhs,
- const TensorDimensions &rhsDimensions) {
- Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions));
- auto slime = SlimeBinaryFormat::serialize(*rhsTensor);
- vespalib::Memory memory_exp(exp);
- vespalib::Slime expSlime;
- size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime);
- EXPECT_TRUE(used > 0);
- EXPECT_EQUAL(expSlime, *slime);
- }
-};
-
-template <>
-uint32_t
-Fixture<SparseTensorBuilder>::getTensorTypeId() { return 2u; }
-
-
-using SparseFixture = Fixture<SparseTensorBuilder>;
-
-
-namespace {
-vespalib::string twoCellsJson[3] =
-{
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { y:'3'}, value: 4.0 },"
- "{ address: { x:'1'}, value: 3.0 }"
- "] }",
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 },"
- "{ address: { y:'3'}, value: 4.0 }"
- "] }",
- "{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 },"
- "{ address: { y:'3'}, value: 4.0 }"
- "] }",
-};
-}
-
-
-template <typename FixtureType>
-void
-testTensorSlimeSerialization(FixtureType &f)
-{
- TEST_DO(f.assertSerialized("{ dimensions: [], cells: [] }", {}, {}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [] }",
- {}, { "x" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [] }",
- {}, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ],"
- "cells: ["
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, { "x" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { }, value: 3.0 }"
- "] }",
- { {{}, 3} }, { "x", "y"}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { y: '3' }, value: 3.0 }"
- "] }",
- { {{{"y","3"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'2', y:'4'}, value: 3.0 }"
- "] }",
- { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x:'1'}, value: 3.0 }"
- "] }",
- { {{{"x","1"}}, 3} }, {"x", "y"}));
- TEST_DO(f.assertSerialized(twoCellsJson[FixtureType::getTensorTypeId()],
- { {{{"x","1"}}, 3}, {{{"y","3"}}, 4} },
- {"x", "y"}));
-}
-
-TEST_F("test tensor slime serialization for SparseTensor", SparseFixture)
-{
- testTensorSlimeSerialization(f);
-}
-
-
-struct DenseFixture
-{
- DenseFixture() {}
-
- Tensor::UP createTensor(const DenseTensorCells &cells) {
- return vespalib::tensor::TensorFactory::createDense(cells);
- }
-
- void assertSerialized(const vespalib::string &exp,
- const DenseTensorCells &rhs) {
- Tensor::UP rhsTensor(createTensor(rhs));
- auto slime = SlimeBinaryFormat::serialize(*rhsTensor);
- vespalib::Memory memory_exp(exp);
- vespalib::Slime expSlime;
- size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime);
- EXPECT_TRUE(used > 0);
- EXPECT_EQUAL(expSlime, *slime);
- }
-};
-
-
-TEST_F("test tensor slime serialization for DenseTensor", DenseFixture)
-{
- TEST_DO(f.assertSerialized("{ dimensions: [], cells: ["
- "{ address: { }, value: 0.0 }"
- "] }", {}));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: ["
- "{ address: { x: '0' }, value: 0.0 }"
- "] }",
- { {{{"x",0}}, 0} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 }"
- "] }",
- { {{{"x",0},{"y",0}}, 0} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ],"
- "cells: ["
- "{ address: { x: '0' }, value: 0.0 },"
- "{ address: { x: '1' }, value: 3.0 }"
- "] }",
- { {{{"x",1}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 3.0 }"
- "] }",
- { {{{"x",0},{"y",0}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 },"
- "{ address: { x: '1', y: '0' }, value: 3.0 }"
- "] }",
- { {{{"x",1},{"y", 0}}, 3} }));
- TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ],"
- " cells: ["
- "{ address: { x: '0', y: '0' }, value: 0.0 },"
- "{ address: { x: '0', y: '1' }, value: 0.0 },"
- "{ address: { x: '0', y: '2' }, value: 0.0 },"
- "{ address: { x: '0', y: '3' }, value: 3.0 }"
- "] }",
- { {{{"x",0},{"y",3}}, 3} }));
-}
-
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt
index ca4d275ea44..4e9940d3c0a 100644
--- a/eval/src/vespa/eval/tensor/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_library(eval_tensor OBJECT
tensor.cpp
tensor_address.cpp
tensor_apply.cpp
- tensor_factory.cpp
tensor_mapper.cpp
wrapped_simple_tensor.cpp
)
diff --git a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
index cd3cd455e55..fc9ac64ea68 100644
--- a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt
@@ -3,6 +3,5 @@ vespa_add_library(eval_tensor_serialization OBJECT
SOURCES
sparse_binary_format.cpp
dense_binary_format.cpp
- slime_binary_format.cpp
typed_binary_format.cpp
)
diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp
deleted file mode 100644
index ece3c2e4a07..00000000000
--- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "slime_binary_format.h"
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/tensor_builder.h>
-#include <vespa/eval/tensor/tensor_visitor.h>
-#include <vespa/vespalib/data/slime/inserter.h>
-#include <vespa/vespalib/data/slime/cursor.h>
-#include <vespa/vespalib/data/slime/slime.h>
-#include <vespa/vespalib/data/memory.h>
-
-namespace vespalib::tensor {
-
-
-using slime::Inserter;
-using slime::SlimeInserter;
-using slime::Cursor;
-using slime::ObjectInserter;
-
-namespace {
-
-Memory memory_address("address");
-Memory memory_cells("cells");
-Memory memory_dimensions("dimensions");
-Memory memory_value("value");
-
-void writeTensorAddress(Cursor &cursor, const TensorAddress &value) {
- ObjectInserter addressInserter(cursor, memory_address);
- Cursor &addressCursor = addressInserter.insertObject();
- for (const auto &elem : value.elements()) {
- Memory dimension(elem.dimension());
- Memory label(elem.label());
- addressCursor.setString(dimension, label);
- }
-}
-
-}
-
-class SlimeBinaryFormatSerializer : public TensorVisitor
-{
- Cursor &_tensor; // cursor for whole tensor
- Cursor &_dimensions; // cursor for dimensions array
- Cursor &_cells; // cursor for cells array
-public:
- SlimeBinaryFormatSerializer(Inserter &inserter);
- virtual ~SlimeBinaryFormatSerializer() override;
- virtual void visit(const TensorAddress &address, double value) override;
- void serialize(const Tensor &tensor);
-};
-
-SlimeBinaryFormatSerializer::SlimeBinaryFormatSerializer(Inserter &inserter)
- : _tensor(inserter.insertObject()),
- _dimensions(_tensor.setArray(memory_dimensions)),
- _cells(_tensor.setArray(memory_cells))
-{
-}
-
-
-SlimeBinaryFormatSerializer::~SlimeBinaryFormatSerializer() = default;
-
-void
-SlimeBinaryFormatSerializer::visit(const TensorAddress &address, double value)
-{
- Cursor &cellCursor = _cells.addObject();
- writeTensorAddress(cellCursor, address);
- cellCursor.setDouble(memory_value, value);
-}
-
-
-void
-SlimeBinaryFormatSerializer::serialize(const Tensor &tensor)
-{
- eval::ValueType type(tensor.type());
- for (const auto & dimension : type.dimensions()) {
- _dimensions.addString(Memory(dimension.name));
- }
- tensor.accept(*this);
-}
-
-
-void
-SlimeBinaryFormat::serialize(Inserter &inserter, const Tensor &tensor)
-{
- SlimeBinaryFormatSerializer serializer(inserter);
- serializer.serialize(tensor);
-}
-
-
-std::unique_ptr<Slime>
-SlimeBinaryFormat::serialize(const Tensor &tensor)
-{
- auto slime = std::make_unique<Slime>();
- SlimeInserter inserter(*slime);
- serialize(inserter, tensor);
- return slime;
-}
-
-
-}
diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h
deleted file mode 100644
index c9e9ff2c3e9..00000000000
--- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <memory>
-
-namespace vespalib { class Slime; }
-
-namespace vespalib::slime { struct Inserter; }
-
-namespace vespalib::tensor {
-
-class Tensor;
-class TensorBuilder;
-
-/**
- * Class for serializing a tensor into a slime object.
- */
-class SlimeBinaryFormat
-{
-public:
- static void serialize(slime::Inserter &inserter, const Tensor &tensor);
- static std::unique_ptr<Slime> serialize(const Tensor &tensor);
-};
-
-}
diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
index 4ca037e82a4..7be4db46372 100644
--- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
+++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp
@@ -25,9 +25,9 @@ namespace {
constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u;
constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u;
constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u;
-constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; //Future
+constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u;
constexpr uint32_t DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6u;
-constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; //Future
+constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u;
constexpr uint32_t DOUBLE_VALUE_TYPE = 0;
constexpr uint32_t FLOAT_VALUE_TYPE = 1;
@@ -91,10 +91,11 @@ TypedBinaryFormat::deserialize(nbostream &stream)
if (formatId == DENSE_BINARY_FORMAT_TYPE) {
return DenseBinaryFormat(SerializeFormat::DOUBLE).deserialize(stream);
}
- if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) {
- return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserialize(stream);
- }
- if (formatId == MIXED_BINARY_FORMAT_TYPE) {
+ if ((formatId == SPARSE_BINARY_FORMAT_WITH_CELLTYPE) ||
+ (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) ||
+ (formatId == MIXED_BINARY_FORMAT_TYPE) ||
+ (formatId == MIXED_BINARY_FORMAT_WITH_CELLTYPE))
+ {
stream.adjustReadPos(read_pos - stream.rp());
return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::decode(stream));
}
diff --git a/eval/src/vespa/eval/tensor/tensor_factory.cpp b/eval/src/vespa/eval/tensor/tensor_factory.cpp
deleted file mode 100644
index 0b7fa3b9c2e..00000000000
--- a/eval/src/vespa/eval/tensor/tensor_factory.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "tensor.h"
-#include "tensor_factory.h"
-#include "tensor_builder.h"
-#include <vespa/eval/tensor/dense/dense_tensor_builder.h>
-
-namespace vespalib::tensor {
-
-std::unique_ptr<Tensor>
-TensorFactory::create(const TensorCells &cells, TensorBuilder &builder) {
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- builder.define_dimension(dimension);
- }
- }
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- const auto &label = addressElem.second;
- builder.add_label(builder.define_dimension(dimension), label);
- }
- builder.add_cell(cell.second);
- }
- return builder.build();
-}
-
-
-std::unique_ptr<Tensor>
-TensorFactory::create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder) {
- for (const auto &dimension : dimensions) {
- builder.define_dimension(dimension);
- }
- return create(cells, builder);
-}
-
-
-std::unique_ptr<Tensor>
-TensorFactory::createDense(const DenseTensorCells &cells)
-{
- std::map<std::string, size_t> dimensionSizes;
- DenseTensorBuilder builder;
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- dimensionSizes[addressElem.first] = std::max(dimensionSizes[addressElem.first], (addressElem.second + 1));
- }
- }
- std::map<std::string, typename DenseTensorBuilder::Dimension> dimensionEnums;
- for (const auto &dimensionElem : dimensionSizes) {
- dimensionEnums[dimensionElem.first] = builder.defineDimension(dimensionElem.first, dimensionElem.second);
- }
- for (const auto &cell : cells) {
- for (const auto &addressElem : cell.first) {
- const auto &dimension = addressElem.first;
- size_t label = addressElem.second;
- builder.addLabel(dimensionEnums[dimension], label);
- }
- builder.addCell(cell.second);
- }
- return builder.build();
-}
-
-
-}
diff --git a/eval/src/vespa/eval/tensor/tensor_factory.h b/eval/src/vespa/eval/tensor/tensor_factory.h
deleted file mode 100644
index 5364c28c8ff..00000000000
--- a/eval/src/vespa/eval/tensor/tensor_factory.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "types.h"
-
-namespace vespalib::tensor {
-
-
-class Tensor;
-class TensorBuilder;
-
-/**
- * A factory for creating tensors based on stl structures (TensorCells and
- * TensorDimensions) in unit tests.
- */
-class TensorFactory {
-public:
- static std::unique_ptr<Tensor>
- create(const TensorCells &cells, TensorBuilder &builder);
- static std::unique_ptr<Tensor>
- create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder);
- static std::unique_ptr<Tensor>
- createDense(const DenseTensorCells &cells);
-};
-
-}
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index 992a9c56dfb..3734d2fe1dc 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -7,9 +7,8 @@
#include <vespa/document/update/arithmeticvalueupdate.h>
#include <vespa/document/update/assignvalueupdate.h>
#include <vespa/document/update/documentupdate.h>
-#include <vespa/eval/tensor/default_tensor.h>
#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/types.h>
#include <vespa/fastos/file.h>
#include <vespa/searchcommon/attribute/attributecontent.h>
@@ -75,9 +74,9 @@ using search::tensor::TensorAttribute;
using search::test::DirectoryHandler;
using std::string;
using vespalib::eval::ValueType;
+using vespalib::eval::TensorSpec;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
+using vespalib::tensor::DefaultTensorEngine;
using AVConfig = search::attribute::Config;
using AVBasicType = search::attribute::BasicType;
@@ -612,13 +611,11 @@ TEST_F("require that filter attribute manager can return flushed serial number",
namespace {
-Tensor::UP
-createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+Tensor::UP make_tensor(const TensorSpec &spec) {
+ auto tensor = DefaultTensorEngine::ref().from_spec(spec);
+ return Tensor::UP(dynamic_cast<Tensor*>(tensor.release()));
}
-
AttributeVector::SP
createTensorAttribute(Fixture &f) {
AVConfig cfg(AVBasicType::TENSOR);
@@ -650,8 +647,8 @@ TEST_F("Test that we can use attribute writer to write to tensor attribute",
AttributeVector::SP a1 = createTensorAttribute(f);
Schema s = createTensorSchema();
DocBuilder builder(s);
- auto tensor = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"});
+ auto tensor = make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "4"}, {"y", "5"}}, 7));
Document::UP doc = createTensorPutDoc(builder, *tensor);
f.put(1, *doc, 1);
EXPECT_EQUAL(2u, a1->getNumDocs());
@@ -668,8 +665,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture)
AttributeVector::SP a1 = createTensorAttribute(f);
Schema s = createTensorSchema();
DocBuilder builder(s);
- auto tensor = createTensor({ {{{"x", "6"}, {"y", "7"}}, 9} },
- {"x", "y"});
+ auto tensor = make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "6"}, {"y", "7"}}, 9));
Document::UP doc = createTensorPutDoc(builder, *tensor);
f.put(1, *doc, 1);
EXPECT_EQUAL(2u, a1->getNumDocs());
@@ -682,8 +679,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture)
const document::DocumentType &dt(builder.getDocumentType());
DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("doc::1"));
- auto new_tensor = createTensor({ {{{"x", "8"}, {"y", "9"}}, 11} },
- {"x", "y"});
+ auto new_tensor = make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "8"}, {"y", "9"}}, 11));
TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})"));
TensorFieldValue new_value(xySparseTensorDataType);
new_value = new_tensor->clone();
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index 38b85594a62..e8152161faa 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -2,9 +2,9 @@
#include <tests/proton/common/dummydbowner.h>
#include <vespa/config/helper/configgetter.hpp>
-#include <vespa/eval/tensor/default_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
-#include <vespa/eval/tensor/tensor_factory.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/test/make_bucket_space.h>
#include <vespa/searchcore/proton/attribute/attribute_writer.h>
@@ -55,10 +55,9 @@ using search::index::schema::CollectionType;
using storage::spi::Timestamp;
using vespa::config::search::core::ProtonConfig;
using vespa::config::content::core::BucketspacesConfig;
+using vespalib::eval::TensorSpec;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
-using vespalib::tensor::TensorFactory;
+using vespalib::tensor::DefaultTensorEngine;
using namespace vespalib::slime;
typedef std::unique_ptr<GeneralResult> GeneralResultPtr;
@@ -137,10 +136,9 @@ getDocTypeName()
return "searchdocument";
}
-Tensor::UP createTensor(const TensorCells &cells,
- const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return TensorFactory::create(cells, dimensions, builder);
+Tensor::UP make_tensor(const TensorSpec &spec) {
+ auto tensor = DefaultTensorEngine::ref().from_spec(spec);
+ return Tensor::UP(dynamic_cast<Tensor*>(tensor.release()));
}
vespalib::string asVstring(vespalib::Memory str) {
@@ -750,7 +748,8 @@ Test::requireThatAttributesAreUsed()
endElement().
endField().
startAttributeField("bj").
- addTensor(createTensor({ {{{"x","f"},{"y","g"}}, 3} }, { "x", "y"})).
+ addTensor(make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "f"}, {"y", "g"}}, 3))).
endField().
endDocument(),
2);
@@ -776,7 +775,8 @@ Test::requireThatAttributesAreUsed()
"bh:[{item:40.4,weight:4},{item:50.5,weight:5}],"
"bi:[{item:'quux',weight:7},{item:'qux',weight:6}],"
"bj:'0x01020178017901016601674008000000000000'}", *rep, 0, true));
- TEST_DO(assertTensor(createTensor({ {{{"x","f"},{"y","g"}}, 3} }, { "x", "y"}),
+ TEST_DO(assertTensor(make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "f"}, {"y", "g"}}, 3)),
"bj", *rep, 0, rclass));
// empty doc
@@ -790,13 +790,15 @@ Test::requireThatAttributesAreUsed()
attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(bjAttr->getNamePrefix()),
[&]() {
- bjTensorAttr->setTensor(3, *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"}));
+ bjTensorAttr->setTensor(3, *make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "a"}, {"y", "b"}}, 4)));
bjTensorAttr->commit();
});
attributeFieldWriter.sync();
DocsumReply::UP rep2 = dc._ddb->getDocsums(req);
- TEST_DO(assertTensor(createTensor({ {{{"x","a"},{"y","b"}}, 4} }, { "x", "y"}),
+ TEST_DO(assertTensor(make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "a"}, {"y", "b"}}, 4)),
"bj", *rep2, 1, rclass));
DocsumRequest req3;
diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
index b82fec85d47..cc6eef14fd6 100644
--- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
+++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
@@ -47,8 +47,7 @@
#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
#include <vespa/eval/tensor/tensor.h>
#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/vespalib/data/slime/slime.h>
using document::Annotation;
@@ -95,12 +94,12 @@ using vespa::config::search::SummarymapConfig;
using vespa::config::search::SummarymapConfigBuilder;
using vespalib::Slime;
using vespalib::eval::ValueType;
+using vespalib::eval::TensorSpec;
using vespalib::geo::ZCurve;
using vespalib::slime::Cursor;
using vespalib::string;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
+using vespalib::tensor::DefaultTensorEngine;
using namespace search::docsummary;
@@ -676,11 +675,9 @@ Test::requireThatPredicateIsPrinted()
SFC::convertSummaryField(false, *doc.getValue("predicate")).get());
}
-
-Tensor::UP
-createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+Tensor::UP make_tensor(const TensorSpec &spec) {
+ auto tensor = DefaultTensorEngine::ref().from_spec(spec);
+ return Tensor::UP(dynamic_cast<Tensor*>(tensor.release()));
}
void
@@ -688,14 +685,14 @@ Test::requireThatTensorIsNotConverted()
{
TensorDataType tensorDataType(ValueType::from_spec("tensor(x{},y{})"));
TensorFieldValue tensorFieldValue(tensorDataType);
- tensorFieldValue = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"});
+ tensorFieldValue = make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "4"}, {"y", "5"}}, 7));
Document doc(getDocType(), DocumentId("doc:scheme:"));
doc.setRepo(*_documentRepo);
doc.setValue("tensor", tensorFieldValue);
- TEST_CALL(checkTensor(createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
- {"x", "y"}),
+ TEST_CALL(checkTensor(make_tensor(TensorSpec("tensor(x{},y{})")
+ .add({{"x", "4"}, {"y", "5"}}, 7)),
SFC::convertSummaryField(false,
*doc.getValue("tensor")).get()));
doc.setValue("tensor", TensorFieldValue());
diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
index 1e24a784fd9..cfa05b7a765 100644
--- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
+++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
@@ -1,8 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/eval/tensor/default_tensor.h>
#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/searchcommon/attribute/search_context_params.h>
#include <vespa/searchlib/fef/termfieldmatchdata.h>
#include <vespa/searchlib/tensor/i_tensor_attribute.h>
@@ -13,11 +12,17 @@ using search::attribute::IAttributeVector;
using search::tensor::ITensorAttribute;
using search::tensor::TensorAttribute;
using vespalib::eval::ValueType;
-using vespalib::tensor::DenseTensorCells;
+using vespalib::eval::TensorSpec;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
-using vespalib::tensor::TensorFactory;
+using vespalib::tensor::DefaultTensorEngine;
+
+Tensor::UP createTensor(const TensorSpec &spec) {
+ auto value = DefaultTensorEngine::ref().from_spec(spec);
+ Tensor *tensor = dynamic_cast<Tensor*>(value.get());
+ ASSERT_TRUE(tensor != nullptr);
+ value.release();
+ return Tensor::UP(tensor);
+}
namespace search::attribute {
@@ -482,32 +487,23 @@ TEST("onSerializeForDescendingSort() is forwarded with remapped LID to target ve
}
struct TensorAttrFixture : Fixture {
- vespalib::tensor::DefaultTensor::builder builder;
std::shared_ptr<Tensor> tensor1;
std::shared_ptr<Tensor> tensor2;
TensorAttrFixture(bool dense)
: Fixture(),
- builder(),
tensor1(),
tensor2()
{
setup(dense);
}
- Tensor::UP createTensor(const TensorCells &cells,
- const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, builder);
- }
- Tensor::UP createDenseTensor(const DenseTensorCells &cells) const {
- return TensorFactory::createDense(cells);
- }
void setup(bool dense) {
if (dense) {
- tensor1 = createDenseTensor({ {{{"x",1}}, 11} });
- tensor2 = createDenseTensor({ {{{"x",0}}, 12}, {{{"x", 1}}, 0} });
+ tensor1 = createTensor(TensorSpec("tensor(x[2])").add({{"x", 1}}, 11));
+ tensor2 = createTensor(TensorSpec("tensor(x[2])").add({{"x", 0}}, 12).add({{"x", 1}}, 0));
} else {
- tensor1 = createTensor({ {{{"x","1"}}, 11} }, { "x" });
- tensor2 = createTensor({ {{{"x","0"}}, 12} }, { "x" });
+ tensor1 = createTensor(TensorSpec("tensor(x{})").add({{"x", "1"}}, 11));
+ tensor2 = createTensor(TensorSpec("tensor(x{})").add({{"x", "0"}}, 12));
}
const std::vector<ImportedAttributeFixture::LidToLidMapping<std::shared_ptr<Tensor>>> mappings =
{ {DocId(2), dummy_gid(3), DocId(3), tensor1 },
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
index cbbaa518b16..e0ec4a6fd06 100644
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
+++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
@@ -6,8 +6,9 @@
#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
#include <vespa/searchlib/attribute/attributeguard.h>
-#include <vespa/eval/tensor/tensor_factory.h>
-#include <vespa/eval/tensor/default_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/vespalib/io/fileutil.h>
#include <vespa/vespalib/data/fileheader.h>
#include <vespa/fastos/file.h>
@@ -21,11 +22,10 @@ using search::tensor::GenericTensorAttribute;
using search::AttributeGuard;
using search::AttributeVector;
using vespalib::eval::ValueType;
+using vespalib::eval::TensorSpec;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::DenseTensorCells;
-using vespalib::tensor::TensorDimensions;
-using vespalib::tensor::TensorFactory;
+using vespalib::tensor::DenseTensor;
+using vespalib::tensor::DefaultTensorEngine;
namespace vespalib {
namespace tensor {
@@ -41,6 +41,17 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs)
vespalib::string sparseSpec("tensor(x{},y{})");
vespalib::string denseSpec("tensor(x[2],y[3])");
+Tensor::UP createTensor(const TensorSpec &spec) {
+ auto value = DefaultTensorEngine::ref().from_spec(spec);
+ if (value->is_double()) {
+ return Tensor::UP(new DenseTensor(ValueType::double_type(), {value->as_double()}));
+ }
+ Tensor *tensor = dynamic_cast<Tensor*>(value.get());
+ ASSERT_TRUE(tensor != nullptr);
+ value.release();
+ return Tensor::UP(tensor);
+}
+
struct Fixture
{
using BasicType = search::attribute::BasicType;
@@ -52,7 +63,6 @@ struct Fixture
vespalib::string _typeSpec;
std::shared_ptr<TensorAttribute> _tensorAttr;
std::shared_ptr<AttributeVector> _attr;
- vespalib::tensor::DefaultTensor::builder _builder;
bool _denseTensors;
bool _useDenseTensorAttribute;
@@ -63,7 +73,6 @@ struct Fixture
_typeSpec(typeSpec),
_tensorAttr(),
_attr(),
- _builder(),
_denseTensors(false),
_useDenseTensorAttribute(useDenseTensorAttribute)
{
@@ -85,17 +94,6 @@ struct Fixture
}
}
- Tensor::UP createTensor(const TensorCells &cells) {
- return TensorFactory::create(cells, _builder);
- }
- Tensor::UP createTensor(const TensorCells &cells,
- const TensorDimensions &dimensions) {
- return TensorFactory::create(cells, dimensions, _builder);
- }
- Tensor::UP createDenseTensor(const DenseTensorCells &cells) const {
- return TensorFactory::createDense(cells);
- }
-
void ensureSpace(uint32_t docId) {
while (_attr->getNumDocs() <= docId) {
uint32_t newDocId = 0u;
@@ -138,19 +136,9 @@ struct Fixture
}
void
- assertGetTensor(const TensorCells &expCells,
- const TensorDimensions &expDimensions,
- uint32_t docId)
- {
- Tensor::UP expTensor = createTensor(expCells, expDimensions);
- assertGetTensor(*expTensor, docId);
- }
-
- void
- assertGetDenseTensor(const DenseTensorCells &expCells,
- uint32_t docId)
+ assertGetTensor(const TensorSpec &expSpec, uint32_t docId)
{
- Tensor::UP expTensor = createDenseTensor(expCells);
+ Tensor::UP expTensor = createTensor(expSpec);
assertGetTensor(*expTensor, docId);
}
@@ -166,72 +154,27 @@ struct Fixture
EXPECT_TRUE(loadok);
}
- bool isUnbound(const vespalib::string &dimensionName) const
- {
- ValueType type = _cfg.tensorType();
- for (const auto &dim : type.dimensions()) {
- if (dim.name == dimensionName && !dim.is_bound()) {
- return true;
- }
- }
- return false;
- }
-
Tensor::UP expDenseTensor3() const
{
- if (isUnbound("x")) {
- if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",0},{"y",1}}, 11} });
- }
- return createDenseTensor({ {{{"x",0},{"y",1}}, 11},
- {{{"x",0},{"y",2}}, 0} });
- } else if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",0},{"y",1}}, 11},
- {{{"x",1},{"y",0}}, 0} });
- }
- return createDenseTensor({ {{{"x",0},{"y",1}}, 11},
- {{{"x",1},{"y",2}}, 0} });
+ return createTensor(TensorSpec(denseSpec)
+ .add({{"x", 0}, {"y", 1}}, 11)
+ .add({{"x", 1}, {"y", 2}}, 0));
}
Tensor::UP expDenseFillTensor() const
{
- if (isUnbound("x")) {
- if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",0},{"y",0}}, 5} });
- }
- return createDenseTensor({ {{{"x",0},{"y",0}}, 5},
- {{{"x",0},{"y",2}}, 0} });
- } else if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",0},{"y",0}}, 5},
- {{{"x",1},{"y",0}}, 0} });
- }
- return createDenseTensor({ {{{"x",0},{"y",0}}, 5},
- {{{"x",1},{"y",2}}, 0} });
+ return createTensor(TensorSpec(denseSpec)
+ .add({{"x", 0}, {"y", 0}}, 5)
+ .add({{"x", 1}, {"y", 2}}, 0));
}
Tensor::UP expEmptyDenseTensor() const
{
- if (isUnbound("x")) {
- if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",0},{"y",0}}, 0} });
- }
- return createDenseTensor({ {{{"x",0},{"y",2}}, 0} });
- } else if (isUnbound("y")) {
- return createDenseTensor({ {{{"x",1},{"y",0}}, 0} });
- }
- return createDenseTensor({ {{{"x",1},{"y",2}}, 0} });
+ return createTensor(TensorSpec(denseSpec));
}
vespalib::string expEmptyDenseTensorSpec() const {
- if (isUnbound("x")) {
- if (isUnbound("y")) {
- return "tensor(x[1],y[1])";
- }
- return "tensor(x[1],y[3])";
- } else if (isUnbound("y")) {
- return "tensor(x[2],y[1])";
- }
- return "tensor(x[2],y[3])";
+ return denseSpec;
}
void testEmptyAttribute();
@@ -257,7 +200,7 @@ Fixture::testSetTensorValue()
EXPECT_EQUAL(5u, _attr->getNumDocs());
EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit());
TEST_DO(assertGetNoTensor(4));
- EXPECT_EXCEPTION(setTensor(4, *createTensor({}, {})),
+ EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))),
WrongTensorTypeException,
"but other tensor type is 'double'");
TEST_DO(assertGetNoTensor(4));
@@ -267,9 +210,11 @@ Fixture::testSetTensorValue()
setTensor(3, *expDenseTensor3());
TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
} else {
- TEST_DO(assertGetTensor({}, {"x", "y"}, 4));
- setTensor(3, *createTensor({ {{}, 11} }, { "x", "y"}));
- TEST_DO(assertGetTensor({ {{}, 11} }, { "x", "y"}, 3));
+ TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
+ setTensor(3, *createTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", ""}}, 11)));
+ TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", ""}}, 11), 3));
}
TEST_DO(assertGetNoTensor(2));
TEST_DO(clearTensor(3));
@@ -284,7 +229,8 @@ Fixture::testSaveLoad()
if (_denseTensors) {
setTensor(3, *expDenseTensor3());
} else {
- setTensor(3, *createTensor({ {{{"y","1"}}, 11} }, { "x", "y"}));
+ setTensor(3, *createTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", "1"}}, 11)));
}
TEST_DO(save());
TEST_DO(load());
@@ -294,8 +240,9 @@ Fixture::testSaveLoad()
TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4));
} else {
- TEST_DO(assertGetTensor({ {{{"y","1"}}, 11} }, { "x", "y"}, 3));
- TEST_DO(assertGetTensor({}, {"x", "y"}, 4));
+ TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", "1"}}, 11), 3));
+ TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
}
TEST_DO(assertGetNoTensor(2));
}
@@ -310,9 +257,11 @@ Fixture::testCompaction()
}
ensureSpace(4);
Tensor::UP emptytensor = _tensorAttr->getEmptyTensor();
- Tensor::UP emptyxytensor = createTensor({}, {"x", "y"});
- Tensor::UP simpletensor = createTensor({ {{{"y","1"}}, 11} }, { "x", "y"});
- Tensor::UP filltensor = createTensor({ {{}, 5} }, { "x", "y"});
+ Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec));
+ Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", "1"}}, 11));
+ Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", ""}}, 5));
if (_denseTensors) {
emptyxytensor = expEmptyDenseTensor();
simpletensor = expDenseTensor3();
diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp
index f9d1736ba9e..e558839d2cf 100644
--- a/searchlib/src/tests/features/constant/constant_test.cpp
+++ b/searchlib/src/tests/features/constant/constant_test.cpp
@@ -7,9 +7,8 @@
#include <vespa/searchlib/fef/test/indexenvironment.h>
#include <vespa/eval/eval/function.h>
#include <vespa/eval/eval/tensor_spec.h>
-#include <vespa/eval/tensor/default_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
-#include <vespa/eval/tensor/tensor_factory.h>
using search::feature_t;
using namespace search::fef;
@@ -22,21 +21,11 @@ using vespalib::eval::DoubleValue;
using vespalib::eval::TensorSpec;
using vespalib::eval::ValueType;
using vespalib::tensor::DefaultTensorEngine;
-using vespalib::tensor::DenseTensorCells;
using vespalib::tensor::Tensor;
-using vespalib::tensor::TensorCells;
-using vespalib::tensor::TensorDimensions;
-using vespalib::tensor::TensorFactory;
namespace
{
-Tensor::UP createTensor(const TensorCells &cells,
- const TensorDimensions &dimensions) {
- vespalib::tensor::DefaultTensor::builder builder;
- return TensorFactory::create(cells, dimensions, builder);
-}
-
Tensor::UP make_tensor(const TensorSpec &spec) {
auto tensor = DefaultTensorEngine::ref().from_spec(spec);
return Tensor::UP(dynamic_cast<Tensor*>(tensor.release()));
@@ -72,10 +61,9 @@ struct ExecFixture
return extractDouble(docId);
}
void addTensor(const vespalib::string &name,
- const TensorCells &cells,
- const TensorDimensions &dimensions)
+ const TensorSpec &spec)
{
- Tensor::UP tensor = createTensor(cells, dimensions);
+ Tensor::UP tensor = make_tensor(spec);
ValueType type(tensor->type());
test.getIndexEnv().addConstantValue(name,
std::move(type),
@@ -100,10 +88,10 @@ TEST_F("require that existing tensor constant is detected",
ExecFixture("constant(foo)"))
{
f.addTensor("foo",
- { {{{"x", "a"}}, 3},
- {{{"x", "b"}}, 5},
- {{{"x", "c"}}, 7} },
- { "x" });
+ TensorSpec("tensor(x{})")
+ .add({{"x","a"}}, 3)
+ .add({{"x","b"}}, 5)
+ .add({{"x","c"}}, 7));
EXPECT_TRUE(f.setup());
auto expect = make_tensor(TensorSpec("tensor(x{})")
.add({{"x","b"}}, 5)
diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp
index abe54906791..3aa2671fd25 100644
--- a/searchlib/src/tests/features/tensor/tensor_test.cpp
+++ b/searchlib/src/tests/features/tensor/tensor_test.cpp
@@ -15,7 +15,6 @@
#include <vespa/eval/tensor/default_tensor.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
-#include <vespa/eval/tensor/tensor_factory.h>
#include <vespa/eval/tensor/test/test_utils.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -36,7 +35,6 @@ using vespalib::tensor::DenseTensorCells;
using vespalib::tensor::Tensor;
using vespalib::tensor::TensorCells;
using vespalib::tensor::TensorDimensions;
-using vespalib::tensor::TensorFactory;
using vespalib::tensor::test::makeTensor;
using AVC = search::attribute::Config;
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
index 72cedb05f7c..1cac789ad21 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
@@ -31,7 +31,6 @@
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/data/slime/slime.h>
-#include <vespa/eval/tensor/serialization/slime_binary_format.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>