From b74a5049bb1bc8b7dce4167f4b639dfb3761c411 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 27 May 2019 12:24:06 +0000 Subject: remove tensor factory --- .../serialization/vespadocumentserializer_test.cpp | 54 +-- .../tensor_fieldvalue/tensor_fieldvalue_test.cpp | 33 +- .../vespa/document/fieldvalue/tensorfieldvalue.cpp | 14 +- eval/CMakeLists.txt | 1 - .../tensor_serialization_test.cpp | 402 +++++++++------------ .../tensor/tensor_slime_serialization/.gitignore | 1 - .../tensor_slime_serialization/CMakeLists.txt | 8 - .../tensor_slime_serialization_test.cpp | 185 ---------- eval/src/vespa/eval/tensor/CMakeLists.txt | 1 - .../vespa/eval/tensor/serialization/CMakeLists.txt | 1 - .../tensor/serialization/slime_binary_format.cpp | 100 ----- .../tensor/serialization/slime_binary_format.h | 26 -- .../tensor/serialization/typed_binary_format.cpp | 13 +- eval/src/vespa/eval/tensor/tensor_factory.cpp | 65 ---- eval/src/vespa/eval/tensor/tensor_factory.h | 27 -- .../src/tests/proton/attribute/attribute_test.cpp | 27 +- .../src/tests/proton/docsummary/docsummary.cpp | 28 +- .../docsummary/summaryfieldconverter_test.cpp | 23 +- .../imported_attribute_vector_test.cpp | 34 +- .../tensorattribute/tensorattribute_test.cpp | 137 +++---- .../src/tests/features/constant/constant_test.cpp | 26 +- .../src/tests/features/tensor/tensor_test.cpp | 2 - .../docsummary/summaryfieldconverter.cpp | 1 - 23 files changed, 321 insertions(+), 888 deletions(-) delete mode 100644 eval/src/tests/tensor/tensor_slime_serialization/.gitignore delete mode 100644 eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt delete mode 100644 eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp delete mode 100644 eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp delete mode 100644 eval/src/vespa/eval/tensor/serialization/slime_binary_format.h delete mode 100644 eval/src/vespa/eval/tensor/tensor_factory.cpp delete mode 100644 eval/src/vespa/eval/tensor/tensor_factory.h diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index c573eef6147..4b5c3e6f195 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -40,8 +40,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -53,10 +52,9 @@ using vespalib::Slime; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::slime::Cursor; +using vespalib::eval::TensorSpec; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorBuilder; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; +using vespalib::tensor::DefaultTensorEngine; using vespalib::compression::CompressionConfig; using namespace document; using std::string; @@ -828,13 +826,14 @@ TEST("Require that predicate deserialization matches Java") { namespace { -Tensor::UP -createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return vespalib::tensor::TensorFactory::create(cells, dimensions, builder); +Tensor::UP createTensor(const TensorSpec &spec) { + auto value = DefaultTensorEngine::ref().from_spec(spec); + Tensor *tensor = dynamic_cast(value.get()); + ASSERT_TRUE(tensor != nullptr); + value.release(); + return Tensor::UP(tensor); } - } TEST("Require that tensors can be serialized") @@ -846,12 +845,12 @@ TEST("Require that tensors can be serialized") nbostream stream; serializeAndDeserialize(noTensorValue, stream); stream.clear(); - emptyTensorValue = createTensor({}, {"x", "y"}); + emptyTensorValue = createTensor(TensorSpec("tensor(x{},y{})")); serializeAndDeserialize(emptyTensorValue, stream); stream.clear(); - twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3}, - {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}); + twoCellsTwoDimsValue = createTensor(TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3) + .add({{"x", "4"}, {"y", "5"}}, 7)); serializeAndDeserialize(twoCellsTwoDimsValue, stream); EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue); EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue); @@ -859,6 +858,7 @@ TEST("Require that tensors can be serialized") } + const int tensor_doc_type_id = 321; const string tensor_field_name = "my_tensor"; @@ -904,15 +904,15 @@ void checkDeserialization(const string &name, std::unique_ptr tensor) { deserializeAndCheck(data_dir + name + "__java", value); } + TEST("Require that tensor deserialization matches Java") { checkDeserialization("non_existing_tensor", std::unique_ptr()); - checkDeserialization("empty_tensor", createTensor({}, {"dimX", "dimY"})); + checkDeserialization("empty_tensor", createTensor(TensorSpec("tensor(dimX{},dimY{})"))); checkDeserialization("multi_cell_tensor", - createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 }, - {{{"dimX", "ccc"}, - {"dimY", "dddd"}}, 3.0}, - {{{"dimX", "e"},{"dimY","ff"}}, 5.0} }, - { "dimX", "dimY" })); + createTensor(TensorSpec("tensor(dimX{},dimY{})") + .add({{"dimX", "a"}, {"dimY", "bb"}}, 2.0) + .add({{"dimX", "ccc"}, {"dimY", "dddd"}}, 3.0) + .add({{"dimX", "e"}, {"dimY", "ff"}}, 5.0))); } struct TensorDocFixture { @@ -980,14 +980,14 @@ DeserializedTensorDoc::getTensor() const TEST("Require that wrong tensor type hides tensor") { TensorDocFixture f(tensor_doc_repo, - createTensor({ {{{"dimX", "a"},{"dimY", "bb"}}, 2.0 }, - {{{"dimX", "ccc"},{"dimY", "dddd"}}, 3.0}, - {{{"dimX", "e"},{"dimY","ff"}}, 5.0} }, - { "dimX", "dimY" })); + createTensor(TensorSpec("tensor(dimX{},dimY{})") + .add({{"dimX", "a"}, {"dimY", "bb"}}, 2.0) + .add({{"dimX", "ccc"}, {"dimY", "dddd"}}, 3.0) + .add({{"dimX", "e"}, {"dimY", "ff"}}, 5.0))); TensorDocFixture f1(tensor_doc_repo1, - createTensor({ {{{"dimX", "a"}}, 20.0 }, - {{{"dimX", "ccc"}}, 30.0} }, - { "dimX" })); + createTensor(TensorSpec("tensor(dimX{})") + .add({{"dimX", "a"}}, 20.0) + .add({{"dimX", "ccc"}}, 30.0))); DeserializedTensorDoc doc; doc.setup(tensor_doc_repo, f._blob); EXPECT_TRUE(doc.getTensor() != nullptr); diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp index dc91df0b5a0..9d2da9c983a 100644 --- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp +++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp @@ -9,8 +9,7 @@ LOG_SETUP("fieldvalue_test"); #include #include #include -#include -#include +#include #include #include @@ -19,6 +18,7 @@ using namespace document; using namespace vespalib::tensor; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; +using vespalib::tensor::DefaultTensorEngine; using vespalib::tensor::test::makeTensor; namespace @@ -27,10 +27,12 @@ namespace TensorDataType xSparseTensorDataType(ValueType::from_spec("tensor(x{})")); TensorDataType xySparseTensorDataType(ValueType::from_spec("tensor(x{},y{})")); -Tensor::UP -createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return vespalib::tensor::TensorFactory::create(cells, dimensions, builder); +Tensor::UP createTensor(const TensorSpec &spec) { + auto value = DefaultTensorEngine::ref().from_spec(spec); + Tensor *tensor = dynamic_cast(value.get()); + ASSERT_TRUE(tensor != nullptr); + value.release(); + return Tensor::UP(tensor); } std::unique_ptr @@ -54,10 +56,10 @@ TEST("require that TensorFieldValue can be assigned tensors and cloned") { TensorFieldValue noTensorValue(xySparseTensorDataType); TensorFieldValue emptyTensorValue(xySparseTensorDataType); TensorFieldValue twoCellsTwoDimsValue(xySparseTensorDataType); - emptyTensorValue = createTensor({}, {"x", "y"}); - twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3}, - {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}); + emptyTensorValue = createTensor(TensorSpec("tensor(x{},y{})")); + twoCellsTwoDimsValue = createTensor(TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3) + .add({{"x", "4"}, {"y", "5"}}, 7)); EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue); EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue); EXPECT_NOT_EQUAL(emptyTensorValue, noTensorValue); @@ -74,10 +76,9 @@ TEST("require that TensorFieldValue can be assigned tensors and cloned") { EXPECT_NOT_EQUAL(*twoClone, *noneClone); EXPECT_NOT_EQUAL(*twoClone, *emptyClone); TensorFieldValue twoCellsTwoDimsValue2(xySparseTensorDataType); - twoCellsTwoDimsValue2 = - createTensor({ {{{"y", "3"}}, 3}, - {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}); + twoCellsTwoDimsValue2 = createTensor(TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3) + .add({{"x", "4"}, {"y", "5"}}, 7)); EXPECT_NOT_EQUAL(*noneClone, twoCellsTwoDimsValue2); EXPECT_NOT_EQUAL(*emptyClone, twoCellsTwoDimsValue2); EXPECT_EQUAL(*twoClone, twoCellsTwoDimsValue2); @@ -87,8 +88,8 @@ TEST("require that TensorFieldValue::toString works") { TensorFieldValue tensorFieldValue(xSparseTensorDataType); EXPECT_EQUAL("{TensorFieldValue: null}", tensorFieldValue.toString()); - tensorFieldValue = createTensor({{{{"x","a"}}, 3}}, {"x"}); - EXPECT_EQUAL("{TensorFieldValue: {\"dimensions\":[\"x\"],\"cells\":[{\"address\":{\"x\":\"a\"},\"value\":3}]}}", tensorFieldValue.toString()); + tensorFieldValue = createTensor(TensorSpec("tensor(x{})").add({{"x", "a"}}, 3)); + EXPECT_EQUAL("{TensorFieldValue: spec(tensor(x{})) {\n [a]: 3\n}}", tensorFieldValue.toString()); } TEST("require that wrong tensor type for special case assign throws exception") diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp index f209358c901..9b318a39f0a 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp @@ -4,16 +4,16 @@ #include #include #include +#include #include -#include -#include +#include #include #include -using vespalib::slime::JsonFormat; using vespalib::tensor::Tensor; -using vespalib::tensor::SlimeBinaryFormat; +using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; +using Engine = vespalib::tensor::DefaultTensorEngine; using namespace vespalib::xml; namespace document { @@ -148,11 +148,7 @@ TensorFieldValue::print(std::ostream& out, bool verbose, (void) indent; out << "{TensorFieldValue: "; if (_tensor) { - auto slime = SlimeBinaryFormat::serialize(*_tensor); - vespalib::SimpleBuffer buf; - JsonFormat::encode(*slime, buf, true); - auto json = buf.get().make_string(); - out << json; + out << Engine::ref().to_spec(*_tensor).to_string(); } else { out << "null"; } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index d15f1c37d2d..4468ac46f61 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -44,7 +44,6 @@ vespa_define_module( src/tests/tensor/tensor_performance src/tests/tensor/tensor_remove_operation src/tests/tensor/tensor_serialization - src/tests/tensor/tensor_slime_serialization src/tests/tensor/typed_cells src/tests/tensor/vector_from_doubles_function diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index b7aa988775d..2c113e814e9 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -4,8 +4,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -14,6 +13,7 @@ #include using namespace vespalib::tensor; +using vespalib::eval::TensorSpec; using vespalib::nbostream; using ExpBuffer = std::vector; @@ -33,257 +33,181 @@ std::ostream &operator<<(std::ostream &out, const std::vector &rhs) } -namespace vespalib::tensor { - -static bool operator==(const Tensor &lhs, const Tensor &rhs) -{ - return lhs.equals(rhs); -} - -} - -template -void -checkDeserialize(vespalib::nbostream &stream, const Tensor &rhs) -{ - (void) stream; - (void) rhs; -} - -template <> -void -checkDeserialize(nbostream &stream, const Tensor &rhs) -{ - nbostream wrapStream(stream.peek(), stream.size()); - auto chk = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_EQUAL(0u, wrapStream.size()); - EXPECT_EQUAL(*chk, rhs); -} - -template -struct Fixture -{ - BuilderType _builder; - Fixture() : _builder() {} - - Tensor::UP createTensor(const TensorCells &cells) { - return TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); +//----------------------------------------------------------------------------- + +template +void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { + nbostream input(&exp[0], exp.size()); + std::vector cells; + TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(input, cells); + ASSERT_EQUAL(cells.size(), spec.cells().size()); + size_t i = 0; + for (const auto &cell: spec.cells()) { + EXPECT_EQUAL(cells[i++], cell.second.value); } + ASSERT_EQUAL(i, cells.size()); +} - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - Tensor::UP deserialize(nbostream &stream) { - BuilderType builder; - nbostream wrapStream(stream.peek(), stream.size()); - auto formatId = wrapStream.getInt1_4Bytes(); - ASSERT_EQUAL(formatId, 1u); // sparse format - SparseBinaryFormat::deserialize(wrapStream, builder); - EXPECT_TRUE(wrapStream.empty()); - auto ret = builder.build(); - checkDeserialize(stream, *ret); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const TensorCells &rhs, - const TensorDimensions &rhsDimensions) { - Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); - nbostream rhsStream; - serialize(rhsStream, *rhsTensor); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); +void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { + auto &engine = DefaultTensorEngine::ref(); + auto value = engine.from_spec(spec); + auto value_spec = engine.to_spec(*value); + nbostream actual; + engine.encode(*value, actual); + EXPECT_EQUAL(exp, actual); + auto decoded = engine.decode(actual); + auto decoded_spec = engine.to_spec(*decoded); + EXPECT_EQUAL(0u, actual.size()); + EXPECT_EQUAL(value_spec, decoded_spec); + if (value->type().is_dense()) { + TEST_DO(verify_cells_only(exp, value_spec)); + TEST_DO(verify_cells_only(exp, value_spec)); } -}; - -using SparseFixture = Fixture; - +} -template -void -testTensorSerialization(FixtureType &f) -{ - TEST_DO(f.assertSerialized({ 0x01, 0x00, 0x00 }, {}, {})); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, - {}, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, - {}, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, +//----------------------------------------------------------------------------- + +TEST("test tensor serialization for SparseTensor") { + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x00 }, + TensorSpec("tensor(x{})"))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x00 }, + TensorSpec("tensor(x{},y{})"))); + TEST_DO(verify_serialized({ 0x01, 0x01, 0x01, 0x78, 0x01, 0x01, 0x31, 0x40, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + TensorSpec("tensor(x{})") + .add({{"x", "1"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, - 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, - 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00 }, - { {{{"y","3"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, + 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x00, + 0x01, 0x33, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 }, + TensorSpec("tensor(x{},y{})") + .add({{"x", ""}, {"y", "3"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x32, 0x01, 0x34, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, + TensorSpec("tensor(x{},y{})") + .add({{"x", "2"}, {"y", "4"}}, 3))); + TEST_DO(verify_serialized({ 0x01, 0x02, 0x01, 0x78, 0x01, 0x79, 0x01, 0x01, 0x31, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, - { {{{"x","1"}}, 3} }, {"x", "y"})); + TensorSpec("tensor(x{},y{})") + .add({{"x", "1"}, {"y", ""}}, 3))); } -TEST_F("test tensor serialization for SparseTensor", SparseFixture) -{ - testTensorSerialization(f); +TEST("test tensor serialization for DenseTensor") { + TEST_DO(verify_serialized({0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("double"))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1])") + .add({{"x", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 0))); + TEST_DO(verify_serialized({0x02, 0x01, 0x01, 0x78, 0x02, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2])") + .add({{"x", 1}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x01, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[1])") + .add({{"x", 0}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x02, + 0x01, 0x79, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[2],y[1])") + .add({{"x", 1}, {"y", 0}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x01, + 0x01, 0x79, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[1],y[4])") + .add({{"x", 0}, {"y", 3}}, 3))); + TEST_DO(verify_serialized({0x02, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + TensorSpec("tensor(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - -struct DenseFixture -{ - Tensor::UP createTensor(const DenseTensorCells &cells) { - return TensorFactory::createDense(cells); - } - - void serialize(nbostream &stream, const Tensor &tensor) { - TypedBinaryFormat::serialize(stream, tensor); - } - - Tensor::UP deserialize(nbostream &stream) { - nbostream wrapStream(stream.peek(), stream.size()); - auto ret = TypedBinaryFormat::deserialize(wrapStream); - EXPECT_TRUE(wrapStream.size() == 0); - stream.adjustReadPos(stream.size()); - return ret; - } - void assertSerialized(const ExpBuffer &exp, const DenseTensorCells &rhs) { - assertSerialized(exp, SerializeFormat::DOUBLE, rhs); - } - template - void assertCellsOnly(const ExpBuffer &exp, const DenseTensorView & rhs) { - nbostream a(&exp[0], exp.size()); - std::vector v; - TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(a, v); - EXPECT_EQUAL(v.size(), rhs.cellsRef().size()); - for (size_t i(0); i < v.size(); i++) { - EXPECT_EQUAL(v[i], rhs.cellsRef()[i]); - } - } - void assertSerialized(const ExpBuffer &exp, SerializeFormat cellType, const DenseTensorCells &rhs) { - Tensor::UP rhsTensor(createTensor(rhs)); - nbostream rhsStream; - TypedBinaryFormat::serialize(rhsStream, *rhsTensor, cellType); - EXPECT_EQUAL(exp, rhsStream); - auto rhs2 = deserialize(rhsStream); - EXPECT_EQUAL(*rhs2, *rhsTensor); - - assertCellsOnly(exp, dynamic_cast(*rhs2)); - assertCellsOnly(exp, dynamic_cast(*rhs2)); - } -}; - - -TEST_F("test tensor serialization for DenseTensor", DenseFixture) { - TEST_DO(f.assertSerialized({0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 0}})); - TEST_DO(f.assertSerialized({0x02, 0x01, 0x01, 0x78, 0x02, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x01, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x02, - 0x01, 0x79, 0x01, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 1}, {"y", 0}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x01, - 0x01, 0x79, 0x04, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 0}, {"y", 3}}, 3}})); - TEST_DO(f.assertSerialized({0x02, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}, - {{{{"x", 2}, {"y", 4}}, 3}})); -} - -TEST_F("test 'float' cells", DenseFixture) { - TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, - 0x01, 0x79, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x40, 0x40, 0x00, 0x00 }, - SerializeFormat::FLOAT, { {{{"x",2}, {"y",4}}, 3} })); +TEST("test 'float' cells") { + TEST_DO(verify_serialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, + 0x01, 0x79, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x40, 0x40, 0x00, 0x00 }, + TensorSpec("tensor(x[3],y[5])") + .add({{"x", 2}, {"y", 4}}, 3))); } - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore b/eval/src/tests/tensor/tensor_slime_serialization/.gitignore deleted file mode 100644 index 9cb3b664d58..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_slime_serialization_test_app diff --git a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt b/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt deleted file mode 100644 index 9a9b8b7a436..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_slime_serialization_test_app TEST - SOURCES - tensor_slime_serialization_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_tensor_slime_serialization_test_app COMMAND eval_tensor_slime_serialization_test_app) diff --git a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp deleted file mode 100644 index e6e6c7de686..00000000000 --- a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace vespalib::tensor; - -template -struct Fixture -{ - BuilderType _builder; - Fixture() : _builder() {} - - Tensor::UP createTensor(const TensorCells &cells) { - return vespalib::tensor::TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); - } - - static inline uint32_t getTensorTypeId(); - - void assertSerialized(const vespalib::string &exp, const TensorCells &rhs, - const TensorDimensions &rhsDimensions) { - Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); - auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::Memory memory_exp(exp); - vespalib::Slime expSlime; - size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); - EXPECT_TRUE(used > 0); - EXPECT_EQUAL(expSlime, *slime); - } -}; - -template <> -uint32_t -Fixture::getTensorTypeId() { return 2u; } - - -using SparseFixture = Fixture; - - -namespace { -vespalib::string twoCellsJson[3] = -{ - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { y:'3'}, value: 4.0 }," - "{ address: { x:'1'}, value: 3.0 }" - "] }", - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }," - "{ address: { y:'3'}, value: 4.0 }" - "] }", - "{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }," - "{ address: { y:'3'}, value: 4.0 }" - "] }", -}; -} - - -template -void -testTensorSlimeSerialization(FixtureType &f) -{ - TEST_DO(f.assertSerialized("{ dimensions: [], cells: [] }", {}, {})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [] }", - {}, { "x" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [] }", - {}, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ]," - "cells: [" - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, { "x" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { }, value: 3.0 }" - "] }", - { {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { y: '3' }, value: 3.0 }" - "] }", - { {{{"y","3"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'2', y:'4'}, value: 3.0 }" - "] }", - { {{{"x","2"}, {"y", "4"}}, 3} }, { "x", "y" })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x:'1'}, value: 3.0 }" - "] }", - { {{{"x","1"}}, 3} }, {"x", "y"})); - TEST_DO(f.assertSerialized(twoCellsJson[FixtureType::getTensorTypeId()], - { {{{"x","1"}}, 3}, {{{"y","3"}}, 4} }, - {"x", "y"})); -} - -TEST_F("test tensor slime serialization for SparseTensor", SparseFixture) -{ - testTensorSlimeSerialization(f); -} - - -struct DenseFixture -{ - DenseFixture() {} - - Tensor::UP createTensor(const DenseTensorCells &cells) { - return vespalib::tensor::TensorFactory::createDense(cells); - } - - void assertSerialized(const vespalib::string &exp, - const DenseTensorCells &rhs) { - Tensor::UP rhsTensor(createTensor(rhs)); - auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::Memory memory_exp(exp); - vespalib::Slime expSlime; - size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); - EXPECT_TRUE(used > 0); - EXPECT_EQUAL(expSlime, *slime); - } -}; - - -TEST_F("test tensor slime serialization for DenseTensor", DenseFixture) -{ - TEST_DO(f.assertSerialized("{ dimensions: [], cells: [" - "{ address: { }, value: 0.0 }" - "] }", {})); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ], cells: [" - "{ address: { x: '0' }, value: 0.0 }" - "] }", - { {{{"x",0}}, 0} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ], cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }" - "] }", - { {{{"x",0},{"y",0}}, 0} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x' ]," - "cells: [" - "{ address: { x: '0' }, value: 0.0 }," - "{ address: { x: '1' }, value: 3.0 }" - "] }", - { {{{"x",1}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 3.0 }" - "] }", - { {{{"x",0},{"y",0}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }," - "{ address: { x: '1', y: '0' }, value: 3.0 }" - "] }", - { {{{"x",1},{"y", 0}}, 3} })); - TEST_DO(f.assertSerialized("{ dimensions: [ 'x', 'y' ]," - " cells: [" - "{ address: { x: '0', y: '0' }, value: 0.0 }," - "{ address: { x: '0', y: '1' }, value: 0.0 }," - "{ address: { x: '0', y: '2' }, value: 0.0 }," - "{ address: { x: '0', y: '3' }, value: 3.0 }" - "] }", - { {{{"x",0},{"y",3}}, 3} })); -} - - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index ca4d275ea44..4e9940d3c0a 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(eval_tensor OBJECT tensor.cpp tensor_address.cpp tensor_apply.cpp - tensor_factory.cpp tensor_mapper.cpp wrapped_simple_tensor.cpp ) diff --git a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt index cd3cd455e55..fc9ac64ea68 100644 --- a/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/serialization/CMakeLists.txt @@ -3,6 +3,5 @@ vespa_add_library(eval_tensor_serialization OBJECT SOURCES sparse_binary_format.cpp dense_binary_format.cpp - slime_binary_format.cpp typed_binary_format.cpp ) diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp deleted file mode 100644 index ece3c2e4a07..00000000000 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "slime_binary_format.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace vespalib::tensor { - - -using slime::Inserter; -using slime::SlimeInserter; -using slime::Cursor; -using slime::ObjectInserter; - -namespace { - -Memory memory_address("address"); -Memory memory_cells("cells"); -Memory memory_dimensions("dimensions"); -Memory memory_value("value"); - -void writeTensorAddress(Cursor &cursor, const TensorAddress &value) { - ObjectInserter addressInserter(cursor, memory_address); - Cursor &addressCursor = addressInserter.insertObject(); - for (const auto &elem : value.elements()) { - Memory dimension(elem.dimension()); - Memory label(elem.label()); - addressCursor.setString(dimension, label); - } -} - -} - -class SlimeBinaryFormatSerializer : public TensorVisitor -{ - Cursor &_tensor; // cursor for whole tensor - Cursor &_dimensions; // cursor for dimensions array - Cursor &_cells; // cursor for cells array -public: - SlimeBinaryFormatSerializer(Inserter &inserter); - virtual ~SlimeBinaryFormatSerializer() override; - virtual void visit(const TensorAddress &address, double value) override; - void serialize(const Tensor &tensor); -}; - -SlimeBinaryFormatSerializer::SlimeBinaryFormatSerializer(Inserter &inserter) - : _tensor(inserter.insertObject()), - _dimensions(_tensor.setArray(memory_dimensions)), - _cells(_tensor.setArray(memory_cells)) -{ -} - - -SlimeBinaryFormatSerializer::~SlimeBinaryFormatSerializer() = default; - -void -SlimeBinaryFormatSerializer::visit(const TensorAddress &address, double value) -{ - Cursor &cellCursor = _cells.addObject(); - writeTensorAddress(cellCursor, address); - cellCursor.setDouble(memory_value, value); -} - - -void -SlimeBinaryFormatSerializer::serialize(const Tensor &tensor) -{ - eval::ValueType type(tensor.type()); - for (const auto & dimension : type.dimensions()) { - _dimensions.addString(Memory(dimension.name)); - } - tensor.accept(*this); -} - - -void -SlimeBinaryFormat::serialize(Inserter &inserter, const Tensor &tensor) -{ - SlimeBinaryFormatSerializer serializer(inserter); - serializer.serialize(tensor); -} - - -std::unique_ptr -SlimeBinaryFormat::serialize(const Tensor &tensor) -{ - auto slime = std::make_unique(); - SlimeInserter inserter(*slime); - serialize(inserter, tensor); - return slime; -} - - -} diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h deleted file mode 100644 index c9e9ff2c3e9..00000000000 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include - -namespace vespalib { class Slime; } - -namespace vespalib::slime { struct Inserter; } - -namespace vespalib::tensor { - -class Tensor; -class TensorBuilder; - -/** - * Class for serializing a tensor into a slime object. - */ -class SlimeBinaryFormat -{ -public: - static void serialize(slime::Inserter &inserter, const Tensor &tensor); - static std::unique_ptr serialize(const Tensor &tensor); -}; - -} diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index 4ca037e82a4..7be4db46372 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -25,9 +25,9 @@ namespace { constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u; constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u; constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u; -constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; //Future +constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; constexpr uint32_t DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6u; -constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; //Future +constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; constexpr uint32_t DOUBLE_VALUE_TYPE = 0; constexpr uint32_t FLOAT_VALUE_TYPE = 1; @@ -91,10 +91,11 @@ TypedBinaryFormat::deserialize(nbostream &stream) if (formatId == DENSE_BINARY_FORMAT_TYPE) { return DenseBinaryFormat(SerializeFormat::DOUBLE).deserialize(stream); } - if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) { - return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserialize(stream); - } - if (formatId == MIXED_BINARY_FORMAT_TYPE) { + if ((formatId == SPARSE_BINARY_FORMAT_WITH_CELLTYPE) || + (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) || + (formatId == MIXED_BINARY_FORMAT_TYPE) || + (formatId == MIXED_BINARY_FORMAT_WITH_CELLTYPE)) + { stream.adjustReadPos(read_pos - stream.rp()); return std::make_unique(eval::SimpleTensor::decode(stream)); } diff --git a/eval/src/vespa/eval/tensor/tensor_factory.cpp b/eval/src/vespa/eval/tensor/tensor_factory.cpp deleted file mode 100644 index 0b7fa3b9c2e..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_factory.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor.h" -#include "tensor_factory.h" -#include "tensor_builder.h" -#include - -namespace vespalib::tensor { - -std::unique_ptr -TensorFactory::create(const TensorCells &cells, TensorBuilder &builder) { - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - builder.define_dimension(dimension); - } - } - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - const auto &label = addressElem.second; - builder.add_label(builder.define_dimension(dimension), label); - } - builder.add_cell(cell.second); - } - return builder.build(); -} - - -std::unique_ptr -TensorFactory::create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder) { - for (const auto &dimension : dimensions) { - builder.define_dimension(dimension); - } - return create(cells, builder); -} - - -std::unique_ptr -TensorFactory::createDense(const DenseTensorCells &cells) -{ - std::map dimensionSizes; - DenseTensorBuilder builder; - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - dimensionSizes[addressElem.first] = std::max(dimensionSizes[addressElem.first], (addressElem.second + 1)); - } - } - std::map dimensionEnums; - for (const auto &dimensionElem : dimensionSizes) { - dimensionEnums[dimensionElem.first] = builder.defineDimension(dimensionElem.first, dimensionElem.second); - } - for (const auto &cell : cells) { - for (const auto &addressElem : cell.first) { - const auto &dimension = addressElem.first; - size_t label = addressElem.second; - builder.addLabel(dimensionEnums[dimension], label); - } - builder.addCell(cell.second); - } - return builder.build(); -} - - -} diff --git a/eval/src/vespa/eval/tensor/tensor_factory.h b/eval/src/vespa/eval/tensor/tensor_factory.h deleted file mode 100644 index 5364c28c8ff..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_factory.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "types.h" - -namespace vespalib::tensor { - - -class Tensor; -class TensorBuilder; - -/** - * A factory for creating tensors based on stl structures (TensorCells and - * TensorDimensions) in unit tests. - */ -class TensorFactory { -public: - static std::unique_ptr - create(const TensorCells &cells, TensorBuilder &builder); - static std::unique_ptr - create(const TensorCells &cells, const TensorDimensions &dimensions, TensorBuilder &builder); - static std::unique_ptr - createDense(const DenseTensorCells &cells); -}; - -} diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 992a9c56dfb..3734d2fe1dc 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -7,9 +7,8 @@ #include #include #include -#include #include -#include +#include #include #include #include @@ -75,9 +74,9 @@ using search::tensor::TensorAttribute; using search::test::DirectoryHandler; using std::string; using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; +using vespalib::tensor::DefaultTensorEngine; using AVConfig = search::attribute::Config; using AVBasicType = search::attribute::BasicType; @@ -612,13 +611,11 @@ TEST_F("require that filter attribute manager can return flushed serial number", namespace { -Tensor::UP -createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return vespalib::tensor::TensorFactory::create(cells, dimensions, builder); +Tensor::UP make_tensor(const TensorSpec &spec) { + auto tensor = DefaultTensorEngine::ref().from_spec(spec); + return Tensor::UP(dynamic_cast(tensor.release())); } - AttributeVector::SP createTensorAttribute(Fixture &f) { AVConfig cfg(AVBasicType::TENSOR); @@ -650,8 +647,8 @@ TEST_F("Test that we can use attribute writer to write to tensor attribute", AttributeVector::SP a1 = createTensorAttribute(f); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}); + auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "4"}, {"y", "5"}}, 7)); Document::UP doc = createTensorPutDoc(builder, *tensor); f.put(1, *doc, 1); EXPECT_EQUAL(2u, a1->getNumDocs()); @@ -668,8 +665,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) AttributeVector::SP a1 = createTensorAttribute(f); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = createTensor({ {{{"x", "6"}, {"y", "7"}}, 9} }, - {"x", "y"}); + auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "6"}, {"y", "7"}}, 9)); Document::UP doc = createTensorPutDoc(builder, *tensor); f.put(1, *doc, 1); EXPECT_EQUAL(2u, a1->getNumDocs()); @@ -682,8 +679,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) const document::DocumentType &dt(builder.getDocumentType()); DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("doc::1")); - auto new_tensor = createTensor({ {{{"x", "8"}, {"y", "9"}}, 11} }, - {"x", "y"}); + auto new_tensor = make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "8"}, {"y", "9"}}, 11)); TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); TensorFieldValue new_value(xySparseTensorDataType); new_value = new_tensor->clone(); diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 38b85594a62..e8152161faa 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -2,9 +2,9 @@ #include #include -#include +#include +#include #include -#include #include #include #include @@ -55,10 +55,9 @@ using search::index::schema::CollectionType; using storage::spi::Timestamp; using vespa::config::search::core::ProtonConfig; using vespa::config::content::core::BucketspacesConfig; +using vespalib::eval::TensorSpec; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; -using vespalib::tensor::TensorFactory; +using vespalib::tensor::DefaultTensorEngine; using namespace vespalib::slime; typedef std::unique_ptr GeneralResultPtr; @@ -137,10 +136,9 @@ getDocTypeName() return "searchdocument"; } -Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return TensorFactory::create(cells, dimensions, builder); +Tensor::UP make_tensor(const TensorSpec &spec) { + auto tensor = DefaultTensorEngine::ref().from_spec(spec); + return Tensor::UP(dynamic_cast(tensor.release())); } vespalib::string asVstring(vespalib::Memory str) { @@ -750,7 +748,8 @@ Test::requireThatAttributesAreUsed() endElement(). endField(). startAttributeField("bj"). - addTensor(createTensor({ {{{"x","f"},{"y","g"}}, 3} }, { "x", "y"})). + addTensor(make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "f"}, {"y", "g"}}, 3))). endField(). endDocument(), 2); @@ -776,7 +775,8 @@ Test::requireThatAttributesAreUsed() "bh:[{item:40.4,weight:4},{item:50.5,weight:5}]," "bi:[{item:'quux',weight:7},{item:'qux',weight:6}]," "bj:'0x01020178017901016601674008000000000000'}", *rep, 0, true)); - TEST_DO(assertTensor(createTensor({ {{{"x","f"},{"y","g"}}, 3} }, { "x", "y"}), + TEST_DO(assertTensor(make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "f"}, {"y", "g"}}, 3)), "bj", *rep, 0, rclass)); // empty doc @@ -790,13 +790,15 @@ Test::requireThatAttributesAreUsed() attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(bjAttr->getNamePrefix()), [&]() { - bjTensorAttr->setTensor(3, *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"})); + bjTensorAttr->setTensor(3, *make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "a"}, {"y", "b"}}, 4))); bjTensorAttr->commit(); }); attributeFieldWriter.sync(); DocsumReply::UP rep2 = dc._ddb->getDocsums(req); - TEST_DO(assertTensor(createTensor({ {{{"x","a"},{"y","b"}}, 4} }, { "x", "y"}), + TEST_DO(assertTensor(make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "a"}, {"y", "b"}}, 4)), "bj", *rep2, 1, rclass)); DocsumRequest req3; diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp index b82fec85d47..cc6eef14fd6 100644 --- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp +++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp @@ -47,8 +47,7 @@ #include #include #include -#include -#include +#include #include using document::Annotation; @@ -95,12 +94,12 @@ using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; using vespalib::Slime; using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; using vespalib::geo::ZCurve; using vespalib::slime::Cursor; using vespalib::string; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; +using vespalib::tensor::DefaultTensorEngine; using namespace search::docsummary; @@ -676,11 +675,9 @@ Test::requireThatPredicateIsPrinted() SFC::convertSummaryField(false, *doc.getValue("predicate")).get()); } - -Tensor::UP -createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return vespalib::tensor::TensorFactory::create(cells, dimensions, builder); +Tensor::UP make_tensor(const TensorSpec &spec) { + auto tensor = DefaultTensorEngine::ref().from_spec(spec); + return Tensor::UP(dynamic_cast(tensor.release())); } void @@ -688,14 +685,14 @@ Test::requireThatTensorIsNotConverted() { TensorDataType tensorDataType(ValueType::from_spec("tensor(x{},y{})")); TensorFieldValue tensorFieldValue(tensorDataType); - tensorFieldValue = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}); + tensorFieldValue = make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "4"}, {"y", "5"}}, 7)); Document doc(getDocType(), DocumentId("doc:scheme:")); doc.setRepo(*_documentRepo); doc.setValue("tensor", tensorFieldValue); - TEST_CALL(checkTensor(createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} }, - {"x", "y"}), + TEST_CALL(checkTensor(make_tensor(TensorSpec("tensor(x{},y{})") + .add({{"x", "4"}, {"y", "5"}}, 7)), SFC::convertSummaryField(false, *doc.getValue("tensor")).get())); doc.setValue("tensor", TensorFieldValue()); diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 1e24a784fd9..cfa05b7a765 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include #include -#include +#include #include #include #include @@ -13,11 +12,17 @@ using search::attribute::IAttributeVector; using search::tensor::ITensorAttribute; using search::tensor::TensorAttribute; using vespalib::eval::ValueType; -using vespalib::tensor::DenseTensorCells; +using vespalib::eval::TensorSpec; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; -using vespalib::tensor::TensorFactory; +using vespalib::tensor::DefaultTensorEngine; + +Tensor::UP createTensor(const TensorSpec &spec) { + auto value = DefaultTensorEngine::ref().from_spec(spec); + Tensor *tensor = dynamic_cast(value.get()); + ASSERT_TRUE(tensor != nullptr); + value.release(); + return Tensor::UP(tensor); +} namespace search::attribute { @@ -482,32 +487,23 @@ TEST("onSerializeForDescendingSort() is forwarded with remapped LID to target ve } struct TensorAttrFixture : Fixture { - vespalib::tensor::DefaultTensor::builder builder; std::shared_ptr tensor1; std::shared_ptr tensor2; TensorAttrFixture(bool dense) : Fixture(), - builder(), tensor1(), tensor2() { setup(dense); } - Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, builder); - } - Tensor::UP createDenseTensor(const DenseTensorCells &cells) const { - return TensorFactory::createDense(cells); - } void setup(bool dense) { if (dense) { - tensor1 = createDenseTensor({ {{{"x",1}}, 11} }); - tensor2 = createDenseTensor({ {{{"x",0}}, 12}, {{{"x", 1}}, 0} }); + tensor1 = createTensor(TensorSpec("tensor(x[2])").add({{"x", 1}}, 11)); + tensor2 = createTensor(TensorSpec("tensor(x[2])").add({{"x", 0}}, 12).add({{"x", 1}}, 0)); } else { - tensor1 = createTensor({ {{{"x","1"}}, 11} }, { "x" }); - tensor2 = createTensor({ {{{"x","0"}}, 12} }, { "x" }); + tensor1 = createTensor(TensorSpec("tensor(x{})").add({{"x", "1"}}, 11)); + tensor2 = createTensor(TensorSpec("tensor(x{})").add({{"x", "0"}}, 12)); } const std::vector>> mappings = { {DocId(2), dummy_gid(3), DocId(3), tensor1 }, diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index cbbaa518b16..e0ec4a6fd06 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -6,8 +6,9 @@ #include #include #include -#include -#include +#include +#include +#include #include #include #include @@ -21,11 +22,10 @@ using search::tensor::GenericTensorAttribute; using search::AttributeGuard; using search::AttributeVector; using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::DenseTensorCells; -using vespalib::tensor::TensorDimensions; -using vespalib::tensor::TensorFactory; +using vespalib::tensor::DenseTensor; +using vespalib::tensor::DefaultTensorEngine; namespace vespalib { namespace tensor { @@ -41,6 +41,17 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs) vespalib::string sparseSpec("tensor(x{},y{})"); vespalib::string denseSpec("tensor(x[2],y[3])"); +Tensor::UP createTensor(const TensorSpec &spec) { + auto value = DefaultTensorEngine::ref().from_spec(spec); + if (value->is_double()) { + return Tensor::UP(new DenseTensor(ValueType::double_type(), {value->as_double()})); + } + Tensor *tensor = dynamic_cast(value.get()); + ASSERT_TRUE(tensor != nullptr); + value.release(); + return Tensor::UP(tensor); +} + struct Fixture { using BasicType = search::attribute::BasicType; @@ -52,7 +63,6 @@ struct Fixture vespalib::string _typeSpec; std::shared_ptr _tensorAttr; std::shared_ptr _attr; - vespalib::tensor::DefaultTensor::builder _builder; bool _denseTensors; bool _useDenseTensorAttribute; @@ -63,7 +73,6 @@ struct Fixture _typeSpec(typeSpec), _tensorAttr(), _attr(), - _builder(), _denseTensors(false), _useDenseTensorAttribute(useDenseTensorAttribute) { @@ -85,17 +94,6 @@ struct Fixture } } - Tensor::UP createTensor(const TensorCells &cells) { - return TensorFactory::create(cells, _builder); - } - Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); - } - Tensor::UP createDenseTensor(const DenseTensorCells &cells) const { - return TensorFactory::createDense(cells); - } - void ensureSpace(uint32_t docId) { while (_attr->getNumDocs() <= docId) { uint32_t newDocId = 0u; @@ -138,19 +136,9 @@ struct Fixture } void - assertGetTensor(const TensorCells &expCells, - const TensorDimensions &expDimensions, - uint32_t docId) - { - Tensor::UP expTensor = createTensor(expCells, expDimensions); - assertGetTensor(*expTensor, docId); - } - - void - assertGetDenseTensor(const DenseTensorCells &expCells, - uint32_t docId) + assertGetTensor(const TensorSpec &expSpec, uint32_t docId) { - Tensor::UP expTensor = createDenseTensor(expCells); + Tensor::UP expTensor = createTensor(expSpec); assertGetTensor(*expTensor, docId); } @@ -166,72 +154,27 @@ struct Fixture EXPECT_TRUE(loadok); } - bool isUnbound(const vespalib::string &dimensionName) const - { - ValueType type = _cfg.tensorType(); - for (const auto &dim : type.dimensions()) { - if (dim.name == dimensionName && !dim.is_bound()) { - return true; - } - } - return false; - } - Tensor::UP expDenseTensor3() const { - if (isUnbound("x")) { - if (isUnbound("y")) { - return createDenseTensor({ {{{"x",0},{"y",1}}, 11} }); - } - return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, - {{{"x",0},{"y",2}}, 0} }); - } else if (isUnbound("y")) { - return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, - {{{"x",1},{"y",0}}, 0} }); - } - return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, - {{{"x",1},{"y",2}}, 0} }); + return createTensor(TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 1}}, 11) + .add({{"x", 1}, {"y", 2}}, 0)); } Tensor::UP expDenseFillTensor() const { - if (isUnbound("x")) { - if (isUnbound("y")) { - return createDenseTensor({ {{{"x",0},{"y",0}}, 5} }); - } - return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, - {{{"x",0},{"y",2}}, 0} }); - } else if (isUnbound("y")) { - return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, - {{{"x",1},{"y",0}}, 0} }); - } - return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, - {{{"x",1},{"y",2}}, 0} }); + return createTensor(TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 0}}, 5) + .add({{"x", 1}, {"y", 2}}, 0)); } Tensor::UP expEmptyDenseTensor() const { - if (isUnbound("x")) { - if (isUnbound("y")) { - return createDenseTensor({ {{{"x",0},{"y",0}}, 0} }); - } - return createDenseTensor({ {{{"x",0},{"y",2}}, 0} }); - } else if (isUnbound("y")) { - return createDenseTensor({ {{{"x",1},{"y",0}}, 0} }); - } - return createDenseTensor({ {{{"x",1},{"y",2}}, 0} }); + return createTensor(TensorSpec(denseSpec)); } vespalib::string expEmptyDenseTensorSpec() const { - if (isUnbound("x")) { - if (isUnbound("y")) { - return "tensor(x[1],y[1])"; - } - return "tensor(x[1],y[3])"; - } else if (isUnbound("y")) { - return "tensor(x[2],y[1])"; - } - return "tensor(x[2],y[3])"; + return denseSpec; } void testEmptyAttribute(); @@ -257,7 +200,7 @@ Fixture::testSetTensorValue() EXPECT_EQUAL(5u, _attr->getNumDocs()); EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); TEST_DO(assertGetNoTensor(4)); - EXPECT_EXCEPTION(setTensor(4, *createTensor({}, {})), + EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))), WrongTensorTypeException, "but other tensor type is 'double'"); TEST_DO(assertGetNoTensor(4)); @@ -267,9 +210,11 @@ Fixture::testSetTensorValue() setTensor(3, *expDenseTensor3()); TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); } else { - TEST_DO(assertGetTensor({}, {"x", "y"}, 4)); - setTensor(3, *createTensor({ {{}, 11} }, { "x", "y"})); - TEST_DO(assertGetTensor({ {{}, 11} }, { "x", "y"}, 3)); + TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); + setTensor(3, *createTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 11))); + TEST_DO(assertGetTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 11), 3)); } TEST_DO(assertGetNoTensor(2)); TEST_DO(clearTensor(3)); @@ -284,7 +229,8 @@ Fixture::testSaveLoad() if (_denseTensors) { setTensor(3, *expDenseTensor3()); } else { - setTensor(3, *createTensor({ {{{"y","1"}}, 11} }, { "x", "y"})); + setTensor(3, *createTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11))); } TEST_DO(save()); TEST_DO(load()); @@ -294,8 +240,9 @@ Fixture::testSaveLoad() TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); } else { - TEST_DO(assertGetTensor({ {{{"y","1"}}, 11} }, { "x", "y"}, 3)); - TEST_DO(assertGetTensor({}, {"x", "y"}, 4)); + TEST_DO(assertGetTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11), 3)); + TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); } TEST_DO(assertGetNoTensor(2)); } @@ -310,9 +257,11 @@ Fixture::testCompaction() } ensureSpace(4); Tensor::UP emptytensor = _tensorAttr->getEmptyTensor(); - Tensor::UP emptyxytensor = createTensor({}, {"x", "y"}); - Tensor::UP simpletensor = createTensor({ {{{"y","1"}}, 11} }, { "x", "y"}); - Tensor::UP filltensor = createTensor({ {{}, 5} }, { "x", "y"}); + Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec)); + Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11)); + Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 5)); if (_denseTensors) { emptyxytensor = expEmptyDenseTensor(); simpletensor = expDenseTensor3(); diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp index f9d1736ba9e..e558839d2cf 100644 --- a/searchlib/src/tests/features/constant/constant_test.cpp +++ b/searchlib/src/tests/features/constant/constant_test.cpp @@ -7,9 +7,8 @@ #include #include #include -#include +#include #include -#include using search::feature_t; using namespace search::fef; @@ -22,21 +21,11 @@ using vespalib::eval::DoubleValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; -using vespalib::tensor::DenseTensorCells; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorCells; -using vespalib::tensor::TensorDimensions; -using vespalib::tensor::TensorFactory; namespace { -Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return TensorFactory::create(cells, dimensions, builder); -} - Tensor::UP make_tensor(const TensorSpec &spec) { auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast(tensor.release())); @@ -72,10 +61,9 @@ struct ExecFixture return extractDouble(docId); } void addTensor(const vespalib::string &name, - const TensorCells &cells, - const TensorDimensions &dimensions) + const TensorSpec &spec) { - Tensor::UP tensor = createTensor(cells, dimensions); + Tensor::UP tensor = make_tensor(spec); ValueType type(tensor->type()); test.getIndexEnv().addConstantValue(name, std::move(type), @@ -100,10 +88,10 @@ TEST_F("require that existing tensor constant is detected", ExecFixture("constant(foo)")) { f.addTensor("foo", - { {{{"x", "a"}}, 3}, - {{{"x", "b"}}, 5}, - {{{"x", "c"}}, 7} }, - { "x" }); + TensorSpec("tensor(x{})") + .add({{"x","a"}}, 3) + .add({{"x","b"}}, 5) + .add({{"x","c"}}, 7)); EXPECT_TRUE(f.setup()); auto expect = make_tensor(TensorSpec("tensor(x{})") .add({{"x","b"}}, 5) diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index abe54906791..3aa2671fd25 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -36,7 +35,6 @@ using vespalib::tensor::DenseTensorCells; using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; -using vespalib::tensor::TensorFactory; using vespalib::tensor::test::makeTensor; using AVC = search::attribute::Config; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 72cedb05f7c..1cac789ad21 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3