diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-03-01 09:35:56 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-03-01 09:35:56 +0000 |
commit | 2aee5c3732b9b1ea5181031a01f18645c63b4b3e (patch) | |
tree | f5fcd2171f1c746d819b8d74c169edfcb21c7556 /document | |
parent | 930cca0f9a44cd43762558b66353e0099d7a173f (diff) |
Add cross-language serialization tests for tensor updates.
Diffstat (limited to 'document')
-rw-r--r-- | document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java | 78 | ||||
-rw-r--r-- | document/src/tests/data/serialize-tensor-update-cpp.dat | bin | 0 -> 231 bytes | |||
-rw-r--r-- | document/src/tests/data/serialize-tensor-update-java.dat | bin | 0 -> 231 bytes | |||
-rw-r--r-- | document/src/tests/documentupdatetestcase.cpp | 86 | ||||
-rw-r--r-- | document/src/vespa/document/update/documentupdate.cpp | 7 | ||||
-rw-r--r-- | document/src/vespa/document/update/documentupdate.h | 3 |
6 files changed, 169 insertions, 5 deletions
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index b86094e5a4d..0712adca3a5 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -13,9 +13,11 @@ import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.document.serialization.DocumentUpdateFlags; -import com.yahoo.document.serialization.DocumentUpdateWriter; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.TensorAddUpdate; +import com.yahoo.document.update.TensorModifyUpdate; +import com.yahoo.document.update.TensorRemoveUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; @@ -402,12 +404,14 @@ public class DocumentUpdateTestCase { GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f); upd.serialize(DocumentSerializerFactory.create42(buf)); + buf.flip(); - int size = buf.position(); - buf.position(0); + writeBufferToFile(buf, "src/tests/data/serializeupdatejava.dat"); + } - FileOutputStream fos = new FileOutputStream("src/tests/data/serializeupdatejava.dat"); - fos.write(buf.array(), 0, size); + private static void writeBufferToFile(GrowableByteBuffer buf, String fileName) throws IOException { + FileOutputStream fos = new FileOutputStream(fileName); + fos.write(buf.array(), 0, buf.remaining()); fos.close(); } @@ -787,4 +791,68 @@ public class DocumentUpdateTestCase { return result; } + private static class TensorUpdateSerializeFixture { + private DocumentTypeManager docMan; + private DocumentType docType; + + public TensorUpdateSerializeFixture() { + docMan = new DocumentTypeManager(); + docType = new DocumentType("test"); + docType.addHeaderField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})"))); + docType.addHeaderField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])"))); + docMan.registerDocumentType(docType); + } + + Field getField(String name) { + return docType.getField(name); + } + + TensorFieldValue createTensor() { + return new TensorFieldValue(Tensor.from("tensor(x{})", "{{x:2}:5, {x:3}:7}")); + } + + DocumentUpdate createUpdate() { + var result = new DocumentUpdate(docType, "id:test:test::0"); + + result.addFieldUpdate(FieldUpdate.create(getField("sparse_tensor")) + .addValueUpdate(new AssignValueUpdate(createTensor())) + .addValueUpdate(new TensorAddUpdate(createTensor())) + .addValueUpdate(new TensorRemoveUpdate(createTensor()))); + + result.addFieldUpdate(FieldUpdate.create(getField("dense_tensor")) + .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.REPLACE, createTensor())) + .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.ADD, createTensor())) + .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.MULTIPLY, createTensor()))); + return result; + } + + void serializeUpdateToFile(DocumentUpdate update, String fileName) throws IOException { + GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f); + update.serialize(DocumentSerializerFactory.createHead(buf)); + buf.flip(); + + writeBufferToFile(buf, fileName); + } + + DocumentUpdate deserializeUpdateFromFile(String fileName) throws IOException { + byte[] data = DocumentTestCase.readFile(fileName); + DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); + return new DocumentUpdate(buf); + } + } + + @Test + public void tensor_update_file_cpp_can_be_deserialized() throws IOException { + var f = new TensorUpdateSerializeFixture(); + var update = f.deserializeUpdateFromFile("src/tests/data/serialize-tensor-update-cpp.dat"); + assertEquals(f.createUpdate(), update); + } + + @Test + public void generate_serialized_tensor_update_file_java() throws IOException { + var f = new TensorUpdateSerializeFixture(); + var update = f.createUpdate(); + f.serializeUpdateToFile(update, "src/tests/data/serialize-tensor-update-java.dat"); + } + } diff --git a/document/src/tests/data/serialize-tensor-update-cpp.dat b/document/src/tests/data/serialize-tensor-update-cpp.dat Binary files differnew file mode 100644 index 00000000000..ad0e9d706b0 --- /dev/null +++ b/document/src/tests/data/serialize-tensor-update-cpp.dat diff --git a/document/src/tests/data/serialize-tensor-update-java.dat b/document/src/tests/data/serialize-tensor-update-java.dat Binary files differnew file mode 100644 index 00000000000..ad0e9d706b0 --- /dev/null +++ b/document/src/tests/data/serialize-tensor-update-java.dat diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index b2a9426a502..7164ee106e5 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -27,6 +27,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <fcntl.h> +#include <fstream> #include <gtest/gtest.h> #include <unistd.h> @@ -1044,6 +1045,91 @@ TEST(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_spar vespalib::IllegalStateException); } +struct TensorUpdateSerializeFixture { + std::unique_ptr<DocumentTypeRepo> repo; + const DocumentType &docType; + + const TensorDataType &extractTensorDataType(const vespalib::string &fieldName) { + const auto &dataType = docType.getField(fieldName).getDataType(); + return dynamic_cast<const TensorDataType &>(dataType); + } + + TensorUpdateSerializeFixture() + : repo(makeDocumentTypeRepo()), + docType(*repo->getDocumentType("test")) + { + } + + std::unique_ptr<DocumentTypeRepo> makeDocumentTypeRepo() { + config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(222, "test", + Struct("test.header") + .addTensorField("sparse_tensor", "tensor(x{})") + .addTensorField("dense_tensor", "tensor(x[4])"), + Struct("testdoc.body")); + return std::make_unique<DocumentTypeRepo>(builder.config()); + } + + std::unique_ptr<TensorFieldValue> makeTensor() { + return makeTensorFieldValue(TensorSpec("tensor(x{})").add({{"x", "2"}}, 5) + .add({{"x", "3"}}, 7), + extractTensorDataType("sparse_tensor")); + } + + const Field &getField(const vespalib::string &name) { + return docType.getField(name); + } + + DocumentUpdate::UP makeUpdate() { + auto result = std::make_unique<DocumentUpdate> + (*repo, docType, DocumentId("id:test:test::0")); + + result->addUpdate(FieldUpdate(getField("sparse_tensor")) + .addUpdate(AssignValueUpdate(*makeTensor())) + .addUpdate(TensorAddUpdate(makeTensor())) + .addUpdate(TensorRemoveUpdate(makeTensor()))); + result->addUpdate(FieldUpdate(getField("dense_tensor")) + .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, makeTensor())) + .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::ADD, makeTensor())) + .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, makeTensor()))); + return result; + } + + void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { + ByteBuffer::UP buf = serializeHEAD(update); + auto file = std::fstream(fileName, std::ios::out | std::ios::binary); + file.write(buf->getBuffer(), buf->getPos()); + file.close(); + } + + DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { + auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); + auto size = file.tellg(); + ByteBuffer buf(size); + file.seekg(0); + file.read(buf.getBuffer(), size); + file.close(); + + nbostream inStream(buf.getBufferAtPos(), buf.getRemaining()); + return DocumentUpdate::createHEAD(*repo, inStream); + } + +}; + +TEST(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized) +{ + TensorUpdateSerializeFixture f; + auto update = f.deserializeUpdateFromFile("data/serialize-tensor-update-java.dat"); + EXPECT_EQ(*f.makeUpdate(), *update); +} + +TEST(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp) +{ + TensorUpdateSerializeFixture f; + auto update = f.makeUpdate(); + f.serializeUpdateToFile(*update, "data/serialize-tensor-update-cpp.dat"); +} + void assertDocumentUpdateFlag(bool createIfNonExistent, int value) diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index d23165cc815..48f32cb7d65 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -410,4 +410,11 @@ DocumentUpdate::reserialize() _needHardReserialize = false; } +std::ostream & +operator<<(std::ostream &out, const DocumentUpdate &update) +{ + update.print(out, false, ""); + return out; +} + } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index a7ff138af87..902bfe63a0a 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -141,5 +141,8 @@ private: friend VespaDocumentSerializer; }; +std::ostream &operator<<(std::ostream &out, const DocumentUpdate &update); + + } |