Add cross-language serialization tests for tensor updates.

6 files changed, 169 insertions, 5 deletions

diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
index b86094e5a4d..0712adca3a5 100644
--- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -13,9 +13,11 @@ import com.yahoo.document.serialization.DocumentDeserializerFactory;
 import com.yahoo.document.serialization.DocumentSerializer;
 import com.yahoo.document.serialization.DocumentSerializerFactory;
 import com.yahoo.document.serialization.DocumentUpdateFlags;
-import com.yahoo.document.serialization.DocumentUpdateWriter;
 import com.yahoo.document.update.AssignValueUpdate;
 import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.TensorAddUpdate;
+import com.yahoo.document.update.TensorModifyUpdate;
+import com.yahoo.document.update.TensorRemoveUpdate;
 import com.yahoo.document.update.ValueUpdate;
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.tensor.Tensor;
@@ -402,12 +404,14 @@ public class DocumentUpdateTestCase {
 
         GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f);
         upd.serialize(DocumentSerializerFactory.create42(buf));
+        buf.flip();
 
-        int size = buf.position();
-        buf.position(0);
+        writeBufferToFile(buf, "src/tests/data/serializeupdatejava.dat");
+    }
 
-        FileOutputStream fos = new FileOutputStream("src/tests/data/serializeupdatejava.dat");
-        fos.write(buf.array(), 0, size);
+    private static void writeBufferToFile(GrowableByteBuffer buf, String fileName) throws IOException {
+        FileOutputStream fos = new FileOutputStream(fileName);
+        fos.write(buf.array(), 0, buf.remaining());
         fos.close();
     }
 
@@ -787,4 +791,68 @@ public class DocumentUpdateTestCase {
         return result;
     }
 
+    private static class TensorUpdateSerializeFixture {
+        private DocumentTypeManager docMan;
+        private DocumentType docType;
+
+        public TensorUpdateSerializeFixture() {
+            docMan = new DocumentTypeManager();
+            docType = new DocumentType("test");
+            docType.addHeaderField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})")));
+            docType.addHeaderField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])")));
+            docMan.registerDocumentType(docType);
+        }
+
+        Field getField(String name) {
+            return docType.getField(name);
+        }
+
+        TensorFieldValue createTensor() {
+            return new TensorFieldValue(Tensor.from("tensor(x{})", "{{x:2}:5, {x:3}:7}"));
+        }
+
+        DocumentUpdate createUpdate() {
+            var result = new DocumentUpdate(docType, "id:test:test::0");
+
+            result.addFieldUpdate(FieldUpdate.create(getField("sparse_tensor"))
+                    .addValueUpdate(new AssignValueUpdate(createTensor()))
+                    .addValueUpdate(new TensorAddUpdate(createTensor()))
+                    .addValueUpdate(new TensorRemoveUpdate(createTensor())));
+
+            result.addFieldUpdate(FieldUpdate.create(getField("dense_tensor"))
+                    .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.REPLACE, createTensor()))
+                    .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.ADD, createTensor()))
+                    .addValueUpdate(new TensorModifyUpdate(TensorModifyUpdate.Operation.MULTIPLY, createTensor())));
+            return result;
+        }
+
+        void serializeUpdateToFile(DocumentUpdate update, String fileName) throws IOException {
+            GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f);
+            update.serialize(DocumentSerializerFactory.createHead(buf));
+            buf.flip();
+
+            writeBufferToFile(buf, fileName);
+        }
+
+        DocumentUpdate deserializeUpdateFromFile(String fileName) throws IOException {
+            byte[] data = DocumentTestCase.readFile(fileName);
+            DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data));
+            return new DocumentUpdate(buf);
+        }
+    }
+
+    @Test
+    public void tensor_update_file_cpp_can_be_deserialized() throws IOException {
+        var f = new TensorUpdateSerializeFixture();
+        var update = f.deserializeUpdateFromFile("src/tests/data/serialize-tensor-update-cpp.dat");
+        assertEquals(f.createUpdate(), update);
+    }
+
+    @Test
+    public void generate_serialized_tensor_update_file_java() throws IOException {
+        var f = new TensorUpdateSerializeFixture();
+        var update = f.createUpdate();
+        f.serializeUpdateToFile(update, "src/tests/data/serialize-tensor-update-java.dat");
+    }
+
 }
diff --git a/document/src/tests/data/serialize-tensor-update-cpp.dat b/document/src/tests/data/serialize-tensor-update-cpp.dat
new file mode 100644
index 00000000000..ad0e9d706b0
--- /dev/null
+++ b/document/src/tests/data/serialize-tensor-update-cpp.dat
diff --git a/document/src/tests/data/serialize-tensor-update-java.dat b/document/src/tests/data/serialize-tensor-update-java.dat
new file mode 100644
index 00000000000..ad0e9d706b0
--- /dev/null
+++ b/document/src/tests/data/serialize-tensor-update-java.dat
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
index b2a9426a502..7164ee106e5 100644
--- a/document/src/tests/documentupdatetestcase.cpp
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -27,6 +27,7 @@
 #include <vespa/vespalib/util/exceptions.h>
 
 #include <fcntl.h>
+#include <fstream>
 #include <gtest/gtest.h>
 #include <unistd.h>
 
@@ -1044,6 +1045,91 @@ TEST(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_spar
             vespalib::IllegalStateException);
 }
 
+struct TensorUpdateSerializeFixture {
+    std::unique_ptr<DocumentTypeRepo> repo;
+    const DocumentType &docType;
+
+    const TensorDataType &extractTensorDataType(const vespalib::string &fieldName) {
+        const auto &dataType = docType.getField(fieldName).getDataType();
+        return dynamic_cast<const TensorDataType &>(dataType);
+    }
+
+    TensorUpdateSerializeFixture()
+        : repo(makeDocumentTypeRepo()),
+          docType(*repo->getDocumentType("test"))
+    {
+    }
+
+    std::unique_ptr<DocumentTypeRepo> makeDocumentTypeRepo() {
+        config_builder::DocumenttypesConfigBuilderHelper builder;
+        builder.document(222, "test",
+                         Struct("test.header")
+                                 .addTensorField("sparse_tensor", "tensor(x{})")
+                                 .addTensorField("dense_tensor", "tensor(x[4])"),
+                         Struct("testdoc.body"));
+        return std::make_unique<DocumentTypeRepo>(builder.config());
+    }
+
+    std::unique_ptr<TensorFieldValue> makeTensor() {
+        return makeTensorFieldValue(TensorSpec("tensor(x{})").add({{"x", "2"}}, 5)
+                                            .add({{"x", "3"}}, 7),
+                                    extractTensorDataType("sparse_tensor"));
+    }
+
+    const Field &getField(const vespalib::string &name) {
+        return docType.getField(name);
+    }
+
+    DocumentUpdate::UP makeUpdate() {
+        auto result = std::make_unique<DocumentUpdate>
+                (*repo, docType, DocumentId("id:test:test::0"));
+
+        result->addUpdate(FieldUpdate(getField("sparse_tensor"))
+                                  .addUpdate(AssignValueUpdate(*makeTensor()))
+                                  .addUpdate(TensorAddUpdate(makeTensor()))
+                                  .addUpdate(TensorRemoveUpdate(makeTensor())));
+        result->addUpdate(FieldUpdate(getField("dense_tensor"))
+                                  .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, makeTensor()))
+                                  .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::ADD, makeTensor()))
+                                  .addUpdate(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, makeTensor())));
+        return result;
+    }
+
+    void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) {
+        ByteBuffer::UP buf = serializeHEAD(update);
+        auto file = std::fstream(fileName, std::ios::out | std::ios::binary);
+        file.write(buf->getBuffer(), buf->getPos());
+        file.close();
+    }
+
+    DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) {
+        auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate);
+        auto size = file.tellg();
+        ByteBuffer buf(size);
+        file.seekg(0);
+        file.read(buf.getBuffer(), size);
+        file.close();
+
+        nbostream inStream(buf.getBufferAtPos(), buf.getRemaining());
+        return DocumentUpdate::createHEAD(*repo, inStream);
+    }
+
+};
+
+TEST(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized)
+{
+    TensorUpdateSerializeFixture f;
+    auto update = f.deserializeUpdateFromFile("data/serialize-tensor-update-java.dat");
+    EXPECT_EQ(*f.makeUpdate(), *update);
+}
+
+TEST(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp)
+{
+    TensorUpdateSerializeFixture f;
+    auto update = f.makeUpdate();
+    f.serializeUpdateToFile(*update, "data/serialize-tensor-update-cpp.dat");
+}
+
 
 void
 assertDocumentUpdateFlag(bool createIfNonExistent, int value)
diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp
index d23165cc815..48f32cb7d65 100644
--- a/document/src/vespa/document/update/documentupdate.cpp
+++ b/document/src/vespa/document/update/documentupdate.cpp
@@ -410,4 +410,11 @@ DocumentUpdate::reserialize()
     _needHardReserialize = false;
 }
 
+std::ostream &
+operator<<(std::ostream &out, const DocumentUpdate &update)
+{
+    update.print(out, false, "");
+    return out;
+}
+
 }
diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h
index a7ff138af87..902bfe63a0a 100644
--- a/document/src/vespa/document/update/documentupdate.h
+++ b/document/src/vespa/document/update/documentupdate.h
@@ -141,5 +141,8 @@ private:
     friend VespaDocumentSerializer;
 };
 
+std::ostream &operator<<(std::ostream &out, const DocumentUpdate &update);
+
+
 }