add create-if-non-existent flag for document put

19 files changed, 151 insertions, 14 deletions

diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json
index 37b8eb6a75d..dbea284e39e 100644
--- a/documentapi/abi-spec.json
+++ b/documentapi/abi-spec.json
@@ -2458,7 +2458,9 @@
       "public long getSequenceId()",
       "public int getType()",
       "public com.yahoo.document.TestAndSetCondition getCondition()",
-      "public void setCondition(com.yahoo.document.TestAndSetCondition)"
+      "public void setCondition(com.yahoo.document.TestAndSetCondition)",
+      "public boolean getCreateIfNonExistent()",
+      "public void setCreateIfNonExistent(boolean)"
     ],
     "fields" : [ ]
   },
@@ -3291,4 +3293,4 @@
     ],
     "fields" : [ ]
   }
-}
\ No newline at end of file
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java
index 4482e5bfc47..585b24550e3 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java
@@ -129,4 +129,11 @@ public class PutDocumentMessage extends TestAndSetMessage {
         put.setCondition(condition);
     }
 
+    public void setCreateIfNonExistent(boolean value) {
+        put.setCreateIfNonExistent(value);
+    }
+    public boolean getCreateIfNonExistent() {
+        deserialize();
+        return put.getCreateIfNonExistent();
+    }
 }
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java
index 099839672a2..9812f214066 100644
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java
@@ -607,6 +607,10 @@ public abstract class RoutableFactories60 {
             msg.setDocumentPut(new DocumentPut(Document.createDocument(buf)));
             msg.setTimestamp(buf.getLong(null));
             decodeTasCondition(msg, buf);
+            if (buf.getBuf().hasRemaining()) {
+                byte value = buf.getBuf().get();
+                msg.setCreateIfNonExistent(value != 0);
+            }
         }
 
         @Override
@@ -627,6 +631,11 @@ public abstract class RoutableFactories60 {
                 msg.getDocumentPut().getDocument().serialize(buf);
                 buf.putLong(null, msg.getTimestamp());
                 encodeTasCondition(buf, (TestAndSetMessage) obj);
+                if (msg.getCreateIfNonExistent()) {
+                    buf.getBuf().put((byte)1);
+                } else {
+                    buf.getBuf().put((byte)0);
+                }
             }
             return true;
         }
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java
index 35f8e7cf0c8..97cef53695a 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java
@@ -597,6 +597,26 @@ public class Messages60TestCase extends MessagesTestBase {
 
     public class testPutDocumentMessage implements RunnableTest {
 
+        void verifyCreateIfNonExistentFlag() {
+            var msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "id:ns:testdoc::")));
+            msg.setCreateIfNonExistent(true);
+            int size_of_create_if_non_existent_flag = 1;
+            int expected_serialized_size = BASE_MESSAGE_LENGTH + 45 + serializedLength(msg.getCondition().getSelection()) + size_of_create_if_non_existent_flag;
+            assertEquals(expected_serialized_size, serialize("PutDocumentMessage-create", msg));
+            assertEquals(expected_serialized_size - 1, serialize("PutDocumentMessage-create-truncate", msg, data -> DataTamper.truncate(data, 1)));
+            assertEquals(expected_serialized_size + 1, serialize("PutDocumentMessage-create-pad", msg, data -> DataTamper.pad(data, 1)));
+            for (Language lang: LANGUAGES) {
+                var decoded = (PutDocumentMessage)deserialize("PutDocumentMessage-create", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+                var decoded_trunc = (PutDocumentMessage)deserialize("PutDocumentMessage-create-truncate", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+                var decoded_pad = (PutDocumentMessage)deserialize("PutDocumentMessage-create-pad", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+                assertEquals(true, decoded.getCreateIfNonExistent());
+                assertEquals(false, decoded_trunc.getCreateIfNonExistent());
+                assertEquals(true, decoded_pad.getCreateIfNonExistent());
+                assertTrue(decoded.getDocumentPut().equals(decoded_pad.getDocumentPut()));
+                assertFalse(decoded.getDocumentPut().equals(decoded_trunc.getDocumentPut()));
+            }
+        }
+
         @Override
         public void run() {
             PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "id:ns:testdoc::")));
@@ -604,7 +624,9 @@ public class Messages60TestCase extends MessagesTestBase {
             msg.setTimestamp(666);
             msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
 
-            assertEquals(BASE_MESSAGE_LENGTH + 45 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg));
+            int size_of_create_if_non_existent_flag = 1;
+            int expected_serialized_size = BASE_MESSAGE_LENGTH + 45 + serializedLength(msg.getCondition().getSelection()) + size_of_create_if_non_existent_flag;
+            assertEquals(expected_serialized_size, serialize("PutDocumentMessage", msg));
 
             for (Language lang : LANGUAGES) {
                 final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
@@ -612,7 +634,9 @@ public class Messages60TestCase extends MessagesTestBase {
                 assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString());
                 assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp());
                 assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+                assertEquals(false, deserializedMsg.getCreateIfNonExistent());
             }
+            verifyCreateIfNonExistentFlag();
         }
     }
 
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
index f43449646be..f0fded23d82 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
@@ -94,17 +94,36 @@ public abstract class MessagesTestBase {
         return Arrays.equals(existingData, dataToWrite);
     }
 
+    public interface DataTamper {
+        byte[] tamperWith(byte[] data);
+        static byte[] truncate(byte[] data, int bytes) {
+            int newLength = data.length - bytes;
+            assertTrue(newLength > 0);
+            byte[] res = new byte[newLength];
+            System.arraycopy(data, 0, res, 0, newLength);
+            return res;
+        }
+        static byte[] pad(byte[] data, int bytes) {
+            int newLength = data.length + bytes;
+            byte[] res = new byte[newLength];
+            System.arraycopy(data, 0, res, 0, data.length);
+            return res;
+        }
+    }
+
     /**
      * Writes the content of the given routable to the given file.
      *
      * @param filename The name of the file to write to.
      * @param routable The routable to serialize.
+     * @param tamper allows tampering with serialized data
      * @return The size of the written file.
      */
-    public int serialize(String filename, Routable routable) {
+    public int serialize(String filename, Routable routable, DataTamper tamper) {
         Version version = version();
         String path = getPath(version + "-java-" + filename + ".dat");
         byte[] data = protocol.encode(version, routable);
+        data = tamper.tamperWith(data);
         assertNotNull(data);
         assertTrue(data.length > 0);
         try {
@@ -122,6 +141,9 @@ public abstract class MessagesTestBase {
         assertEquals(routable.getType(), protocol.decode(version, data).getType());
         return data.length;
     }
+    public int serialize(String filename, Routable routable) {
+        return serialize(filename, routable, data -> data);
+    }
 
     /**
      * Reads the content of the given file and creates a corresponding routable.
diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp
index 258e8821e0c..93c7b5ef7cb 100644
--- a/documentapi/src/tests/messages/messages60test.cpp
+++ b/documentapi/src/tests/messages/messages60test.cpp
@@ -15,6 +15,17 @@
 using document::DataType;
 using document::DocumentTypeRepo;
 
+template <typename T>
+struct Unwrap {
+    mbus::Routable::UP value;
+    const T *ptr = nullptr;
+    explicit Unwrap(mbus::Routable::UP value_in) : value(std::move(value_in)) {
+        ptr = dynamic_cast<T*>(value.get());
+        ASSERT_TRUE(ptr != nullptr);
+    }
+    const T *operator->() const noexcept { return ptr; }
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 // Setup
@@ -399,10 +410,12 @@ Messages60Test::testPutDocumentMessage()
     EXPECT_EQUAL(sizeof(vespalib::string), sizeof(TestAndSetCondition));
     EXPECT_EQUAL(112u, sizeof(DocumentMessage));
     EXPECT_EQUAL(sizeof(TestAndSetCondition) + sizeof(DocumentMessage), sizeof(TestAndSetMessage));
-    EXPECT_EQUAL(sizeof(TestAndSetMessage) + 24, sizeof(PutDocumentMessage));
+    EXPECT_EQUAL(sizeof(TestAndSetMessage) + 32, sizeof(PutDocumentMessage));
+    int size_of_create_if_non_existent_flag = 1;
     EXPECT_EQUAL(MESSAGE_BASE_LENGTH +
                  45u +
-                 serializedLength(msg.getCondition().getSelection()),
+                 serializedLength(msg.getCondition().getSelection()) +
+                 size_of_create_if_non_existent_flag,
                  serialize("PutDocumentMessage", msg));
 
     for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) {
@@ -413,11 +426,34 @@ Messages60Test::testPutDocumentMessage()
             EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName());
             EXPECT_EQUAL(msg.getDocument().getId().toString(), deserializedMsg.getDocument().getId().toString());
             EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp());
-            EXPECT_EQUAL(71u, deserializedMsg.getApproxSize());
+            EXPECT_EQUAL(72u, deserializedMsg.getApproxSize());
             EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+            EXPECT_EQUAL(false, deserializedMsg.get_create_if_non_existent());
         }
     }
 
+    //-------------------------------------------------------------------------
+
+    PutDocumentMessage msg2(createDoc(getTypeRepo(), "testdoc", "id:ns:testdoc::"));
+    msg2.set_create_if_non_existent(true);
+    uint32_t expected_message_size = MESSAGE_BASE_LENGTH + 45u +
+        serializedLength(msg2.getCondition().getSelection()) +
+        size_of_create_if_non_existent_flag;
+    auto trunc1 = [](mbus::Blob x) noexcept { return truncate(std::move(x), 1); };
+    auto pad1 = [](mbus::Blob x) noexcept { return pad(std::move(x), 1); };
+    EXPECT_EQUAL(expected_message_size, serialize("PutDocumentMessage-create", msg2));
+    EXPECT_EQUAL(expected_message_size - 1, serialize("PutDocumentMessage-create-truncate", msg2, trunc1));
+    EXPECT_EQUAL(expected_message_size + 1, serialize("PutDocumentMessage-create-pad", msg2, pad1));
+
+    for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) {
+        auto decoded = Unwrap<PutDocumentMessage>(deserialize("PutDocumentMessage-create", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang));
+        auto decoded_trunc = Unwrap<PutDocumentMessage>(deserialize("PutDocumentMessage-create-truncate", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang));
+        auto decoded_pad = Unwrap<PutDocumentMessage>(deserialize("PutDocumentMessage-create-pad", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang));
+        EXPECT_EQUAL(true, decoded->get_create_if_non_existent());
+        EXPECT_EQUAL(false, decoded_trunc->get_create_if_non_existent());
+        EXPECT_EQUAL(true, decoded_pad->get_create_if_non_existent());
+    }
+
     return true;
 }
 
diff --git a/documentapi/src/tests/messages/testbase.cpp b/documentapi/src/tests/messages/testbase.cpp
index db2c08704e1..ddce9f82a5d 100644
--- a/documentapi/src/tests/messages/testbase.cpp
+++ b/documentapi/src/tests/messages/testbase.cpp
@@ -76,6 +76,24 @@ TestBase::putTest(uint32_t type, TEST_METHOD_PT test)
     return *this;
 }
 
+mbus::Blob
+TestBase::truncate(mbus::Blob data, size_t bytes)
+{
+    ASSERT_GREATER(data.size(), bytes);
+    mbus::Blob res(data.size() - bytes);
+    memcpy(res.data(), data.data(), res.size());
+    return res;
+}
+
+mbus::Blob
+TestBase::pad(mbus::Blob data, size_t bytes)
+{
+    mbus::Blob res(data.size() + bytes);
+    memset(res.data(), 0, res.size());
+    memcpy(res.data(), data.data(), data.size());
+    return res;
+}
+
 bool
 TestBase::testCoverage(const std::vector<uint32_t> &expected, const std::vector<uint32_t> &actual, bool report) const
 {
@@ -119,13 +137,13 @@ bool TestBase::file_content_is_unchanged(const string& filename, const mbus::Blo
 }
 
 uint32_t
-TestBase::serialize(const string &filename, const mbus::Routable &routable)
+TestBase::serialize(const string &filename, const mbus::Routable &routable, Tamper tamper)
 {
     const vespalib::Version version = getVersion();
     string path = getPath(version.toString() + "-cpp-" + filename + ".dat");
     LOG(info, "Serializing to '%s'..", path.c_str());
 
-    mbus::Blob blob = _protocol.encode(version, routable);
+    mbus::Blob blob = tamper(_protocol.encode(version, routable));
     if (file_content_is_unchanged(path, blob)) {
         LOG(info, "Serialization for '%s' is unchanged; not overwriting it", path.c_str());
     } else if (!EXPECT_TRUE(writeFile(path, blob))) {
@@ -213,5 +231,3 @@ TestBase::readFile(const string &filename) const
 
     return blob;
 }
-
-
diff --git a/documentapi/src/tests/messages/testbase.h b/documentapi/src/tests/messages/testbase.h
index bf384fc2fb9..a2cd5ee5649 100644
--- a/documentapi/src/tests/messages/testbase.h
+++ b/documentapi/src/tests/messages/testbase.h
@@ -43,13 +43,20 @@ protected:
     int Main() override;
 
 public:
+    using Tamper = std::function<mbus::Blob(mbus::Blob)>;
+    static mbus::Blob truncate(mbus::Blob data, size_t bytes);
+    static mbus::Blob pad(mbus::Blob data, size_t bytes);
+
     const document::DocumentTypeRepo &getTypeRepo() { return *_repo; }
     std::shared_ptr<const document::DocumentTypeRepo> &getTypeRepoSp() { return _repo; }
 
     bool testCoverage(const std::vector<uint32_t> &expected, const std::vector<uint32_t> &actual, bool report = false) const;
     bool writeFile(const string &filename, const mbus::Blob& blob) const;
     mbus::Blob readFile(const string &filename) const;
-    uint32_t serialize(const string &filename, const mbus::Routable &routable);
+    uint32_t serialize(const string &filename, const mbus::Routable &routable, Tamper tamper);
+    uint32_t serialize(const string &filename, const mbus::Routable &routable) {
+        return serialize(filename, routable, [](auto x)noexcept{ return x; });
+    }
     mbus::Routable::UP deserialize(const string &filename, uint32_t classId, uint32_t lang);
     void dump(const mbus::Blob &blob) const;
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
index 5a7f8acb60a..018aea69a71 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
@@ -11,6 +11,7 @@ private:
     using DocumentSP = std::shared_ptr<document::Document>;
     DocumentSP _document;
     uint64_t   _time;
+    bool _create_if_non_existent = false;
 
 protected:
     DocumentReply::UP doCreateReply() const override;
@@ -65,7 +66,9 @@ public:
     uint64_t getSequenceId() const override;
     uint32_t getType() const override;
     string toString() const override { return "putdocumentmessage"; }
+
+    void set_create_if_non_existent(bool value) { _create_if_non_existent = value; }
+    bool get_create_if_non_existent() const { return _create_if_non_existent; }
 };
 
 }
-
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp
index 0757db9f5fc..1dce7ff281f 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp
@@ -532,6 +532,11 @@ RoutableFactories60::PutDocumentMessageFactory::decodeInto(PutDocumentMessage &
     msg.setDocument(decodeDocument(_repo, buf));
     msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf)));
     decodeTasCondition(msg, buf);
+    if (buf.getRemaining() > 0) {
+        uint8_t value = 0;
+        buf.getByte(value);
+        msg.set_create_if_non_existent(value != 0);
+    }
 }
 
 bool
@@ -544,7 +549,12 @@ RoutableFactories60::PutDocumentMessageFactory::doEncode(const DocumentMessage &
     buf.putBytes(stream.peek(), stream.size());
     buf.putLong(static_cast<int64_t>(msg.getTimestamp()));
     encodeTasCondition(buf, msg);
-
+    if (msg.get_create_if_non_existent()) {
+        buf.putByte(1);
+    } else {
+        buf.putByte(0);
+    }
+    
     return true;
 }
 
diff --git a/documentapi/test/crosslanguagefiles/.gitattributes b/documentapi/test/crosslanguagefiles/.gitattributes
new file mode 100644
index 00000000000..8697fef7d46
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/.gitattributes
@@ -0,0 +1 @@
+*.dat binary
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-pad.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-pad.dat
new file mode 100644
index 00000000000..bbe9bede8d6
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-pad.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-truncate.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-truncate.dat
new file mode 100644
index 00000000000..344d0a4b58c
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create-truncate.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create.dat
new file mode 100644
index 00000000000..7f7c55b2b88
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage-create.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage.dat
index 706a002c8c9..93f85f7e419 100644
--- a/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-PutDocumentMessage.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-pad.dat b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-pad.dat
new file mode 100644
index 00000000000..bbe9bede8d6
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-pad.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-truncate.dat b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-truncate.dat
new file mode 100644
index 00000000000..344d0a4b58c
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create-truncate.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create.dat b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create.dat
new file mode 100644
index 00000000000..7f7c55b2b88
--- /dev/null
+++ b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage-create.dat
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage.dat
index 706a002c8c9..93f85f7e419 100644
--- a/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-java-PutDocumentMessage.dat