First take at making a gid based remove.

10 files changed, 137 insertions, 25 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
index 37b23449315..f1161c8ebdd 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
@@ -41,9 +41,14 @@ DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const
 void
 DocumentOperation::assertValidBucketId(const document::DocumentId &docId) const
 {
+   assertValidBucketId(docId.getGlobalId());
+}
+
+void
+DocumentOperation::assertValidBucketId(const document::GlobalId &gid) const
+{
     assert(_bucketId.valid());
     uint8_t bucketUsedBits = _bucketId.getUsedBits();
-    const GlobalId &gid = docId.getGlobalId();
     BucketId verId(gid.convertToBucketId());
     verId.setUsedBits(bucketUsedBits);
     assert(_bucketId.getRawId() == verId.getRawId() ||
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
index 9a823c553bd..6847dbfd943 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
@@ -26,6 +26,7 @@ protected:
                       const storage::spi::Timestamp &timestamp);
 
     void assertValidBucketId(const document::DocumentId &docId) const;
+    void assertValidBucketId(const document::GlobalId &docId) const;
     vespalib::string docArgsToString() const;
 
 public:
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
index 3509af0de5c..10518c74340 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
@@ -36,7 +36,8 @@ public:
         MOVE                    = 15,
         CREATE_BUCKET           = 16,
         COMPACT_LID_SPACE       = 17,
-        UPDATE                  = 18
+        UPDATE                  = 18,
+        REMOVE_GID              = 19
     };
 
 private:
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
index 1f703be47ab..d388b02852c 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
@@ -4,6 +4,7 @@
 
 using document::BucketId;
 using document::DocumentId;
+using document::GlobalId;
 using document::DocumentTypeRepo;
 using storage::spi::Timestamp;
 using vespalib::make_string;
@@ -50,8 +51,57 @@ RemoveOperationWithDocId::deserialize(vespalib::nbostream &is,
 vespalib::string
 RemoveOperationWithDocId::toString() const {
     return make_string("Remove(%s, %s)",
-                       _docId.getScheme().toString().c_str(),
-                       docArgsToString().c_str());
+                       _docId.getScheme().toString().c_str(), docArgsToString().c_str());
+}
+
+RemoveOperationWithGid::RemoveOperationWithGid()
+        : RemoveOperation(FeedOperation::REMOVE_GID),
+          _gid(),
+          _docType(),
+          _lid(0)
+{}
+
+
+RemoveOperationWithGid::RemoveOperationWithGid(BucketId bucketId, Timestamp timestamp, const GlobalId &gid, vespalib::stringref docType, uint32_t lid)
+        : RemoveOperation(FeedOperation::REMOVE_GID, bucketId, timestamp),
+          _gid(gid),
+          _docType(docType),
+          _lid(lid)
+{}
+
+RemoveOperationWithGid::~RemoveOperationWithGid() = default;
+
+void
+RemoveOperationWithGid::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_gid);
+    RemoveOperation::serialize(os);
+    size_t oldSize = os.size();
+    os.write(_gid.get(), GlobalId::LENGTH);
+    os << _lid;
+    os.writeSmallString(_docType);
+    _serializedDocSize = os.size() - oldSize;
+}
+
+
+void
+RemoveOperationWithGid::deserialize(vespalib::nbostream &is,
+                                      const DocumentTypeRepo &repo)
+{
+    RemoveOperation::deserialize(is, repo);
+    size_t oldSize = is.size();
+    char buf[GlobalId::LENGTH];
+    is.read(buf, sizeof(buf));
+    _gid.set(buf);
+    is >> _lid;
+    is.readSmallString(_docType);
+    _serializedDocSize = oldSize - is.size();
+}
+
+vespalib::string
+RemoveOperationWithGid::toString() const {
+    return make_string("RemoveGid(%s, %u, %s, %s)",
+                       _gid.toString().c_str(), _lid, _docType.c_str(), docArgsToString().c_str());
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
index a22a6cee184..8bd19d55ac9 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
@@ -37,5 +37,28 @@ public:
     vespalib::stringref getDocType() const override { return _docId.getDocType(); }
 };
 
+class RemoveOperationWithGid : public RemoveOperation {
+    document::GlobalId _gid;
+    vespalib::string   _docType;
+    uint32_t           _lid;
+
+public:
+    RemoveOperationWithGid();
+    RemoveOperationWithGid(document::BucketId bucketId,
+                           storage::spi::Timestamp timestamp,
+                           const document::GlobalId & gid,
+                           vespalib::stringref docType,
+                           uint32_t lid);
+    ~RemoveOperationWithGid() override;
+    uint32_t  getLid() const { return _lid; }
+    const document::GlobalId & getGlobalId() const override { return _gid; }
+    void serialize(vespalib::nbostream &os) const override;
+    void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override;
+    vespalib::string toString() const override;
+
+    bool hasDocType() const override { return true; }
+    vespalib::stringref getDocType() const override { return _docType; }
+};
+
 } // namespace proton
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
index a98504a210e..0eb11eddaf3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/feedoperation/removeoperation.h>
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
 #include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
 
@@ -127,7 +128,7 @@ void DocStoreValidator::performRemoves(FeedHandler & feedHandler, const search::
             document::Document::UP document = store.read(lid, repo);
             assert(document);
             LOG(info, "Removing document with id %s and lid %u with gid %s in bucket %s", document->getId().toString().c_str(), lid, metaData.gid.toString().c_str(), metaData.bucketId.toString().c_str());
-            auto remove = std::make_unique<RemoveOperationWithDocId>(metaData.bucketId, metaData.timestamp, document->getId());
+            auto remove = std::make_unique<RemoveOperationWithGid>(metaData.bucketId, metaData.timestamp, gid, document->getType().getName(), lid);
             feedHandler.performOperation(FeedToken(), std::move(remove));
         }
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index e02347b20fe..362de7ee780 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -564,6 +564,7 @@ FeedHandler::performOperation(FeedToken token, FeedOperation::UP op)
         performPut(std::move(token), static_cast<PutOperation &>(*op));
         return;
     case FeedOperation::REMOVE:
+    case FeedOperation::REMOVE_GID:
         performRemove(std::move(token), static_cast<RemoveOperation &>(*op));
         return;
     case FeedOperation::UPDATE_42:
diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
index 5cc84c5937b..a1138503085 100644
--- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
@@ -39,6 +39,10 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry)
         RemoveOperationWithDocId op;
         replay(op, is, entry);
         break;
+    } case FeedOperation::REMOVE_GID: {
+        RemoveOperationWithGid op;
+        replay(op, is, entry);
+        break;
     } case FeedOperation::UPDATE: {
         UpdateOperation op(static_cast<FeedOperation::Type>(entry.type()));
         replay(op, is, entry);
@@ -84,7 +88,7 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry)
         throw IllegalStateException
             (make_string("Got packet entry with unknown type id '%u' from TLS", entry.type()));
     }
-    if (is.size() > 0) {
+    if ( ! is.empty()) {
         throw document::DeserializeException
             (make_string("Too much data in packet entry (type id '%u', %ld bytes)",
                          entry.type(), is.size()));
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 4dc1a9fce91..24e0f2f43b3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -27,6 +27,7 @@ LOG_SETUP(".proton.server.storeonlyfeedview");
 using document::BucketId;
 using document::Document;
 using document::DocumentId;
+using document::GlobalId;
 using document::DocumentTypeRepo;
 using document::DocumentUpdate;
 using proton::attribute::isUpdateableInMemoryOnly;
@@ -142,46 +143,45 @@ std::vector<document::GlobalId> getGidsToRemove(const IDocumentMetaStore &metaSt
     return gids;
 }
 
-void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+void putMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentId & doc_id,
                  const DocumentOperation &op, bool is_removed_doc)
 {
     documentmetastore::IStore::Result putRes(
-            meta_store.put(doc_id.getGlobalId(),
+            meta_store.put(gid,
                            op.getBucketId(), op.getTimestamp(), op.getSerializedDocSize(), op.getLid()));
     if (!putRes.ok()) {
         throw IllegalStateException(
                 make_string("Could not put <lid, gid> pair for %sdocument with id '%s' and gid '%s'",
                             is_removed_doc ? "removed " : "", doc_id.toString().c_str(),
-                            doc_id.getGlobalId().toString().c_str()));
+                            gid.toString().c_str()));
     }
     assert(op.getLid() == putRes._lid);
 }
 
-void removeMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+void removeMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentId &doc_id,
                     const DocumentOperation &op, bool is_removed_doc) {
     assert(meta_store.validLid(op.getPrevLid()));
     assert(is_removed_doc == op.getPrevMarkedAsRemoved());
     const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid()));
-    assert(meta.getGid() == doc_id.getGlobalId());
+    assert(meta.getGid() == gid);
     (void) meta;
     if (!meta_store.remove(op.getPrevLid())) {
         throw IllegalStateException(
                 make_string("Could not remove <lid, gid> pair for %sdocument with id '%s' and gid '%s'",
                             is_removed_doc ? "removed " : "", doc_id.toString().c_str(),
-                            doc_id.getGlobalId().toString().c_str()));
+                            gid.toString().c_str()));
     }
 }
 
 void
-moveMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id, const DocumentOperation &op)
+moveMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentOperation &op)
 {
-    (void) doc_id;
     assert(op.getLid() != op.getPrevLid());
     assert(meta_store.validLid(op.getPrevLid()));
     assert(!meta_store.validLid(op.getLid()));
     const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid()));
     (void) meta;
-    assert(meta.getGid() == doc_id.getGlobalId());
+    assert(meta.getGid() == gid);
     assert(meta.getTimestamp() == op.getTimestamp());
     meta_store.move(op.getPrevLid(), op.getLid());
 }
@@ -280,7 +280,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp)
          putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(),
          _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str());
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId.getGlobalId(), docId);
     considerEarlyAck(token);
 
     bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId);
@@ -541,6 +541,8 @@ void
 StoreOnlyFeedView::handleRemove(FeedToken token, const RemoveOperation &rmOp) {
     if (rmOp.getType() == FeedOperation::REMOVE) {
         internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithDocId &>(rmOp));
+    } else if (rmOp.getType() == FeedOperation::REMOVE_GID) {
+        internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithGid &>(rmOp));
     } else {
         assert(rmOp.getType() == FeedOperation::REMOVE);
     }
@@ -560,7 +562,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI
          _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(),
          rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId);
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId.getGlobalId(), docId);
     considerEarlyAck(token);
 
     if (rmOp.getValidDbdId(_params._subDbId)) {
@@ -579,6 +581,29 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI
 }
 
 void
+StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp)
+{
+    assert(rmOp.getValidNewOrPrevDbdId());
+    assert(rmOp.notMovingLidInSameSubDb());
+    const SerialNum serialNum = rmOp.getSerialNum();
+    DocumentId dummy;
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy);
+    considerEarlyAck(token);
+
+    if (rmOp.getValidDbdId(_params._subDbId)) {
+        internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone),
+                       rmOp.getPrevLid(), IDestructorCallback::SP());
+    }
+    if (rmOp.getValidPrevDbdId(_params._subDbId)) {
+        if (rmOp.changedDbdId()) {
+            assert(!rmOp.getValidDbdId(_params._subDbId));
+            internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone),
+                           rmOp.getPrevLid(), IDestructorCallback::SP());
+        }
+    }
+}
+
+void
 StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum,
                                   PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid,
                                   IDestructorCallback::SP moveDoneCtx)
@@ -595,7 +620,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum,
 }
 
 PendingNotifyRemoveDone
-StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId &docId)
+StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gid, const DocumentId &docId)
 {
     PendingNotifyRemoveDone pendingNotifyRemoveDone;
     const SerialNum serialNum = op.getSerialNum();
@@ -605,14 +630,14 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId
                 op.getValidPrevDbdId(_params._subDbId) &&
                 op.getLid() != op.getPrevLid())
             {
-                moveMetaData(_metaStore, docId, op);
+                moveMetaData(_metaStore, docId.getGlobalId(), op);
             } else {
-                putMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED);
+                putMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED);
             }
         } else if (op.getValidPrevDbdId(_params._subDbId)) {
-            _gidToLidChangeHandler.notifyRemove(docId.getGlobalId(), serialNum);
-            pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, docId.getGlobalId(), serialNum);
-            removeMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED);
+            _gidToLidChangeHandler.notifyRemove(gid, serialNum);
+            pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, gid, serialNum);
+            removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED);
         }
         _metaStore.commit(serialNum, serialNum);
     }
@@ -731,7 +756,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback::
          moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(),
          _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str());
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId);
     bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId);
     if (moveOp.getValidDbdId(_params._subDbId)) {
         bool immediateCommit = _commitTimeTracker.needCommit();
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
index 6451d6214ab..be2ed9af126 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -172,12 +172,13 @@ private:
         return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum;
     }
 
-    PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId);
+    PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId);
     void internalPut(FeedToken token, const PutOperation &putOp);
     void internalUpdate(FeedToken token, const UpdateOperation &updOp);
 
     bool lookupDocId(const document::DocumentId &docId, Lid & lid) const;
     void internalRemove(FeedToken token, const RemoveOperationWithDocId &rmOp);
+    void internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp);
 
     // Removes documents from meta store and document store.
     // returns the number of documents removed.