diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-04-08 23:41:05 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-04-08 23:41:05 +0000 |
commit | e862fb33abde3bb6d6a953a2e74af2cc15506508 (patch) | |
tree | fc291198058b60b65874b7186202535aa3535fe1 /searchcore | |
parent | b971dbe48ad2111f24e7b953eaf351e9a0bb4ea1 (diff) |
First take at making a gid based remove.
Diffstat (limited to 'searchcore')
10 files changed, 137 insertions, 25 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index 37b23449315..f1161c8ebdd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -41,9 +41,14 @@ DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const void DocumentOperation::assertValidBucketId(const document::DocumentId &docId) const { + assertValidBucketId(docId.getGlobalId()); +} + +void +DocumentOperation::assertValidBucketId(const document::GlobalId &gid) const +{ assert(_bucketId.valid()); uint8_t bucketUsedBits = _bucketId.getUsedBits(); - const GlobalId &gid = docId.getGlobalId(); BucketId verId(gid.convertToBucketId()); verId.setUsedBits(bucketUsedBits); assert(_bucketId.getRawId() == verId.getRawId() || diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index 9a823c553bd..6847dbfd943 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -26,6 +26,7 @@ protected: const storage::spi::Timestamp ×tamp); void assertValidBucketId(const document::DocumentId &docId) const; + void assertValidBucketId(const document::GlobalId &docId) const; vespalib::string docArgsToString() const; public: diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index 3509af0de5c..10518c74340 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -36,7 +36,8 @@ public: MOVE = 15, CREATE_BUCKET = 16, COMPACT_LID_SPACE = 17, - UPDATE = 18 + UPDATE = 18, + REMOVE_GID = 19 }; private: diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp index 1f703be47ab..d388b02852c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp @@ -4,6 +4,7 @@ using document::BucketId; using document::DocumentId; +using document::GlobalId; using document::DocumentTypeRepo; using storage::spi::Timestamp; using vespalib::make_string; @@ -50,8 +51,57 @@ RemoveOperationWithDocId::deserialize(vespalib::nbostream &is, vespalib::string RemoveOperationWithDocId::toString() const { return make_string("Remove(%s, %s)", - _docId.getScheme().toString().c_str(), - docArgsToString().c_str()); + _docId.getScheme().toString().c_str(), docArgsToString().c_str()); +} + +RemoveOperationWithGid::RemoveOperationWithGid() + : RemoveOperation(FeedOperation::REMOVE_GID), + _gid(), + _docType(), + _lid(0) +{} + + +RemoveOperationWithGid::RemoveOperationWithGid(BucketId bucketId, Timestamp timestamp, const GlobalId &gid, vespalib::stringref docType, uint32_t lid) + : RemoveOperation(FeedOperation::REMOVE_GID, bucketId, timestamp), + _gid(gid), + _docType(docType), + _lid(lid) +{} + +RemoveOperationWithGid::~RemoveOperationWithGid() = default; + +void +RemoveOperationWithGid::serialize(vespalib::nbostream &os) const +{ + assertValidBucketId(_gid); + RemoveOperation::serialize(os); + size_t oldSize = os.size(); + os.write(_gid.get(), GlobalId::LENGTH); + os << _lid; + os.writeSmallString(_docType); + _serializedDocSize = os.size() - oldSize; +} + + +void +RemoveOperationWithGid::deserialize(vespalib::nbostream &is, + const DocumentTypeRepo &repo) +{ + RemoveOperation::deserialize(is, repo); + size_t oldSize = is.size(); + char buf[GlobalId::LENGTH]; + is.read(buf, sizeof(buf)); + _gid.set(buf); + is >> _lid; + is.readSmallString(_docType); + _serializedDocSize = oldSize - is.size(); +} + +vespalib::string +RemoveOperationWithGid::toString() const { + return make_string("RemoveGid(%s, %u, %s, %s)", + _gid.toString().c_str(), _lid, _docType.c_str(), docArgsToString().c_str()); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h index a22a6cee184..8bd19d55ac9 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h @@ -37,5 +37,28 @@ public: vespalib::stringref getDocType() const override { return _docId.getDocType(); } }; +class RemoveOperationWithGid : public RemoveOperation { + document::GlobalId _gid; + vespalib::string _docType; + uint32_t _lid; + +public: + RemoveOperationWithGid(); + RemoveOperationWithGid(document::BucketId bucketId, + storage::spi::Timestamp timestamp, + const document::GlobalId & gid, + vespalib::stringref docType, + uint32_t lid); + ~RemoveOperationWithGid() override; + uint32_t getLid() const { return _lid; } + const document::GlobalId & getGlobalId() const override { return _gid; } + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + vespalib::string toString() const override; + + bool hasDocType() const override { return true; } + vespalib::stringref getDocType() const override { return _docType; } +}; + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp index a98504a210e..0eb11eddaf3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp @@ -5,6 +5,7 @@ #include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> @@ -127,7 +128,7 @@ void DocStoreValidator::performRemoves(FeedHandler & feedHandler, const search:: document::Document::UP document = store.read(lid, repo); assert(document); LOG(info, "Removing document with id %s and lid %u with gid %s in bucket %s", document->getId().toString().c_str(), lid, metaData.gid.toString().c_str(), metaData.bucketId.toString().c_str()); - auto remove = std::make_unique<RemoveOperationWithDocId>(metaData.bucketId, metaData.timestamp, document->getId()); + auto remove = std::make_unique<RemoveOperationWithGid>(metaData.bucketId, metaData.timestamp, gid, document->getType().getName(), lid); feedHandler.performOperation(FeedToken(), std::move(remove)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index e02347b20fe..362de7ee780 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -564,6 +564,7 @@ FeedHandler::performOperation(FeedToken token, FeedOperation::UP op) performPut(std::move(token), static_cast<PutOperation &>(*op)); return; case FeedOperation::REMOVE: + case FeedOperation::REMOVE_GID: performRemove(std::move(token), static_cast<RemoveOperation &>(*op)); return; case FeedOperation::UPDATE_42: diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp index 5cc84c5937b..a1138503085 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp @@ -39,6 +39,10 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry) RemoveOperationWithDocId op; replay(op, is, entry); break; + } case FeedOperation::REMOVE_GID: { + RemoveOperationWithGid op; + replay(op, is, entry); + break; } case FeedOperation::UPDATE: { UpdateOperation op(static_cast<FeedOperation::Type>(entry.type())); replay(op, is, entry); @@ -84,7 +88,7 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry) throw IllegalStateException (make_string("Got packet entry with unknown type id '%u' from TLS", entry.type())); } - if (is.size() > 0) { + if ( ! is.empty()) { throw document::DeserializeException (make_string("Too much data in packet entry (type id '%u', %ld bytes)", entry.type(), is.size())); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 4dc1a9fce91..24e0f2f43b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -27,6 +27,7 @@ LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; using document::Document; using document::DocumentId; +using document::GlobalId; using document::DocumentTypeRepo; using document::DocumentUpdate; using proton::attribute::isUpdateableInMemoryOnly; @@ -142,46 +143,45 @@ std::vector<document::GlobalId> getGidsToRemove(const IDocumentMetaStore &metaSt return gids; } -void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id, +void putMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentId & doc_id, const DocumentOperation &op, bool is_removed_doc) { documentmetastore::IStore::Result putRes( - meta_store.put(doc_id.getGlobalId(), + meta_store.put(gid, op.getBucketId(), op.getTimestamp(), op.getSerializedDocSize(), op.getLid())); if (!putRes.ok()) { throw IllegalStateException( make_string("Could not put <lid, gid> pair for %sdocument with id '%s' and gid '%s'", is_removed_doc ? "removed " : "", doc_id.toString().c_str(), - doc_id.getGlobalId().toString().c_str())); + gid.toString().c_str())); } assert(op.getLid() == putRes._lid); } -void removeMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id, +void removeMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentId &doc_id, const DocumentOperation &op, bool is_removed_doc) { assert(meta_store.validLid(op.getPrevLid())); assert(is_removed_doc == op.getPrevMarkedAsRemoved()); const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid())); - assert(meta.getGid() == doc_id.getGlobalId()); + assert(meta.getGid() == gid); (void) meta; if (!meta_store.remove(op.getPrevLid())) { throw IllegalStateException( make_string("Could not remove <lid, gid> pair for %sdocument with id '%s' and gid '%s'", is_removed_doc ? "removed " : "", doc_id.toString().c_str(), - doc_id.getGlobalId().toString().c_str())); + gid.toString().c_str())); } } void -moveMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id, const DocumentOperation &op) +moveMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentOperation &op) { - (void) doc_id; assert(op.getLid() != op.getPrevLid()); assert(meta_store.validLid(op.getPrevLid())); assert(!meta_store.validLid(op.getLid())); const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid())); (void) meta; - assert(meta.getGid() == doc_id.getGlobalId()); + assert(meta.getGid() == gid); assert(meta.getTimestamp() == op.getTimestamp()); meta_store.move(op.getPrevLid(), op.getLid()); } @@ -280,7 +280,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId.getGlobalId(), docId); considerEarlyAck(token); bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); @@ -541,6 +541,8 @@ void StoreOnlyFeedView::handleRemove(FeedToken token, const RemoveOperation &rmOp) { if (rmOp.getType() == FeedOperation::REMOVE) { internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithDocId &>(rmOp)); + } else if (rmOp.getType() == FeedOperation::REMOVE_GID) { + internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithGid &>(rmOp)); } else { assert(rmOp.getType() == FeedOperation::REMOVE); } @@ -560,7 +562,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(), rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId.getGlobalId(), docId); considerEarlyAck(token); if (rmOp.getValidDbdId(_params._subDbId)) { @@ -579,6 +581,29 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI } void +StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp) +{ + assert(rmOp.getValidNewOrPrevDbdId()); + assert(rmOp.notMovingLidInSameSubDb()); + const SerialNum serialNum = rmOp.getSerialNum(); + DocumentId dummy; + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy); + considerEarlyAck(token); + + if (rmOp.getValidDbdId(_params._subDbId)) { + internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone), + rmOp.getPrevLid(), IDestructorCallback::SP()); + } + if (rmOp.getValidPrevDbdId(_params._subDbId)) { + if (rmOp.changedDbdId()) { + assert(!rmOp.getValidDbdId(_params._subDbId)); + internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone), + rmOp.getPrevLid(), IDestructorCallback::SP()); + } + } +} + +void StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum, PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, IDestructorCallback::SP moveDoneCtx) @@ -595,7 +620,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum, } PendingNotifyRemoveDone -StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId &docId) +StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gid, const DocumentId &docId) { PendingNotifyRemoveDone pendingNotifyRemoveDone; const SerialNum serialNum = op.getSerialNum(); @@ -605,14 +630,14 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId op.getValidPrevDbdId(_params._subDbId) && op.getLid() != op.getPrevLid()) { - moveMetaData(_metaStore, docId, op); + moveMetaData(_metaStore, docId.getGlobalId(), op); } else { - putMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED); + putMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED); } } else if (op.getValidPrevDbdId(_params._subDbId)) { - _gidToLidChangeHandler.notifyRemove(docId.getGlobalId(), serialNum); - pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, docId.getGlobalId(), serialNum); - removeMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED); + _gidToLidChangeHandler.notifyRemove(gid, serialNum); + pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, gid, serialNum); + removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED); } _metaStore.commit(serialNum, serialNum); } @@ -731,7 +756,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { bool immediateCommit = _commitTimeTracker.needCommit(); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 6451d6214ab..be2ed9af126 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -172,12 +172,13 @@ private: return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum; } - PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId); + PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId); void internalPut(FeedToken token, const PutOperation &putOp); void internalUpdate(FeedToken token, const UpdateOperation &updOp); bool lookupDocId(const document::DocumentId &docId, Lid & lid) const; void internalRemove(FeedToken token, const RemoveOperationWithDocId &rmOp); + void internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp); // Removes documents from meta store and document store. // returns the number of documents removed. |