Keep commit result live until force commit is done.

10 files changed, 41 insertions, 20 deletions

diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
index 8890a6cfdda..f98fa5e58d0 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
@@ -136,7 +136,7 @@ struct MyHandler : public ILidSpaceCompactionHandler {
             _moveDoneContexts.push_back(std::move(moveDoneCtx));
         }
     }
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override {
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback>) override {
         _wantedSubDbId = op.getSubDbId();
         _wantedLidLimit = op.getLidLimit();
     }
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 63c7a873eaf..f39b4291182 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -350,7 +350,7 @@ struct MockLidSpaceCompactionHandler : public ILidSpaceCompactionHandler
     IDocumentScanIterator::UP getIterator() const override { return IDocumentScanIterator::UP(); }
     MoveOperation::UP createMoveOperation(const search::DocumentMetaData &, uint32_t) const override { return MoveOperation::UP(); }
     void handleMove(const MoveOperation &, IDestructorCallback::SP) override {}
-    void handleCompactLidSpace(const CompactLidSpaceOperation &) override {}
+    void handleCompactLidSpace(const CompactLidSpaceOperation &, std::shared_ptr<IDestructorCallback>) override {}
 };
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 10af2d16764..35966f2fcaf 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -101,6 +101,13 @@ findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespali
     return &_G_defaultProtonDocumentDBConfig;
 }
 
+template <typename T>
+struct KeepAlive : public search::IDestructorCallback {
+    explicit KeepAlive(T toKeep) noexcept : _toKeep(std::move(toKeep)) { }
+    ~KeepAlive() override = default;
+    T _toKeep;
+};
+
 }
 
 template <typename FunctionType>
@@ -346,7 +353,7 @@ DocumentDB::enterReprocessState()
     if (!runner.empty()) {
         runner.run();
         NoopOperation op;
-        _feedHandler->storeOperationSync(op);
+        (void) _feedHandler->storeOperationSync(op);
         sync(op.getSerialNum());
         _subDBs.pruneRemovedFields(op.getSerialNum());
     }
@@ -433,20 +440,21 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum
         *DocumentDBConfig::makeReplayConfig(configSnapshot) ==
         *DocumentDBConfig::makeReplayConfig(_activeConfigSnapshot);
     bool tlsReplayDone = _feedHandler->getTransactionLogReplayDone();
+    FeedHandler::CommitResult commit_result;
     if (!equalReplayConfig && tlsReplayDone) {
         sync(_feedHandler->getSerialNum());
         serialNum = _feedHandler->incSerialNum();
         _config_store->saveConfig(*configSnapshot, serialNum);
         // save entry in transaction log
         NewConfigOperation op(serialNum, *_config_store);
-        _feedHandler->storeOperationSync(op);
+        commit_result = _feedHandler->storeOperationSync(op);
         sync(op.getSerialNum());
     }
     bool hasVisibilityDelayChanged = false;
     {
         bool elidedConfigSave = equalReplayConfig && tlsReplayDone;
         // Flush changes to attributes and memory index, cf. visibilityDelay
-        _feedView.get()->forceCommit(elidedConfigSave ? serialNum : serialNum - 1);
+        _feedView.get()->forceCommit(elidedConfigSave ? serialNum : serialNum - 1, std::make_shared<KeepAlive<FeedHandler::CommitResult>>(std::move(commit_result)));
         _writeService.sync();
         vespalib::duration visibilityDelay = configSnapshot->getMaintenanceConfigSP()->getVisibilityDelay();
         hasVisibilityDelayChanged = (visibilityDelay != _visibility.getVisibilityDelay());
@@ -631,7 +639,7 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot)
     // pruned at once anyway.
     // save noop entry in transaction log
     NoopOperation op;
-    _feedHandler->storeOperationSync(op);
+    (void) _feedHandler->storeOperationSync(op);
     sync(op.getSerialNum());
     // Wipe everything in transaction log before initial config.
     try {
@@ -655,7 +663,7 @@ DocumentDB::resumeSaveConfig()
     SerialNum confSerial = _feedHandler->incSerialNum();
     // resume operation, i.e. save config entry in transaction log
     NewConfigOperation op(confSerial, *_config_store);
-    _feedHandler->storeOperationSync(op);
+    (void) _feedHandler->storeOperationSync(op);
     sync(op.getSerialNum());
 }
 
@@ -816,7 +824,7 @@ DocumentDB::enterRedoReprocessState()
         runner.run();
         _subDBs.onReprocessDone(_feedHandler->getSerialNum());
         NoopOperation op;
-        _feedHandler->storeOperationSync(op);
+        (void) _feedHandler->storeOperationSync(op);
         sync(op.getSerialNum());
         _subDBs.pruneRemovedFields(op.getSerialNum());
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index b3076d18703..687d23a36ea 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -550,11 +550,12 @@ FeedHandler::startCommit(DoneCallback onDone) {
     return _tlsWriter->startCommit(std::move(onDone));
 }
 
-void
+FeedHandler::CommitResult
 FeedHandler::storeOperationSync(const FeedOperation &op) {
     vespalib::Gate gate;
-    appendAndCommitOperation(op, make_shared<search::GateCallback>(gate));
+    auto commit_result = appendAndCommitOperation(op, make_shared<search::GateCallback>(gate));
     gate.await();
+    return commit_result;
 }
 
 void
@@ -756,7 +757,7 @@ performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp)
 {
     const LidVectorContext::SP lids_to_remove = pruneOp.getLidsToRemove();
     if (lids_to_remove && lids_to_remove->getNumLids() != 0) {
-        storeOperationSync(pruneOp);
+        appendOperation(pruneOp, DoneCallback());
         _activeFeedView->handlePruneRemovedDocuments(pruneOp);
     }
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
index ec7ecbdd3ae..6d551c785a1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
@@ -235,7 +235,7 @@ public:
     void syncTls(SerialNum syncTo);
     void appendOperation(const FeedOperation &op, DoneCallback onDone) override;
     [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override;
-    void storeOperationSync(const FeedOperation & op);
+    [[nodiscard]] CommitResult storeOperationSync(const FeedOperation & op);
     void considerDelayedPrune();
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h b/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h
index c601f516d95..fee5952d790 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h
@@ -66,7 +66,7 @@ struct ILidSpaceCompactionHandler
     /**
      * Compacts the underlying lid space by starting using the new lid limit.
      */
-    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<search::IDestructorCallback> compact_done_context) = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h
index c3b76a9db75..b4e0667aaa4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h
@@ -22,9 +22,9 @@ struct IOperationStorer
      */
     virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0;
     [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0;
-    void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) {
-        appendOperation(op, onDone);
-        (void) startCommit(std::move(onDone));
+    [[nodiscard]] CommitResult appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) {
+        appendOperation(op, DoneCallback());
+        return startCommit(std::move(onDone));
     }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
index 8aeaf7fd9e4..d55e90b7446 100644
--- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
@@ -62,10 +62,11 @@ LidSpaceCompactionHandler::handleMove(const MoveOperation& op, IDestructorCallba
 }
 
 void
-LidSpaceCompactionHandler::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+LidSpaceCompactionHandler::handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback> compact_done_context)
 {
     assert(_subDb.sub_db_id() == op.getSubDbId());
     _subDb.feed_view()->handleCompactLidSpace(op);
+    _subDb.feed_view()->forceCommit(op.getSerialNum(), std::move(compact_done_context));
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h
index dea573122df..faff14d5c35 100644
--- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h
@@ -29,7 +29,7 @@ public:
     virtual IDocumentScanIterator::UP getIterator() const override;
     virtual MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const override;
     virtual void handleMove(const MoveOperation &op, std::shared_ptr<search::IDestructorCallback> doneCtx) override;
-    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<search::IDestructorCallback> compact_done_context) override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
index 468850b4409..6b480f02863 100644
--- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
@@ -76,15 +76,26 @@ LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats)
     return false; // more work to do (scan documents or compact lid space)
 }
 
+namespace {
+
+template <typename T>
+struct KeepAlive : public search::IDestructorCallback {
+    explicit KeepAlive(T toKeep) noexcept : _toKeep(std::move(toKeep)) { }
+    ~KeepAlive() override = default;
+    T _toKeep;
+};
+
+}
+
 void
 LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats)
 {
     uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1;
     CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit);
     vespalib::Gate gate;
-    _opStorer.appendAndCommitOperation(op, std::make_shared<search::GateCallback>(gate));
+    auto commit_result = _opStorer.appendAndCommitOperation(op, std::make_shared<search::GateCallback>(gate));
     gate.await();
-    _handler.handleCompactLidSpace(op);
+    _handler.handleCompactLidSpace(op, std::make_shared<KeepAlive<decltype(commit_result)>>(std::move(commit_result)));
     EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit);
     _shouldCompactLidSpace = false;
 }