Wire in OnDoneCallback in handleLidSpaceCompaction to avoid brute force sync_all_executors.

14 files changed, 38 insertions, 28 deletions

diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
index 5ef02f60152..db20181eae7 100644
--- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
@@ -87,7 +87,7 @@ struct MyFeedView : public test::DummyFeedView
     void handleMove(const MoveOperation &, IDestructorCallback::SP) override { ++_handleMove; }
     void heartBeat(SerialNum) override { ++_heartBeat; }
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &) override { ++_handlePrune; }
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override {
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback) override {
         _wantedLidLimit = op.getLidLimit();
     }
 };
@@ -429,7 +429,7 @@ TEST_F("require that calculator can be updated", Fixture)
 
 TEST_F("require that compactLidSpace() is sent to correct feed view", Fixture)
 {
-    f._view.handleCompactLidSpace(CompactLidSpaceOperation(1, 99));
+    f._view.handleCompactLidSpace(CompactLidSpaceOperation(1, 99), IDestructorCallback::SP());
     EXPECT_EQUAL(0u, f._ready._view->_wantedLidLimit);
     EXPECT_EQUAL(99u, f._removed._view->_wantedLidLimit);
     EXPECT_EQUAL(0u, f._notReady._view->_wantedLidLimit);
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index d3bbcda78f7..6c45470bb42 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -38,6 +38,8 @@ using search::AttributeVector;
 using search::CacheStats;
 using search::DocumentMetaData;
 using vespalib::IDestructorCallback;
+using vespalib::Gate;
+using vespalib::GateCallback;
 using search::SearchableStats;
 using search::index::schema::CollectionType;
 using search::index::schema::DataType;
@@ -404,7 +406,7 @@ MyAttributeWriter::~MyAttributeWriter() = default;
 struct MyTransport : public feedtoken::ITransport
 {
     ResultUP lastResult;
-    vespalib::Gate _gate;
+    Gate _gate;
     MyTracer &_tracer;
     MyTransport(MyTracer &tracer);
     ~MyTransport();
@@ -610,8 +612,8 @@ struct FixtureBase
     void moveAndWait(const DocumentContext &docCtx, uint32_t fromLid, uint32_t toLid) {
         MoveOperation op(docCtx.bid, docCtx.ts, docCtx.doc, DbDocumentId(pc._params._subDbId, fromLid), pc._params._subDbId);
         op.setTargetLid(toLid);
-        vespalib::Gate gate;
-        runInMaster([&, onDone=std::make_shared<vespalib::GateCallback>(gate)]() { performMove(op, std::move(onDone)); });
+        Gate gate;
+        runInMaster([&, onDone=std::make_shared<GateCallback>(gate)]() { performMove(op, std::move(onDone)); });
         gate.await();
     }
 
@@ -621,12 +623,12 @@ struct FixtureBase
         getFeedView().handleDeleteBucket(op);
     }
 
-    void performForceCommit(vespalib::IDestructorCallback::SP onDone) {
+    void performForceCommit(IDestructorCallback::SP onDone) {
         getFeedView().forceCommit(serial, std::move(onDone));
     }
     void forceCommitAndWait() {
-        vespalib::Gate gate;
-        runInMaster([this, onDone=std::make_shared<vespalib::GateCallback>(gate)]() {
+        Gate gate;
+        runInMaster([this, onDone=std::make_shared<GateCallback>(gate)]() {
             performForceCommit(std::move(onDone));
         });
         gate.await();
@@ -650,14 +652,18 @@ struct FixtureBase
         return docs;
     }
 
-    void performCompactLidSpace(uint32_t wantedLidLimit) {
+    void performCompactLidSpace(uint32_t wantedLidLimit, IDestructorCallback::SP onDone) {
         auto &fv = getFeedView();
         CompactLidSpaceOperation op(0, wantedLidLimit);
         op.setSerialNum(++serial);
-        fv.handleCompactLidSpace(op);
+        fv.handleCompactLidSpace(op, std::move(onDone));
     }
     void compactLidSpaceAndWait(uint32_t wantedLidLimit) {
-        runInMasterAndSyncAll([&]() { performCompactLidSpace(wantedLidLimit); });
+        Gate gate;
+        runInMaster([&]() {
+            performCompactLidSpace(wantedLidLimit, std::make_shared<GateCallback>(gate));
+        });
+        gate.await();
     }
     void assertChangeHandler(document::GlobalId expGid, uint32_t expLid, uint32_t expChanges) {
         _gidToLidChangeHandler->assertChanges(expGid, expLid, expChanges);
@@ -1159,7 +1165,11 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
     EXPECT_TRUE(assertThreadObserver(5, 4, 4, f.writeServiceObserver()));
     CompactLidSpaceOperation op(0, 2);
     op.setSerialNum(0);
-    f.runInMasterAndSyncAll([&]() { f.fv.handleCompactLidSpace(op); });
+    Gate gate;
+    f.runInMaster([&, onDone=std::make_shared<GateCallback>(gate)]() {
+        f.fv.handleCompactLidSpace(op, std::move(onDone));
+    });
+    gate.await();
     // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
     EXPECT_TRUE(assertThreadObserver(6, 6, 5, f.writeServiceObserver()));
     EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
index f7702708c53..18215b01ae2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
@@ -241,11 +241,11 @@ handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp)
 }
 
 void
-CombiningFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+CombiningFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone)
 {
     uint32_t subDbId = op.getSubDbId();
     assert(subDbId < _views.size());
-    _views[subDbId]->handleCompactLidSpace(op);
+    _views[subDbId]->handleCompactLidSpace(op, std::move(onDone));
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
index 3827b491896..700787e126e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
@@ -77,7 +77,7 @@ public:
     void handleMove(const MoveOperation &moveOp, std::shared_ptr<vespalib::IDestructorCallback> moveDoneCtx) override;
     void heartBeat(search::SerialNum serialNum) override;
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override;
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone) override;
 
     // Called by document db executor
     void setCalculator(const std::shared_ptr<IBucketStateCalculator> &newCalc);
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index e2b3887c60c..25be7e4f504 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -69,13 +69,13 @@ FastAccessFeedView::FastAccessFeedView(StoreOnlyFeedView::Context storeOnlyCtx,
 FastAccessFeedView::~FastAccessFeedView() = default;
 
 void
-FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone)
 {
     // Drain pending PutDoneContext and ForceCommitContext objects
     forceCommitAndWait(search::CommitParam(op.getSerialNum()));
     _docIdLimit.set(op.getLidLimit());
     getAttributeWriter()->compactLidSpace(op.getLidLimit(), op.getSerialNum());
-    Parent::handleCompactLidSpace(op);
+    Parent::handleCompactLidSpace(op, onDone);
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
index a27bf04d701..3b8c755046c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
@@ -62,7 +62,7 @@ public:
         return _docIdLimit;
     }
 
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone) override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
index 8b616dc672e..8de727d664f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
@@ -102,7 +102,7 @@ public:
     void replay(const CreateBucketOperation &) override {
     }
     void replay(const CompactLidSpaceOperation &op) override {
-        _feed_view_ptr->handleCompactLidSpace(op);
+        _feed_view_ptr->handleCompactLidSpace(op, vespalib::IDestructorCallback::SP());
     }
     NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() override {
         return _config_store;
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
index 14e6d6811e7..4c775e0a765 100644
--- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
@@ -64,7 +64,7 @@ public:
     void forceCommit(search::SerialNum serialNum) { forceCommit(CommitParam(serialNum)); }
     void forceCommitAndWait(CommitParam param);
     virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0;
-    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone) = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
index 185d197add7..84053786f69 100644
--- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
@@ -93,7 +93,7 @@ void
 LidSpaceCompactionHandler::handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback> compact_done_context)
 {
     assert(_subDb.sub_db_id() == op.getSubDbId());
-    _subDb.feed_view()->handleCompactLidSpace(op);
+    _subDb.feed_view()->handleCompactLidSpace(op, compact_done_context);
     _subDb.feed_view()->forceCommit(CommitParam(op.getSerialNum()), std::move(compact_done_context));
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
index 323ca9add17..6164624c210 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
@@ -167,9 +167,9 @@ SearchableFeedView::performIndexForceCommit(SerialNum serialNum, OnForceCommitDo
 }
 
 void
-SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone)
 {
-    Parent::handleCompactLidSpace(op);
+    Parent::handleCompactLidSpace(op, onDone);
     vespalib::Gate gate;
     _writeService.index().execute(
             makeLambdaTask([this, &op, &gate]() {
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
index 5ff309446d2..a97bebef702 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
@@ -58,7 +58,7 @@ public:
 
     ~SearchableFeedView() override;
     const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; }
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone) override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 527eaff07cc..c82a047e5e9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -745,7 +745,7 @@ handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp)
 }
 
 void
-StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone)
 {
     assert(_params._subDbId == op.getSubDbId());
     const SerialNum serialNum = op.getSerialNum();
@@ -754,7 +754,7 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
         auto commitContext(std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore,
                                                                 _pendingLidsForCommit->produceSnapshot(),
                                                                 _gidToLidChangeHandler.grab_pending_changes(),
-                                                                DoneCallback()));
+                                                                onDone));
         commitContext->holdUnblockShrinkLidSpace();
         internalForceCommit(CommitParam(serialNum), commitContext);
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
index c55e451d923..02332fcda6e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -245,7 +245,7 @@ public:
      * Called by writer thread.
      */
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override;
-    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op, DoneCallback onDone) override;
     std::shared_ptr<PendingLidTrackerBase> getUncommittedLidTracker() { return _pendingLidsForCommit; }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
index af88f7fa6b7..61740c414ef 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
@@ -30,7 +30,7 @@ struct DummyFeedView : public IFeedView
     void handleMove(const MoveOperation &, std::shared_ptr<vespalib::IDestructorCallback>) override {}
     void heartBeat(search::SerialNum) override {}
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &) override {}
-    void handleCompactLidSpace(const CompactLidSpaceOperation &) override {}
+    void handleCompactLidSpace(const CompactLidSpaceOperation &, DoneCallback) override {}
     void forceCommit(const CommitParam &, DoneCallback) override { }
 };