Ensure that we call commit and wait before changing feedview.

Use a token to ensure the we do not loose any tracked lids.

16 files changed, 101 insertions, 43 deletions

diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index 7e5006bab38..1da35c9f5c3 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -7,7 +7,7 @@
 #include <vespa/searchcore/proton/attribute/attributemanager.h>
 #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/docsummary/summarymanager.h>
-#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h>
 #include <vespa/searchcore/proton/index/index_writer.h>
 #include <vespa/searchcore/proton/index/indexmanager.h>
 #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 2785b744265..61d6b409175 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchcore/proton/common/icommitable.h>
 #include <vespa/searchcore/proton/initializer/task_runner.h>
 #include <vespa/searchcore/proton/metrics/attribute_metrics.h>
 #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h>
@@ -88,25 +89,25 @@ struct MySubDBOwner : public IDocumentSubDBOwner
 
 struct MySyncProxy : public SyncProxy
 {
-    virtual void sync(SerialNum) override {}
+    void sync(SerialNum) override {}
 };
 
 
 struct MyGetSerialNum : public IGetSerialNum
 {
-    virtual SerialNum getSerialNum() const override { return 0u; }
+    SerialNum getSerialNum() const override { return 0u; }
 };
 
 struct MyFileHeaderContext : public FileHeaderContext
 {
-    virtual void addTags(vespalib::GenericHeader &, const vespalib::string &) const override {}
+    void addTags(vespalib::GenericHeader &, const vespalib::string &) const override {}
 };
 
 struct MyMetricsWireService : public DummyWireService
 {
     std::set<vespalib::string> _attributes;
     MyMetricsWireService() : _attributes() {}
-    virtual void addAttribute(AttributeMetrics &, const std::string &name) override {
+    void addAttribute(AttributeMetrics &, const std::string &name) override {
         _attributes.insert(name);
     }
 };
@@ -146,7 +147,8 @@ struct MyStoreOnlyContext
     StoreOnlyContext _ctx;
     MyStoreOnlyContext(IThreadingService &writeService,
                        std::shared_ptr<BucketDBOwner> bucketDB,
-                       IBucketDBHandlerInitializer & bucketDBHandlerInitializer);
+                       IBucketDBHandlerInitializer & bucketDBHandlerInitializer,
+                       ICommitable & commitable);
     ~MyStoreOnlyContext();
     const MySubDBOwner &getOwner() const {
         return _owner;
@@ -155,7 +157,8 @@ struct MyStoreOnlyContext
 
 MyStoreOnlyContext::MyStoreOnlyContext(IThreadingService &writeService,
                                        std::shared_ptr<BucketDBOwner> bucketDB,
-                                       IBucketDBHandlerInitializer &bucketDBHandlerInitializer)
+                                       IBucketDBHandlerInitializer &bucketDBHandlerInitializer,
+                                       ICommitable &)
     : _owner(), _syncProxy(), _getSerialNum(), _fileHeader(),
       _metrics(DOCTYPE_NAME, 1), _configMutex(), _hwInfo(),
       _ctx(_owner, _syncProxy, _getSerialNum, _fileHeader, writeService, bucketDB,
@@ -182,7 +185,8 @@ struct MyFastAccessContext
     FastAccessContext _ctx;
     MyFastAccessContext(IThreadingService &writeService,
                         std::shared_ptr<BucketDBOwner> bucketDB,
-                        IBucketDBHandlerInitializer & bucketDBHandlerInitializer);
+                        IBucketDBHandlerInitializer & bucketDBHandlerInitializer,
+                        ICommitable & commitable);
     ~MyFastAccessContext();
     const MyMetricsWireService &getWireService() const {
         return _wireService;
@@ -194,8 +198,9 @@ struct MyFastAccessContext
 
 MyFastAccessContext::MyFastAccessContext(IThreadingService &writeService,
                                          std::shared_ptr<BucketDBOwner> bucketDB,
-                                         IBucketDBHandlerInitializer & bucketDBHandlerInitializer)
-    : _storeOnlyCtx(writeService, bucketDB, bucketDBHandlerInitializer),
+                                         IBucketDBHandlerInitializer & bucketDBHandlerInitializer,
+                                         ICommitable & commitable)
+    : _storeOnlyCtx(writeService, bucketDB, bucketDBHandlerInitializer, commitable),
       _attributeMetrics(nullptr),
       _wireService(),
       _ctx(_storeOnlyCtx._ctx, _attributeMetrics, _wireService)
@@ -219,7 +224,8 @@ struct MySearchableContext
     SearchableContext _ctx;
     MySearchableContext(IThreadingService &writeService,
                         std::shared_ptr<BucketDBOwner> bucketDB,
-                        IBucketDBHandlerInitializer & bucketDBHandlerInitializer);
+                        IBucketDBHandlerInitializer & bucketDBHandlerInitializer,
+                        ICommitable & commitable);
     ~MySearchableContext();
     const MyMetricsWireService &getWireService() const {
         return _fastUpdCtx.getWireService();
@@ -232,10 +238,13 @@ struct MySearchableContext
 
 MySearchableContext::MySearchableContext(IThreadingService &writeService,
                                          std::shared_ptr<BucketDBOwner> bucketDB,
-                                         IBucketDBHandlerInitializer & bucketDBHandlerInitializer)
-    : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer),
+                                         IBucketDBHandlerInitializer & bucketDBHandlerInitializer,
+                                         ICommitable & commitable)
+    : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer, commitable),
       _queryLimiter(), _clock(),
-      _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, dynamic_cast<vespalib::SyncableThreadExecutor &>(writeService.shared()))
+      _ctx(_fastUpdCtx._ctx, _queryLimiter,
+           _clock, dynamic_cast<vespalib::SyncableThreadExecutor &>(writeService.shared()),
+           commitable)
 {}
 MySearchableContext::~MySearchableContext() = default;
 
@@ -253,6 +262,14 @@ struct TwoAttrSchema : public OneAttrSchema
     }
 };
 
+struct Committer : public ICommitable {
+    size_t _commitCount;
+    size_t _commitAndWaitCount;
+    Committer() : _commitCount(0), _commitAndWaitCount(0) { }
+    void commit() override { _commitCount++; }
+    void commitAndWait() override { _commitAndWaitCount++; }
+};
+
 struct MyConfigSnapshot
 {
     typedef std::unique_ptr<MyConfigSnapshot> UP;
@@ -291,6 +308,7 @@ struct FixtureBase
     typename Traits::Config _cfg;
     std::shared_ptr<BucketDBOwner> _bucketDB;
     BucketDBHandler _bucketDBHandler;
+    Committer       _committer;
     typename Traits::Context _ctx;
     typename Traits::Schema _baseSchema;
     MyConfigSnapshot::UP _snapshot;
@@ -303,7 +321,7 @@ struct FixtureBase
           _cfg(),
           _bucketDB(std::make_shared<BucketDBOwner>()),
           _bucketDBHandler(*_bucketDB),
-          _ctx(_writeService, _bucketDB, _bucketDBHandler),
+          _ctx(_writeService, _bucketDB, _bucketDBHandler, _committer),
           _baseSchema(),
           _snapshot(new MyConfigSnapshot(_baseSchema, Traits::ConfigDir::dir())),
           _baseDir(BASE_DIR + "/" + SUB_NAME, BASE_DIR),
@@ -327,7 +345,7 @@ struct FixtureBase
                 vespalib::ThreadStackExecutor executor(1, 1024 * 1024);
                 initializer::TaskRunner taskRunner(executor);
                 taskRunner.runTask(task);
-        SessionManager::SP sessionMgr(new SessionManager(1));
+        auto sessionMgr = std::make_shared<SessionManager>(1);
         runInMaster([&] () { _subDb.initViews(*_snapshot->_cfg, sessionMgr); });
     }
     void basicReconfig(SerialNum serialNum) {
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index 315fa1591e4..e6bcc60e120 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -4,7 +4,7 @@
 #include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
 #include <vespa/searchcore/proton/test/bucketfactory.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
-#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h>
 #include <vespa/searchcore/proton/index/i_index_writer.h>
 #include <vespa/searchcore/proton/server/executorthreadingservice.h>
 #include <vespa/searchcore/proton/server/isummaryadapter.h>
@@ -739,7 +739,7 @@ struct SearchableFeedViewFixture : public FixtureBase
            SearchableFeedView::Context(iw))
     {
     }
-    virtual IFeedView &getFeedView() override { return fv; }
+    IFeedView &getFeedView() override { return fv; }
 };
 
 struct FastAccessFeedViewFixture : public FixtureBase
@@ -1221,6 +1221,7 @@ TEST_F("require that commit is not called when inside a commit interval",
                   "remove(adapter=attribute,serialNum=2,lid=1,commit=0),"
                   "remove(adapter=index,serialNum=2,lid=1,commit=0),"
                   "ack(Result(0, ))");
+    f.forceCommitAndWait();
 }
 
 TEST_F("require that commit is not implicitly called",
@@ -1243,6 +1244,7 @@ TEST_F("require that commit is not implicitly called",
                   "remove(adapter=attribute,serialNum=2,lid=1,commit=0),"
                   "remove(adapter=index,serialNum=2,lid=1,commit=0),"
                   "ack(Result(0, ))");
+    f.forceCommitAndWait();
 }
 
 TEST_F("require that forceCommit updates docid limit",
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
index 56a19b2d7eb..7a2fadeaa2d 100644
--- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
@@ -3,7 +3,7 @@
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/searchcommon/common/schema.h>
-#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h>
 #include <vespa/searchcore/proton/server/executorthreadingservice.h>
 #include <vespa/searchcore/proton/server/putdonecontext.h>
 #include <vespa/searchcore/proton/server/removedonecontext.h>
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
index f86829c5b13..32cc9d989d8 100644
--- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
@@ -144,6 +144,10 @@ public:
     {
     }
 
+    ~Fixture() {
+        commit();
+    }
+
     template <typename FunctionType>
     void runInMaster(FunctionType func) {
         test::runInMaster(_writeService, func);
diff --git a/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp
index 79bf970aeac..4616f256863 100644
--- a/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp
@@ -6,6 +6,17 @@
 
 namespace proton {
 
+PendingLidTracker::Token::Token(uint32_t lid, PendingLidTracker &tracker)
+    : _tracker(&tracker),
+      _lid(lid)
+{}
+
+PendingLidTracker::Token::~Token() {
+    if (_tracker != nullptr) {
+        _tracker->consume(_lid);
+    }
+}
+
 PendingLidTracker::PendingLidTracker()
     : _mutex(),
       _cond(),
@@ -16,10 +27,11 @@ PendingLidTracker::~PendingLidTracker() {
     assert(_pending.empty());
 }
 
-void
+PendingLidTracker::Token
 PendingLidTracker::produce(uint32_t lid) {
     std::lock_guard<std::mutex> guard(_mutex);
     _pending[lid]++;
+    return Token(lid, *this);
 }
 void
 PendingLidTracker::consume(uint32_t lid) {
diff --git a/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h
index 4fc718fd465..a9b3a1b05f4 100644
--- a/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h
+++ b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h
@@ -10,12 +10,29 @@ namespace proton {
 
 class PendingLidTracker {
 public:
+    class Token {
+    public:
+        Token(uint32_t lid, PendingLidTracker & tracker);
+        Token(const Token &) = delete;
+        Token & operator = (const Token &) = delete;
+        Token(Token && rhs) noexcept
+            : _tracker(rhs._tracker),
+              _lid(rhs._lid)
+        {
+            rhs._tracker = nullptr;
+        }
+        ~Token();
+    private:
+        PendingLidTracker * _tracker;
+        uint32_t            _lid;
+    };
     PendingLidTracker();
     ~PendingLidTracker();
-    void produce(uint32_t lid);
-    void consume(uint32_t lid);
+    Token produce(uint32_t lid);
     void waitForConsumedLid(uint32_t lid);
 private:
+    friend Token;
+    void consume(uint32_t lid);
     std::mutex _mutex;
     std::condition_variable _cond;
     vespalib::hash_map<uint32_t, uint32_t> _pending;
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp
index b74078691f9..03dfd83a132 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp
@@ -21,7 +21,9 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume
 {
 }
 
-LidReuseDelayer::~LidReuseDelayer() = default;
+LidReuseDelayer::~LidReuseDelayer() {
+    assert(_pendingLids.empty());
+}
 
 bool
 LidReuseDelayer::delayReuse(uint32_t lid)
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 3fea61e0df3..96a93f0ac16 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -167,14 +167,14 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
       _writeFilter(),
       _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()),
       _feedHandler(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsDirectWriter),
-      _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext,
+      _visibility(_feedHandler, _writeService, _feedView),
+      _subDBs(*this, *this, _feedHandler, _visibility, _docTypeName, _writeService, warmupExecutor, fileHeaderContext,
               metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir,
               makeSubDBConfig(protonCfg.distribution,
                               findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation,
                               protonCfg.numsearcherthreads),
               hwInfo),
       _maintenanceController(_writeService.master(), sharedExecutor, _docTypeName),
-      _visibility(_feedHandler, _writeService, _feedView),
       _lidSpaceCompactionHandlers(),
       _jobTrackers(),
       _calc(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 7db4e908878..0c93dc88ada 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -137,10 +137,9 @@ private:
     AttributeUsageFilter          _writeFilter;
     std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider;
     FeedHandler                   _feedHandler;
-
+    VisibilityHandler             _visibility;
     DocumentSubDBCollection       _subDBs;
     MaintenanceController         _maintenanceController;
-    VisibilityHandler             _visibility;
     ILidSpaceCompactionHandler::Vector _lidSpaceCompactionHandlers;
     DocumentDBJobTrackers         _jobTrackers;
     IBucketStateCalculator::SP    _calc;
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 536180fbab4..dbff8e632bb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -32,6 +32,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
         IDocumentSubDBOwner &owner,
         search::transactionlog::SyncProxy &tlSyncer,
         const IGetSerialNum &getSerialNum,
+        ICommitable & commitable,
         const DocTypeName &docTypeName,
         searchcorespi::index::IThreadingService &writeService,
         vespalib::SyncableThreadExecutor &warmupExecutor,
@@ -72,7 +73,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
                     cfg.getNumSearchThreads()),
                 SearchableDocSubDB::Context(
                         FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService),
-                        queryLimiter, clock, warmupExecutor)));
+                        queryLimiter, clock, warmupExecutor, commitable)));
 
     _subDBs.push_back
         (new StoreOnlyDocSubDB(
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index 83ebef18274..8868e43dd8c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -102,6 +102,7 @@ public:
             IDocumentSubDBOwner &owner,
             search::transactionlog::SyncProxy &tlSyncer,
             const IGetSerialNum &getSerialNum,
+            ICommitable & commitable,
             const DocTypeName &docTypeName,
             searchcorespi::index::IThreadingService &writeService,
             vespalib::SyncableThreadExecutor &warmupExecutor,
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 790fa7a0125..6ff5a149641 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -6,6 +6,7 @@
 #include "reconfig_params.h"
 #include "i_document_subdb_owner.h"
 #include "ibucketstatecalculator.h"
+#include <vespa/searchcore/proton/common/icommitable.h>
 #include <vespa/searchcore/proton/attribute/attribute_writer.h>
 #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
 #include <vespa/searchcore/proton/index/index_manager_initializer.h>
@@ -45,6 +46,7 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx)
       _configurer(_iSummaryMgr, _rSearchView, _rFeedView, ctx._queryLimiter, _constantValueRepo, ctx._clock,
                   getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()),
       _warmupExecutor(ctx._warmupExecutor),
+      _commitable(ctx._commitable),
       _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()),
       _flushConfig(),
       _nodeRetired(false)
@@ -245,8 +247,7 @@ SearchableDocSubDB::initFeedView(IAttributeWriter::SP attrWriter,
  * document type, flushing might occur during replay.
  */
 bool
-SearchableDocSubDB::
-reconfigure(vespalib::Closure0<bool>::UP closure)
+SearchableDocSubDB::reconfigure(vespalib::Closure0<bool>::UP closure)
 {
     assert(_writeService.master().isCurrentThread());
 
@@ -270,6 +271,7 @@ SearchableDocSubDB::reconfigureIndexSearchable()
 {
     std::lock_guard<std::mutex> guard(_configMutex);
     // Create new views as needed.
+    _commitable.commitAndWait();
     _configurer.reconfigureIndexSearchable();
     // Activate new feed view at once
     syncViews();
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 31c77acff30..dedfc7e80c1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -27,6 +27,7 @@ class DocumentDBConfig;
 struct IDocumentDBReferenceResolver;
 struct MetricsWireService;
 class GidToLidChangeHandler;
+class ICommitable;
 
 /**
  * The searchable sub database supports searching and keeps all attribute fields in memory and
@@ -56,15 +57,18 @@ public:
         matching::QueryLimiter            &_queryLimiter;
         const vespalib::Clock             &_clock;
         vespalib::SyncableThreadExecutor  &_warmupExecutor;
+        ICommitable                       &_commitable;
 
         Context(const FastAccessDocSubDB::Context &fastUpdCtx,
                 matching::QueryLimiter &queryLimiter,
                 const vespalib::Clock &clock,
-                vespalib::SyncableThreadExecutor &warmupExecutor)
+                vespalib::SyncableThreadExecutor &warmupExecutor,
+                ICommitable & commitable)
             : _fastUpdCtx(fastUpdCtx),
               _queryLimiter(queryLimiter),
               _clock(clock),
-              _warmupExecutor(warmupExecutor)
+              _warmupExecutor(warmupExecutor),
+              _commitable(commitable)
         { }
     };
 
@@ -81,6 +85,7 @@ private:
     matching::ConstantValueRepo                 _constantValueRepo;
     SearchableDocSubDBConfigurer                _configurer;
     vespalib::SyncableThreadExecutor           &_warmupExecutor;
+    ICommitable                                &_commitable;
     std::shared_ptr<GidToLidChangeHandler>      _realGidToLidChangeHandler;
     DocumentDBFlushConfig                       _flushConfig;
     bool                                        _nodeRetired;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 7a3652692fe..362d328bd71 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -14,7 +14,6 @@
 #include <vespa/searchcore/proton/attribute/attribute_utils.h>
 #include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
-#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
 #include <vespa/searchcore/proton/feedoperation/operations.h>
 #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h>
 #include <vespa/searchlib/common/gatecallback.h>
@@ -354,41 +353,38 @@ StoreOnlyFeedView::handleUpdate(FeedToken token, const UpdateOperation &updOp)
 void StoreOnlyFeedView::putSummary(SerialNum serialNum, Lid lid,
                                    FutureStream futureStream, OnOperationDoneType onDone)
 {
-    _pendingLidTracker.produce(lid);
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winline" // Avoid spurious inlining warning from GCC related to lambda destructor.
     summaryExecutor().execute(
-            makeLambdaTask([serialNum, lid, futureStream = std::move(futureStream), onDone, this] () mutable {
+            makeLambdaTask([serialNum, lid, futureStream = std::move(futureStream), trackerToken = _pendingLidTracker.produce(lid), onDone, this] () mutable {
                 (void) onDone;
+                (void) trackerToken;
                 vespalib::nbostream os = futureStream.get();
                 if (!os.empty()) {
                     _summaryAdapter->put(serialNum, lid, os);
                 }
-                _pendingLidTracker.consume(lid);
             }));
 #pragma GCC diagnostic pop
 }
 
 void StoreOnlyFeedView::putSummary(SerialNum serialNum, Lid lid, Document::SP doc, OnOperationDoneType onDone)
 {
-    _pendingLidTracker.produce(lid);
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winline" // Avoid spurious inlining warning from GCC related to lambda destructor.
     summaryExecutor().execute(
-            makeLambdaTask([serialNum, doc = std::move(doc), onDone, lid, this] {
+            makeLambdaTask([serialNum, doc = std::move(doc), trackerToken = _pendingLidTracker.produce(lid), onDone, lid, this] {
                 (void) onDone;
+                (void) trackerToken;
                 _summaryAdapter->put(serialNum, lid, *doc);
-                _pendingLidTracker.consume(lid);
             }));
 #pragma GCC diagnostic pop
 }
 void StoreOnlyFeedView::removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone) {
-    _pendingLidTracker.produce(lid);
     summaryExecutor().execute(
-            makeLambdaTask([serialNum, lid, onDone, this] {
+            makeLambdaTask([serialNum, lid, onDone, trackerToken = _pendingLidTracker.produce(lid), this] {
                 (void) onDone;
+                (void) trackerToken;
                 _summaryAdapter->remove(serialNum, lid);
-                _pendingLidTracker.consume(lid);
             }));
 }
 void StoreOnlyFeedView::heartBeatSummary(SerialNum serialNum) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
index 5cc5743041d..2865d8c0536 100644
--- a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "visibilityhandler.h"
-#include <vespa/searchcore/proton/common/pendinglidtracker.h>
 #include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/util/closuretask.h>