Wait for trackers to drain.

8 files changed, 36 insertions, 13 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
index a5efb0a66e9..5f7f26dfadd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
@@ -248,6 +248,17 @@ CombiningFeedView::getUncommittedLidsTracker() {
     LOG_ABORT("CombiningFeedView::getUncommittedLidsTracker should never be called.");
 }
 
+bool
+CombiningFeedView::isDrained() const
+{
+    for (const auto &view : _views) {
+        if ( ! view->isDrained()) {
+            return false;
+        }
+    }
+    return true;
+}
+
 void
 CombiningFeedView::
 handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp)
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
index 3a37fdc37cb..15bbf73ac04 100644
--- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
@@ -80,6 +80,7 @@ public:
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override;
     void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
     ILidCommitState & getUncommittedLidsTracker() override;
+    bool isDrained() const override;
 
     // Called by document db executor
     void setCalculator(const IBucketStateCalculator::SP &newCalc);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index b452c9335c1..3e3ed260a90 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -493,15 +493,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum
     }
 }
 
-
-namespace {
-void
-doNothing(IFeedView::SP)
-{
-    // Called by index executor, delays when feed view is dropped.
-}
-}  // namespace
-
 void
 DocumentDB::performDropFeedView(IFeedView::SP feedView)
 {
@@ -517,15 +508,25 @@ DocumentDB::performDropFeedView(IFeedView::SP feedView)
 
 
 void
-DocumentDB::performDropFeedView2(IFeedView::SP feedView)
-{
+DocumentDB::performDropFeedView2(IFeedView::SP feedView) {
     // Called by executor task, delays when feed view is dropped.
     // Also called by DocumentDB::receive() method to keep feed view alive
     _writeService.indexFieldInverter().sync();
     _writeService.indexFieldWriter().sync();
+    masterExecute([this, feedView]() { performDropFeedView3(feedView, 10); });
+}
 
-    // Feed view is kept alive in the closure's shared ptr.
-    masterExecute([feedView] () { doNothing(feedView); });
+void
+DocumentDB::performDropFeedView3(IFeedView::SP feedView, uint32_t numRetries) {
+    // We must keep the feedView allive until all operations are drained.
+    // TODO: This is a very brittle appraoch that we should reconsider.
+    if (feedView && ! feedView->isDrained()) {
+        LOG(warning, "FeedView for document type '%s' has not been drained. Reposting to check again. %d retries left",
+            getName().c_str(), numRetries);
+        if (numRetries > 0) {
+            masterExecute([this, feedView, numRetries]() { performDropFeedView3(feedView, numRetries - 1); });
+        }
+    }
 }
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index ef0ed729622..4c4840446fe 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -192,6 +192,7 @@ private:
      */
     void performDropFeedView(IFeedView::SP feedView);
     void performDropFeedView2(IFeedView::SP feedView);
+    void performDropFeedView3(IFeedView::SP feedView, uint32_t numRetries);
 
     /**
      * Implements IFeedHandlerOwner
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
index 8dd7ae8474e..d575b08e47b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
@@ -66,6 +66,7 @@ public:
     virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
     virtual ILidCommitState & getUncommittedLidsTracker() = 0;
     virtual bool allowEarlyAck() const = 0;
+    virtual bool isDrained() const = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 186c321d920..cac32e3a7ed 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -879,4 +879,10 @@ StoreOnlyFeedView::getDocumentMetaStorePtr() const
     return &_documentMetaStoreContext->get();
 }
 
+bool
+StoreOnlyFeedView::isDrained() const
+{
+    return _pendingLidsForDocStore.getState() == ILidCommitState::State::COMPLETED;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
index da1459d521c..93803e3300e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -245,6 +245,7 @@ public:
     void heartBeat(search::SerialNum serialNum) override;
     void sync() override;
     void forceCommit(SerialNum serialNum, DoneCallback onDone) override;
+    bool isDrained() const override;
 
     /**
      * Prune lids present in operation.  Caller must call doneSegment()
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
index b96fd77409c..59afbb8f197 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
@@ -35,6 +35,7 @@ struct DummyFeedView : public IFeedView
     void forceCommit(search::SerialNum, DoneCallback) override { }
     ILidCommitState & getUncommittedLidsTracker() override;
     bool allowEarlyAck() const override { return false; }
+    bool isDrained() const override { return true; }
 };
 
 }