Add a vector of gids to remove to ReferenceAttribute::populateTargetLids()

signature. These gids represent the target documents removed since the last
feed view force commit where the live gid to lid mapping in the document
meta store was made available as a frozen tree.

7 files changed, 58 insertions, 13 deletions

diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
index b64e972c4cf..3474a4297c7 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
@@ -38,6 +38,7 @@ class ListenerStats {
     uint32_t  _createdListeners;
     uint32_t  _registeredListeners;
     uint32_t  _destroyedListeners;
+    std::vector<GlobalId> _initial_removes;
 
 public:
     ListenerStats() noexcept
@@ -46,7 +47,8 @@ public:
           _removeChanges(0u),
           _createdListeners(0u),
           _registeredListeners(0u),
-          _destroyedListeners(0u)
+          _destroyedListeners(0u),
+          _initial_removes()
     {
     }
 
@@ -64,7 +66,7 @@ public:
         ++_removeChanges;
     }
     void markCreatedListener()    { lock_guard guard(_lock); ++_createdListeners; }
-    void markRegisteredListener() { lock_guard guard(_lock); ++_registeredListeners; }
+    void markRegisteredListener(const std::vector<GlobalId>& removes) { lock_guard guard(_lock); ++_registeredListeners; _initial_removes = removes; }
     void markDestroyedListener()  { lock_guard guard(_lock); ++_destroyedListeners; }
 
     uint32_t getCreatedListeners() const { return _createdListeners; }
@@ -84,6 +86,7 @@ public:
         EXPECT_EQUAL(expPutChanges, _putChanges);
         EXPECT_EQUAL(expRemoveChanges, _removeChanges);
     }
+    const std::vector<GlobalId>& get_initial_removes() const noexcept { return _initial_removes; }
 };
 
 class MyListener : public IGidToLidChangeListener
@@ -105,7 +108,7 @@ public:
     ~MyListener() override { _stats.markDestroyedListener(); }
     void notifyPutDone(IDestructorCallbackSP, GlobalId, uint32_t) override { _stats.notifyPutDone(); }
     void notifyRemove(IDestructorCallbackSP, GlobalId) override { _stats.notifyRemove(); }
-    void notifyRegistered() override { _stats.markRegisteredListener(); }
+    void notifyRegistered(const std::vector<GlobalId>& removes) override { _stats.markRegisteredListener(removes); }
     const vespalib::string &getName() const override { return _name; }
     const vespalib::string &getDocTypeName() const override { return _docTypeName; }
 };
@@ -233,6 +236,16 @@ TEST_F("Test that we keep old listener when registering duplicate", Fixture)
     TEST_DO(stats.assertListeners(2, 1, 1));
 }
 
+TEST_F("Test that pending removes are passed on to new listener", Fixture)
+{
+    auto& stats = f.addStats();
+    auto listener = std::make_unique<MyListener>(stats, "test1", "testdoc");
+    f.notifyRemove(toGid(doc1), 20);
+    f.addListener(std::move(listener));
+    EXPECT_TRUE((std::vector<GlobalId>{ toGid(doc1) }) == stats.get_initial_removes());
+    f.commit();
+}
+
 class StatsFixture : public Fixture
 {
     ListenerStats &_stats;
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
index 15281563b93..e3b1fd3aa15 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
@@ -103,8 +103,8 @@ struct Fixture
         gate.await();
     }
 
-    void notifyListenerRegistered() {
-        _listener->notifyRegistered();
+    void notifyListenerRegistered(const std::vector<GlobalId>& removes) {
+        _listener->notifyRegistered(removes);
     }
 };
 
@@ -144,7 +144,7 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur
         std::make_shared<MyGidToLidMapperFactory>();
     f._attr->setGidToLidMapperFactory(factory);
     f.allocListener();
-    f.notifyListenerRegistered();
+    f.notifyListenerRegistered({});
     TEST_DO(f.assertTargetLid(10, 1));
     TEST_DO(f.assertTargetLid(17, 2));
     TEST_DO(f.assertTargetLid(10, 3));
@@ -152,6 +152,31 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur
     TEST_DO(f.assertNoTargetLid(5));
 }
 
+TEST_F("Test that removed target lids are pruned when listener is registered", Fixture)
+{
+    f.ensureDocIdLimit(6);
+    f.set(1, toGid(doc1));
+    f.set(2, toGid(doc2));
+    f.set(3, toGid(doc1));
+    f.set(4, toGid(doc3));
+    f.commit();
+    TEST_DO(f.assertTargetLid(0, 1));
+    TEST_DO(f.assertTargetLid(0, 2));
+    TEST_DO(f.assertTargetLid(0, 3));
+    TEST_DO(f.assertTargetLid(0, 4));
+    TEST_DO(f.assertNoTargetLid(5));
+    std::shared_ptr<search::IGidToLidMapperFactory> factory =
+        std::make_shared<MyGidToLidMapperFactory>();
+    f._attr->setGidToLidMapperFactory(factory);
+    f.allocListener();
+    f.notifyListenerRegistered({ toGid(doc1) });
+    TEST_DO(f.assertTargetLid(0, 1));
+    TEST_DO(f.assertTargetLid(17, 2));
+    TEST_DO(f.assertTargetLid(0, 3));
+    TEST_DO(f.assertTargetLid(0, 4));
+    TEST_DO(f.assertNoTargetLid(5));
+}
+
 }
 
 TEST_MAIN()
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
index 8cff0b1ea6c..c2efa8c2389 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
@@ -26,7 +26,7 @@ public:
     ~MyListener() override { }
     void notifyPutDone(IDestructorCallbackSP, document::GlobalId, uint32_t) override { }
     void notifyRemove(IDestructorCallbackSP, document::GlobalId) override { }
-    void notifyRegistered() override { }
+    void notifyRegistered(const std::vector<document::GlobalId>&) override { }
     const vespalib::string &getName() const override { return _name; }
     const vespalib::string &getDocTypeName() const override { return _docTypeName; }
 };
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
index 3de1ac09c09..ed672cc9974 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
@@ -142,7 +142,13 @@ GidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener> list
             }
         }
         _listeners.emplace_back(std::move(listener));
-        _listeners.back()->notifyRegistered();
+        std::vector<GlobalId> removes;
+        for (auto& change : _pending_changes) {
+            if (change.is_remove()) {
+                removes.emplace_back(change.get_gid());
+            }
+        }
+        _listeners.back()->notifyRegistered(removes);
     } else {
         assert(_listeners.empty());
     }
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
index be03b57e088..6e1650083a2 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
@@ -46,13 +46,13 @@ GidToLidChangeListener::notifyRemove(IDestructorCallbackSP context, document::Gl
 }
 
 void
-GidToLidChangeListener::notifyRegistered()
+GidToLidChangeListener::notifyRegistered(const std::vector<document::GlobalId>& removes)
 {
     std::promise<void> promise;
     auto future = promise.get_future();
     _attributeFieldWriter.executeLambda(_executorId,
-                                        [this, &promise]() {
-                                            _attr->populateTargetLids();
+                                        [this, &promise, removes(std::move(removes))]() {
+                                            _attr->populateTargetLids(removes);
                                             promise.set_value();
                                         });
     future.wait();
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
index 28e9684ed86..f13a74af359 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
@@ -31,7 +31,7 @@ public:
     ~GidToLidChangeListener() override;
     void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) override;
     void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) override;
-    void notifyRegistered() override;
+    void notifyRegistered(const std::vector<document::GlobalId>& removes) override;
     const vespalib::string &getName() const override;
     const vespalib::string &getDocTypeName() const override;
     const std::shared_ptr<search::attribute::ReferenceAttribute> &getReferenceAttribute() const { return _attr; }
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
index 9bdeb2f2e23..75da3adc973 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
@@ -4,6 +4,7 @@
 
 #include <vespa/vespalib/util/idestructorcallback.h>
 #include <vespa/vespalib/stllike/string.h>
+#include <vector>
 
 namespace document { class GlobalId; }
 
@@ -20,7 +21,7 @@ public:
     virtual ~IGidToLidChangeListener() { }
     virtual void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) = 0;
     virtual void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) = 0;
-    virtual void notifyRegistered() = 0;
+    virtual void notifyRegistered(const std::vector<document::GlobalId>& removes) = 0;
     virtual const vespalib::string &getName() const = 0;
     virtual const vespalib::string &getDocTypeName() const = 0;
 };