Add a vector of gids to remove to ReferenceAttribute::populateTargetLids()

signature. These gids represent the target documents removed since the last
feed view force commit where the live gid to lid mapping in the document
meta store was made available as a frozen tree.

3 files changed, 38 insertions, 9 deletions

diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
index bafcd545e33..c077ab83a6e 100644
--- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
+++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
@@ -200,9 +200,9 @@ struct ReferenceAttributeTest : public ::testing::Test {
     void notifyReferencedRemove(const GlobalId &gid) {
         _attr->notifyReferencedRemove(gid);
     }
-    void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory) {
+    void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory, const std::vector<GlobalId>& removes) {
         _attr->setGidToLidMapperFactory(factory);
-        _attr->populateTargetLids();
+        _attr->populateTargetLids(removes);
     }
     uint32_t getUniqueGids() {
         return getStatus().getNumUniqueValues();
@@ -257,7 +257,7 @@ TEST_F(ReferenceAttributeTest, reference_for_a_document_can_be_cleared)
 TEST_F(ReferenceAttributeTest, lid_beyond_range_is_mapped_to_zero)
 {
     auto factory = std::make_shared<MyGidToLidMapperFactory>();
-    setGidToLidMapperFactory(factory);
+    setGidToLidMapperFactory(factory, {});
     ensureDocIdLimit(5);
     _attr->addDocs(1);
     set(5, toGid(doc2));
@@ -318,7 +318,7 @@ TEST_F(ReferenceAttributeTest, update_uses_gid_mapper_to_set_target_lid)
 {
     ensureDocIdLimit(6);
     auto factory = std::make_shared<MyGidToLidMapperFactory>();
-    setGidToLidMapperFactory(factory);
+    setGidToLidMapperFactory(factory, {});
     set(1, toGid(doc1));
     set(2, toGid(doc2));
     set(4, toGid(doc1));
@@ -371,7 +371,7 @@ void
 checkPopulateTargetLids(ReferenceAttributeTest &f)
 {
     auto factory = std::make_shared<MyGidToLidMapperFactory>();
-    f.setGidToLidMapperFactory(factory);
+    f.setGidToLidMapperFactory(factory, {});
     f.assertTargetLid(1, 10);
     f.assertTargetLid(2, 17);
     f.assertTargetLid(3, 10);
@@ -401,6 +401,22 @@ TEST_F(ReferenceAttributeTest, populateTargetLids_uses_gid_mapper_to_update_lid_
     EXPECT_TRUE(vespalib::unlink("test.udat"));
 }
 
+TEST_F(ReferenceAttributeTest, populateTargetLids_handles_removes)
+{
+    preparePopulateTargetLids(*this);
+    auto factory = std::make_shared<MyGidToLidMapperFactory>();
+    setGidToLidMapperFactory(factory, { toGid(doc1) });
+    assertTargetLid(1, 0);
+    assertTargetLid(2, 17);
+    assertTargetLid(3, 0);
+    assertTargetLid(4, 0);
+    assertNoTargetLid(5);
+    assertLids(0, { });
+    assertLids(10, { });
+    assertLids(17, { 2 });
+    assertLids(18, { });
+}
+
 TEST_F(ReferenceAttributeTest, notifyReferencedPut_and_notifyReferencedRemove_changes_reverse_mapping)
 {
     preparePopulateTargetLids(*this);
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
index 342c1ea568b..eb822313d61 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
@@ -366,8 +366,8 @@ ReferenceAttribute::notifyReferencedPut(const GlobalId &gid, DocId targetLid)
     commit();
 }
 
-void
-ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
+bool
+ReferenceAttribute::notifyReferencedRemoveNoCommit(const GlobalId &gid)
 {
     EntryRef ref = _store.find(gid);
     if (ref.valid()) {
@@ -377,6 +377,15 @@ ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
         if (oldTargetLid != 0) {
             _store.remove(ref);
         }
+        return true;
+    }
+    return false;
+}
+
+void
+ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
+{
+    if (notifyReferencedRemoveNoCommit(gid)) {
         commit();
     }
 }
@@ -401,7 +410,7 @@ public:
 }
 
 void
-ReferenceAttribute::populateTargetLids()
+ReferenceAttribute::populateTargetLids(const std::vector<GlobalId>& removes)
 {
     if (_gidToLidMapperFactory) {
         std::unique_ptr<IGidToLidMapper> mapperUP = _gidToLidMapperFactory->getMapper();
@@ -409,6 +418,9 @@ ReferenceAttribute::populateTargetLids()
         TargetLidPopulator populator(*this);
         mapper.foreach(populator);
     }
+    for (auto& remove : removes) {
+        notifyReferencedRemoveNoCommit(remove);
+    }
     commit();
 }
 
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
index b411d6cb923..4016230ef89 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
@@ -84,8 +84,9 @@ public:
 
     void notifyReferencedPutNoCommit(const GlobalId &gid, DocId targetLid);
     void notifyReferencedPut(const GlobalId &gid, DocId targetLid);
+    bool notifyReferencedRemoveNoCommit(const GlobalId &gid);
     void notifyReferencedRemove(const GlobalId &gid);
-    void populateTargetLids();
+    void populateTargetLids(const std::vector<GlobalId>& removes);
     void clearDocs(DocId lidLow, DocId lidLimit) override;
     void onShrinkLidSpace() override;