Update document db reference resolver to register listeners for gid to lid

mapping changes for reference attributes.  Also add teardown method to
unregister existing listeners at shutdown time.

Extend document db referent registry with tryGet() method that is allowed to
fail. This is used to support unregistering, referenced document db might
no longer be present.

Extend unit tests.

3 files changed, 84 insertions, 40 deletions

diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
index ee7151f2075..d57f1de1d41 100644
--- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
+++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
@@ -9,12 +9,17 @@
 #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h>
 #include <vespa/searchcore/proton/reference/i_document_db_referent.h>
 #include <vespa/searchcore/proton/reference/i_document_db_referent_registry.h>
+#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h>
 #include <vespa/searchcore/proton/test/mock_document_db_referent.h>
+#include <vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h>
+#include <vespa/searchcore/proton/common/monitored_refcount.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/reference_attribute.h>
 #include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
 #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/test/mock_attribute_manager.h>
+#include <vespa/vespalib/test/insertion_operators.h>
 
 LOG_SETUP("document_db_reference_resolver_test");
 
@@ -34,13 +39,23 @@ struct MyGidToLidMapperFactory : public IGidToLidMapperFactory {
     }
 };
 
+using proton::test::MockGidToLidChangeHandler;
+using AddEntry = MockGidToLidChangeHandler::AddEntry;
+using RemoveEntry = MockGidToLidChangeHandler::RemoveEntry;
+
 struct MyDocumentDBReferent : public MockDocumentDBReferent {
     using SP = std::shared_ptr<MyDocumentDBReferent>;
     using AttributesMap = std::map<vespalib::string, AttributeVector::SP>;
     MyGidToLidMapperFactory::SP factory;
     AttributesMap attributes;
+    std::shared_ptr<MockGidToLidChangeHandler> _gidToLidChangeHandler;
 
-    MyDocumentDBReferent(MyGidToLidMapperFactory::SP factory_) : factory(factory_) {}
+    MyDocumentDBReferent(MyGidToLidMapperFactory::SP factory_,
+                         std::shared_ptr<MockGidToLidChangeHandler> gidToLidChangeHandler)
+        : factory(factory_),
+          _gidToLidChangeHandler(std::move(gidToLidChangeHandler))
+    {
+    }
     virtual IGidToLidMapperFactory::SP getGidToLidMapperFactory() override {
         return factory;
     }
@@ -52,6 +67,13 @@ struct MyDocumentDBReferent : public MockDocumentDBReferent {
     void addIntAttribute(vespalib::stringref name) {
         attributes[name] = AttributeFactory::createAttribute(name, Config(BasicType::INT32));
     }
+    virtual std::unique_ptr<GidToLidChangeRegistrator> makeGidToLidChangeRegistrator(const vespalib::string &docTypeName) override {
+        return std::make_unique<GidToLidChangeRegistrator>(_gidToLidChangeHandler, docTypeName);
+    }
+
+    MockGidToLidChangeHandler &getGidToLidChangeHandler() {
+        return *_gidToLidChangeHandler;
+    }
 };
 
 struct MyReferentRegistry : public IDocumentDBReferentRegistry {
@@ -62,6 +84,9 @@ struct MyReferentRegistry : public IDocumentDBReferentRegistry {
         ASSERT_TRUE(itr != map.end());
         return itr->second;
     }
+    virtual IDocumentDBReferent::SP tryGet(vespalib::stringref name) const override {
+        return get(name);
+    }
     virtual void add(vespalib::stringref name, IDocumentDBReferent::SP referent) override {
         map[name] = referent;
     }
@@ -131,20 +156,26 @@ asImportedAttribute(const IAttributeVector &attr)
 
 struct Fixture {
     MyGidToLidMapperFactory::SP factory;
+    std::shared_ptr<MockGidToLidChangeHandler> _parentGidToLidChangeHandler;
     MyDocumentDBReferent::SP parentReferent;
     MyReferentRegistry registry;
     MyAttributeManager attrMgr;
+    MyAttributeManager oldAttrMgr;
     DocumentModel docModel;
     ImportedFieldsConfig importedFieldsCfg;
-    DocumentDBReferenceResolver resolver;
+    MonitoredRefCount _gidToLidChangeListenerRefCount;
+    SequencedTaskExecutor _attributeFieldWriter;
     Fixture() :
         factory(std::make_shared<MyGidToLidMapperFactory>()),
-        parentReferent(std::make_shared<MyDocumentDBReferent>(factory)),
+        _parentGidToLidChangeHandler(std::make_shared<MockGidToLidChangeHandler>()),
+        parentReferent(std::make_shared<MyDocumentDBReferent>(factory, _parentGidToLidChangeHandler)),
         registry(),
         attrMgr(),
         docModel(),
         importedFieldsCfg(createImportedFieldsConfig()),
-        resolver(registry, docModel.childDocType, importedFieldsCfg)
+        _gidToLidChangeListenerRefCount(),
+        _attributeFieldWriter(1)
+
     {
         registry.add("parent", parentReferent);
         populateTargetAttributes();
@@ -160,7 +191,12 @@ struct Fixture {
         attrMgr.addIntAttribute("int_attr");
     }
     ImportedAttributesRepo::UP resolve() {
-        return resolver.resolve(attrMgr);
+        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
+        return resolver.resolve(attrMgr, oldAttrMgr);
+    }
+    void teardown() {
+        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
+        resolver.teardown(attrMgr);
     }
     const IGidToLidMapperFactory *getMapperFactoryPtr(const vespalib::string &attrName) {
         return attrMgr.getReferenceAttribute(attrName)->getGidToLidMapperFactory().get();
@@ -175,6 +211,24 @@ struct Fixture {
         EXPECT_EQUAL(attrMgr.getReferenceAttribute(referenceField), importedAttr.getReferenceAttribute().get());
         EXPECT_EQUAL(parentReferent->getAttribute(targetField).get(), importedAttr.getTargetAttribute().get());
     }
+
+    void assertParentAdds(const vespalib::string &referencedDocTypeName, const std::vector<AddEntry> &expAdds)
+    {
+        auto ireferent = registry.get(referencedDocTypeName);
+        auto referent = std::dynamic_pointer_cast<MyDocumentDBReferent>(ireferent);
+        assert(referent);
+        auto &handler = referent->getGidToLidChangeHandler();
+        handler.assertAdds(expAdds);
+    }
+
+    void assertParentRemoves(const vespalib::string &referencedDocTypeName, const std::vector<RemoveEntry> &expRemoves)
+    {
+        auto ireferent = registry.get(referencedDocTypeName);
+        auto referent = std::dynamic_pointer_cast<MyDocumentDBReferent>(ireferent);
+        assert(referent);
+        auto &handler = referent->getGidToLidChangeHandler();
+        handler.assertRemoves(expRemoves);
+    }
 };
 
 TEST_F("require that reference attributes are connected to gid mapper", Fixture)
@@ -192,6 +246,16 @@ TEST_F("require that imported attributes are instantiated", Fixture)
     f.assertImportedAttribute("imported_b", "other_ref", "target_b", repo->get("imported_b"));
 }
 
+TEST_F("require that listeners are added and removed", Fixture)
+{
+    f.resolve();
+    TEST_DO(f.assertParentAdds("parent", {{"child","other_ref"},{"child","ref"}}));
+    TEST_DO(f.assertParentRemoves("parent", {{"child", {"other_ref","ref"}}}));
+    f.teardown();
+    TEST_DO(f.assertParentAdds("parent", {{"child","other_ref"},{"child","ref"}}));
+    TEST_DO(f.assertParentRemoves("parent", {{"child", {"other_ref","ref"}},{"child", {}}}));
+}
+
 TEST_MAIN()
 {
     TEST_RUN_ALL();
diff --git a/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp b/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp
index b8f3675b20f..3a23de37a1f 100644
--- a/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp
+++ b/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp
@@ -78,6 +78,15 @@ TEST_F("Test that we block get until related add is completed", Fixture)
     getFooThread.join();
 }
 
+TEST_F("Test that tryGet method can fail", Fixture)
+{
+    auto referentFoo = f.add("foo");
+    auto referent = f._registry.tryGet("foo");
+    EXPECT_EQUAL(referentFoo.get(), referent.get());
+    referent = f._registry.tryGet("bar");
+    EXPECT_TRUE(referent.get() == nullptr);
+}
+
 }
 
 TEST_MAIN()
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
index a8ffc30c6c7..7fa7263d234 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h>
 #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h>
 #include <vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h>
+#include <vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h>
 #include <vespa/vespalib/test/insertion_operators.h>
 #include <map>
 #include <vespa/log/log.h>
@@ -29,48 +30,18 @@ public:
     virtual const vespalib::string &getDocTypeName() const override { return _docTypeName; }
 };
 
-using AddEntry = std::pair<vespalib::string, vespalib::string>;
-using RemoveEntry = std::pair<vespalib::string, std::set<vespalib::string>>;
 
-class MyHandler : public IGidToLidChangeHandler {
-    std::vector<AddEntry> _adds;
-    std::vector<RemoveEntry> _removes;
-public:
-    MyHandler()
-        : IGidToLidChangeHandler(),
-          _adds(),
-          _removes()
-    {
-    }
-
-    ~MyHandler() { }
-
-    virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override {
-        _adds.emplace_back(listener->getDocTypeName(), listener->getName());
-    }
-
-    virtual void removeListeners(const vespalib::string &docTypeName,
-                                 const std::set<vespalib::string> &keepNames) override {
-        _removes.emplace_back(docTypeName, keepNames);
-    }
-
-    void assertAdds(const std::vector<AddEntry> &expAdds)
-    {
-        EXPECT_EQUAL(expAdds, _adds);
-    }
 
-    void assertRemoves(const std::vector<RemoveEntry> &expRemoves)
-    {
-        EXPECT_EQUAL(expRemoves, _removes);
-    }
-};
+using test::MockGidToLidChangeHandler;
+using AddEntry = MockGidToLidChangeHandler::AddEntry;
+using RemoveEntry = MockGidToLidChangeHandler::RemoveEntry;
 
 struct Fixture
 {
-    std::shared_ptr<MyHandler> _handler;
+    std::shared_ptr<MockGidToLidChangeHandler> _handler;
 
     Fixture()
-        : _handler(std::make_shared<MyHandler>())
+        : _handler(std::make_shared<MockGidToLidChangeHandler>())
     {
     }