Add gid to lid change handler and gid to lid change registrator.

9 files changed, 425 insertions, 1 deletions

diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt
new file mode 100644
index 00000000000..eda9b7dde40
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_gid_to_lid_change_handler_test_app TEST
+    SOURCES
+    gid_to_lid_change_handler_test.cpp
+    DEPENDS
+    searchcore_reference
+    searchcore_server
+)
+vespa_add_test(NAME searchcore_gid_to_lid_change_handler_test_app COMMAND searchcore_gid_to_lid_change_handler_test_app)
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC
new file mode 100644
index 00000000000..362de49669b
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC
@@ -0,0 +1 @@
+gid_to_lid_change_handler test. Take a look at gid_to_lid_change_handler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES
new file mode 100644
index 00000000000..451157507fd
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES
@@ -0,0 +1 @@
+gid_to_lid_change_handler_test.cpp
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
new file mode 100644
index 00000000000..705197630a5
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
@@ -0,0 +1,207 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/searchcore/proton/server/executor_thread_service.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h>
+#include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h>
+#include <map>
+#include <vespa/log/log.h>
+LOG_SETUP("gid_to_lid_change_handler_test");
+
+using document::GlobalId;
+using document::DocumentId;
+using search::makeLambdaTask;
+
+namespace proton {
+
+namespace {
+
+GlobalId toGid(vespalib::stringref docId) {
+    return DocumentId(docId).getGlobalId();
+}
+
+vespalib::string doc1("id:test:music::1");
+
+}
+
+class MyTarget {
+    using lock_guard = std::lock_guard<std::mutex>;
+    std::mutex _lock;
+    std::vector<std::pair<GlobalId, uint32_t>> _changes;
+    uint32_t  _createdListeners;
+    uint32_t  _registeredListeners;
+    uint32_t  _destroyedListeners;
+
+public:
+    MyTarget()
+        : _lock(),
+          _changes(),
+          _createdListeners(0u),
+          _registeredListeners(0u),
+          _destroyedListeners(0u)
+    {
+    }
+
+    ~MyTarget()
+    {
+        EXPECT_EQUAL(_createdListeners, _destroyedListeners);
+    }
+
+    void notifyGidToLidChange(GlobalId gid, uint32_t lid) {
+        lock_guard guard(_lock);
+        _changes.emplace_back(gid, lid);
+    }
+    void markCreatedListener()    { lock_guard guard(_lock); ++_createdListeners; }
+    void markRegisteredListener() { lock_guard guard(_lock); ++_registeredListeners; }
+    void markDestroyedListener()  { lock_guard guard(_lock); ++_destroyedListeners; }
+
+    uint32_t getCreatedListeners() const { return _createdListeners; }
+    uint32_t getRegisteredListeners() const { return _registeredListeners; }
+    uint32_t getDestroyedListeners() const { return _destroyedListeners; }
+    const std::vector<std::pair<GlobalId, uint32_t>> &getChanges() const { return _changes; }
+
+    void assertCounts(uint32_t expCreatedListeners,
+                      uint32_t expRegisteredListeners,
+                      uint32_t expDestroyedListeners,
+                      uint32_t expChanges)
+    {
+        EXPECT_EQUAL(expCreatedListeners, getCreatedListeners());
+        EXPECT_EQUAL(expRegisteredListeners, getRegisteredListeners());
+        EXPECT_EQUAL(expDestroyedListeners, getDestroyedListeners());
+        EXPECT_EQUAL(expChanges, _changes.size());
+    }
+};
+
+class MyListener : public IGidToLidChangeListener
+{
+    MyTarget         &_target;
+    vespalib::string  _name;
+    vespalib::string  _docTypeName;
+public:
+    MyListener(MyTarget &target,
+               const vespalib::string &name,
+               const vespalib::string &docTypeName)
+        : IGidToLidChangeListener(),
+          _target(target),
+          _name(name),
+          _docTypeName(docTypeName)
+    {
+        _target.markCreatedListener();
+    }
+    virtual ~MyListener() { _target.markDestroyedListener(); }
+    virtual void notifyGidToLidChange(GlobalId gid, uint32_t lid) override { _target.notifyGidToLidChange(gid, lid); }
+    virtual void notifyRegistered() override { _target.markRegisteredListener(); }
+    virtual const vespalib::string &getName() const override { return _name; }
+    virtual const vespalib::string &getDocTypeName() const override { return _docTypeName; }
+};
+
+struct Fixture
+{
+    vespalib::ThreadStackExecutor _masterExecutor;
+    ExecutorThreadService _master;
+    std::vector<std::shared_ptr<MyTarget>> _targets;
+    std::shared_ptr<GidToLidChangeHandler> _handler;
+
+    Fixture()
+        : _masterExecutor(1, 128 * 1024),
+          _master(_masterExecutor),
+          _targets(),
+          _handler(std::make_shared<GidToLidChangeHandler>(&_master))
+    {
+    }
+
+    ~Fixture()
+    {
+        close();
+    }
+
+    void close()
+    {
+        _master.execute(makeLambdaTask([this]() { _handler->close(); }));
+        _master.sync();
+    }
+
+    MyTarget &addTarget() {
+        _targets.push_back(std::make_shared<MyTarget>());
+        return *_targets.back();
+    }
+
+    void addListener(std::unique_ptr<IGidToLidChangeListener> listener) {
+        _handler->addListener(std::move(listener));
+        _master.sync();
+    }
+
+    void notifyGidToLidChange(GlobalId gid, uint32_t lid) {
+        _master.execute(makeLambdaTask([this, gid, lid]() { _handler->notifyGidToLidChange(gid, lid); }));
+        _master.sync();
+    }
+
+    void removeListeners(const vespalib::string &docTypeName,
+                         const std::set<vespalib::string> &keepNames) {
+        _handler->removeListeners(docTypeName, keepNames);
+        _master.sync();
+    }
+
+};
+
+TEST_F("Test that we can register a listener", Fixture)
+{
+    auto &target = f.addTarget();
+    auto listener = std::make_unique<MyListener>(target, "test", "testdoc");
+    TEST_DO(target.assertCounts(1, 0, 0, 0));
+    f.addListener(std::move(listener));
+    TEST_DO(target.assertCounts(1, 1, 0, 0));
+    f.notifyGidToLidChange(toGid(doc1), 10);
+    TEST_DO(target.assertCounts(1, 1, 0, 1));
+    f.removeListeners("testdoc", {});
+    TEST_DO(target.assertCounts(1, 1, 1, 1));
+}
+
+TEST_F("Test that we can register multiple listeners", Fixture)
+{
+    auto &target1 = f.addTarget();
+    auto &target2 = f.addTarget();
+    auto &target3 = f.addTarget();
+    auto listener1 = std::make_unique<MyListener>(target1, "test1", "testdoc");
+    auto listener2 = std::make_unique<MyListener>(target2, "test2", "testdoc");
+    auto listener3 = std::make_unique<MyListener>(target3, "test3", "testdoc2");
+    TEST_DO(target1.assertCounts(1, 0, 0, 0));
+    TEST_DO(target2.assertCounts(1, 0, 0, 0));
+    TEST_DO(target3.assertCounts(1, 0, 0, 0));
+    f.addListener(std::move(listener1));
+    f.addListener(std::move(listener2));
+    f.addListener(std::move(listener3));
+    TEST_DO(target1.assertCounts(1, 1, 0, 0));
+    TEST_DO(target2.assertCounts(1, 1, 0, 0));
+    TEST_DO(target3.assertCounts(1, 1, 0, 0));
+    f.notifyGidToLidChange(toGid(doc1), 10);
+    TEST_DO(target1.assertCounts(1, 1, 0, 1));
+    TEST_DO(target2.assertCounts(1, 1, 0, 1));
+    TEST_DO(target3.assertCounts(1, 1, 0, 1));
+    f.removeListeners("testdoc", {"test1"});
+    TEST_DO(target1.assertCounts(1, 1, 0, 1));
+    TEST_DO(target2.assertCounts(1, 1, 1, 1));
+    TEST_DO(target3.assertCounts(1, 1, 0, 1));
+    f.removeListeners("testdoc", {});
+    TEST_DO(target1.assertCounts(1, 1, 1, 1));
+    TEST_DO(target2.assertCounts(1, 1, 1, 1));
+    TEST_DO(target3.assertCounts(1, 1, 0, 1));
+    f.removeListeners("testdoc2", {"test3"});
+    TEST_DO(target1.assertCounts(1, 1, 1, 1));
+    TEST_DO(target2.assertCounts(1, 1, 1, 1));
+    TEST_DO(target3.assertCounts(1, 1, 0, 1));
+    f.removeListeners("testdoc2", {"foo"});
+    TEST_DO(target1.assertCounts(1, 1, 1, 1));
+    TEST_DO(target2.assertCounts(1, 1, 1, 1));
+    TEST_DO(target3.assertCounts(1, 1, 1, 1));
+}
+
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
index fb762bd0d44..418b8079d88 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
@@ -5,6 +5,9 @@
 #include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchcore/proton/common/monitored_refcount.h>
 #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h>
+#include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h>
+#include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
+#include <map>
 #include <vespa/log/log.h>
 LOG_SETUP("gid_to_lid_change_listener_test");
 
@@ -28,6 +31,41 @@ vespalib::string doc1("id:test:music::1");
 vespalib::string doc2("id:test:music::2");
 vespalib::string doc3("id:test:music::3");
 
+using MockGidToLidMap = std::map<GlobalId, uint32_t>;
+
+struct MyGidToLidMapper : public search::IGidToLidMapper
+{
+    const MockGidToLidMap &_map;
+    MyGidToLidMapper(const MockGidToLidMap &map)
+        : _map(map)
+    {
+    }
+    virtual uint32_t mapGidToLid(const document::GlobalId &gid) const override {
+        auto itr = _map.find(gid);
+        if (itr != _map.end()) {
+            return itr->second;
+        } else {
+            return 0u;
+        }
+    }
+};
+
+struct MyGidToLidMapperFactory : public search::IGidToLidMapperFactory
+{
+    MockGidToLidMap _map;
+
+    MyGidToLidMapperFactory()
+        : _map()
+    {
+        _map.insert({toGid(doc1), 10});
+        _map.insert({toGid(doc2), 17});
+    }
+
+    virtual std::unique_ptr<search::IGidToLidMapper> getMapper() const {
+        return std::make_unique<MyGidToLidMapper>(_map);
+    }
+};
+
 }
 
 struct Fixture
@@ -69,13 +107,22 @@ struct Fixture
         EXPECT_EQUAL(expLid, ref->lid());
     }
 
+    void assertNoRefLid(uint32_t doc) {
+        auto ref = getRef(doc);
+        EXPECT_TRUE(ref == nullptr);
+    }
+
     void allocListener() {
-        _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount);
+        _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount, "test", "testdoc");
     }
 
     void notifyGidToLidChange(const GlobalId &gid, uint32_t referencedDoc) {
         _listener->notifyGidToLidChange(gid, referencedDoc);
     }
+
+    void notifyListenerRegistered() {
+        _listener->notifyRegistered();
+    }
 };
 
 TEST_F("Test that we can use gid to lid change listener", Fixture)
@@ -97,6 +144,31 @@ TEST_F("Test that we can use gid to lid change listener", Fixture)
     TEST_DO(f.assertRefLid(10, 3));
 }
 
+TEST_F("Test that notifyRegistered method in gid to lid change listener works", Fixture)
+{
+    f.ensureDocIdLimit(6);
+    f.set(1, toGid(doc1));
+    f.set(2, toGid(doc2));
+    f.set(3, toGid(doc1));
+    f.set(4, toGid(doc3));
+    f.commit();
+    TEST_DO(f.assertRefLid(0, 1));
+    TEST_DO(f.assertRefLid(0, 2));
+    TEST_DO(f.assertRefLid(0, 3));
+    TEST_DO(f.assertRefLid(0, 4));
+    TEST_DO(f.assertNoRefLid(5));
+    std::shared_ptr<search::IGidToLidMapperFactory> factory =
+        std::make_shared<MyGidToLidMapperFactory>();
+    f._attr->setGidToLidMapperFactory(factory);
+    f.allocListener();
+    f.notifyListenerRegistered();
+    TEST_DO(f.assertRefLid(10, 1));
+    TEST_DO(f.assertRefLid(17, 2));
+    TEST_DO(f.assertRefLid(10, 3));
+    TEST_DO(f.assertRefLid(0, 4));
+    TEST_DO(f.assertNoRefLid(5));
+}
+
 }
 
 TEST_MAIN()
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt
new file mode 100644
index 00000000000..0d80606c22f
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_gid_to_lid_change_registrator_test_app TEST
+    SOURCES
+    gid_to_lid_change_registrator_test.cpp
+    DEPENDS
+    searchcore_reference
+    searchcore_server
+)
+vespa_add_test(NAME searchcore_gid_to_lid_change_registrator_test_app COMMAND searchcore_gid_to_lid_change_registrator_test_app)
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC
new file mode 100644
index 00000000000..a5f481dd092
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC
@@ -0,0 +1 @@
+gid_to_lid_change_registrator test. Take a look at gid_to_lid_change_registrator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES
new file mode 100644
index 00000000000..60e12f3f836
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES
@@ -0,0 +1 @@
+gid_to_lid_change_registrator_test.cpp
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
new file mode 100644
index 00000000000..072d618d359
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
@@ -0,0 +1,123 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h>
+#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h>
+#include <vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <map>
+#include <vespa/log/log.h>
+LOG_SETUP("gid_to_lid_change_registrator_test");
+
+namespace proton {
+
+class MyListener : public IGidToLidChangeListener
+{
+    vespalib::string _docTypeName;
+    vespalib::string _name;
+public:
+    MyListener(const vespalib::string &docTypeName, const vespalib::string &name)
+        : _docTypeName(docTypeName),
+          _name(name)
+    {
+    }
+    virtual ~MyListener() { }
+    virtual void notifyGidToLidChange(document::GlobalId, uint32_t) override { }
+    virtual void notifyRegistered() override { }
+    virtual const vespalib::string &getName() const override { return _name; }
+    virtual const vespalib::string &getDocTypeName() const override { return _docTypeName; }
+};
+
+using AddEntry = std::pair<vespalib::string, vespalib::string>;
+using RemoveEntry = std::pair<vespalib::string, std::set<vespalib::string>>;
+
+class MyHandler : public IGidToLidChangeHandler {
+    std::vector<AddEntry> _adds;
+    std::vector<RemoveEntry> _removes;
+public:
+    MyHandler()
+        : IGidToLidChangeHandler(),
+          _adds(),
+          _removes()
+    {
+    }
+
+    ~MyHandler() { }
+
+    virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override {
+        _adds.emplace_back(listener->getDocTypeName(), listener->getName());
+    }
+
+    virtual void removeListeners(const vespalib::string &docTypeName,
+                                 const std::set<vespalib::string> &keepNames) override {
+        _removes.emplace_back(docTypeName, keepNames);
+    }
+
+    void assertAdds(const std::vector<AddEntry> &expAdds)
+    {
+        EXPECT_EQUAL(expAdds, _adds);
+    }
+
+    void assertRemoves(const std::vector<RemoveEntry> &expRemoves)
+    {
+        EXPECT_EQUAL(expRemoves, _removes);
+    }
+};
+
+struct Fixture
+{
+    std::shared_ptr<MyHandler> _handler;
+
+    Fixture()
+        : _handler(std::make_shared<MyHandler>())
+    {
+    }
+
+    ~Fixture()
+    {
+    }
+
+    std::unique_ptr<GidToLidChangeRegistrator>
+    getRegistrator(const vespalib::string &docTypeName) {
+        return std::make_unique<GidToLidChangeRegistrator>(_handler, docTypeName);
+    }
+
+    void assertAdds(const std::vector<AddEntry> &expAdds) {
+        _handler->assertAdds(expAdds);
+    }
+
+    void assertRemoves(const std::vector<RemoveEntry> &expRemoves) {
+        _handler->assertRemoves(expRemoves);
+    }
+};
+
+TEST_F("Test that we can register a listener", Fixture)
+{
+    auto registrator = f.getRegistrator("testdoc");
+    TEST_DO(f.assertAdds({}));
+    TEST_DO(f.assertRemoves({}));
+    registrator->addListener(std::make_unique<MyListener>("testdoc", "f1"));
+    registrator.reset();
+    TEST_DO(f.assertAdds({{"testdoc","f1"}}));
+    TEST_DO(f.assertRemoves({{"testdoc",{"f1"}}}));
+}
+
+TEST_F("Test that we can register multiple listeners", Fixture)
+{
+    auto registrator = f.getRegistrator("testdoc");
+    TEST_DO(f.assertAdds({}));
+    TEST_DO(f.assertRemoves({}));
+    registrator->addListener(std::make_unique<MyListener>("testdoc", "f1"));
+    registrator->addListener(std::make_unique<MyListener>("testdoc", "f2"));
+    registrator.reset();
+    TEST_DO(f.assertAdds({{"testdoc","f1"},{"testdoc","f2"}}));
+    TEST_DO(f.assertRemoves({{"testdoc",{"f1","f2"}}}));
+}
+
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}