diff options
23 files changed, 806 insertions, 7 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 882062baa64..f66ec61319d 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -117,7 +117,9 @@ vespa_define_module( src/tests/proton/persistenceconformance src/tests/proton/persistenceengine src/tests/proton/proton + src/tests/proton/reference/gid_to_lid_change_handler src/tests/proton/reference/gid_to_lid_change_listener + src/tests/proton/reference/gid_to_lid_change_registrator src/tests/proton/reference/gid_to_lid_mapper src/tests/proton/reference/document_db_reference_resolver src/tests/proton/reference/document_db_referent_registry diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt new file mode 100644 index 00000000000..eda9b7dde40 --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_gid_to_lid_change_handler_test_app TEST + SOURCES + gid_to_lid_change_handler_test.cpp + DEPENDS + searchcore_reference + searchcore_server +) +vespa_add_test(NAME searchcore_gid_to_lid_change_handler_test_app COMMAND searchcore_gid_to_lid_change_handler_test_app) diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC new file mode 100644 index 00000000000..362de49669b --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/DESC @@ -0,0 +1 @@ +gid_to_lid_change_handler test. Take a look at gid_to_lid_change_handler_test.cpp for details. diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES new file mode 100644 index 00000000000..451157507fd --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/FILES @@ -0,0 +1 @@ +gid_to_lid_change_handler_test.cpp diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp new file mode 100644 index 00000000000..705197630a5 --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -0,0 +1,207 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/document/base/documentid.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/searchcore/proton/server/executor_thread_service.h> +#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h> +#include <map> +#include <vespa/log/log.h> +LOG_SETUP("gid_to_lid_change_handler_test"); + +using document::GlobalId; +using document::DocumentId; +using search::makeLambdaTask; + +namespace proton { + +namespace { + +GlobalId toGid(vespalib::stringref docId) { + return DocumentId(docId).getGlobalId(); +} + +vespalib::string doc1("id:test:music::1"); + +} + +class MyTarget { + using lock_guard = std::lock_guard<std::mutex>; + std::mutex _lock; + std::vector<std::pair<GlobalId, uint32_t>> _changes; + uint32_t _createdListeners; + uint32_t _registeredListeners; + uint32_t _destroyedListeners; + +public: + MyTarget() + : _lock(), + _changes(), + _createdListeners(0u), + _registeredListeners(0u), + _destroyedListeners(0u) + { + } + + ~MyTarget() + { + EXPECT_EQUAL(_createdListeners, _destroyedListeners); + } + + void notifyGidToLidChange(GlobalId gid, uint32_t lid) { + lock_guard guard(_lock); + _changes.emplace_back(gid, lid); + } + void markCreatedListener() { lock_guard guard(_lock); ++_createdListeners; } + void markRegisteredListener() { lock_guard guard(_lock); ++_registeredListeners; } + void markDestroyedListener() { lock_guard guard(_lock); ++_destroyedListeners; } + + uint32_t getCreatedListeners() const { return _createdListeners; } + uint32_t getRegisteredListeners() const { return _registeredListeners; } + uint32_t getDestroyedListeners() const { return _destroyedListeners; } + const std::vector<std::pair<GlobalId, uint32_t>> &getChanges() const { return _changes; } + + void assertCounts(uint32_t expCreatedListeners, + uint32_t expRegisteredListeners, + uint32_t expDestroyedListeners, + uint32_t expChanges) + { + EXPECT_EQUAL(expCreatedListeners, getCreatedListeners()); + EXPECT_EQUAL(expRegisteredListeners, getRegisteredListeners()); + EXPECT_EQUAL(expDestroyedListeners, getDestroyedListeners()); + EXPECT_EQUAL(expChanges, _changes.size()); + } +}; + +class MyListener : public IGidToLidChangeListener +{ + MyTarget &_target; + vespalib::string _name; + vespalib::string _docTypeName; +public: + MyListener(MyTarget &target, + const vespalib::string &name, + const vespalib::string &docTypeName) + : IGidToLidChangeListener(), + _target(target), + _name(name), + _docTypeName(docTypeName) + { + _target.markCreatedListener(); + } + virtual ~MyListener() { _target.markDestroyedListener(); } + virtual void notifyGidToLidChange(GlobalId gid, uint32_t lid) override { _target.notifyGidToLidChange(gid, lid); } + virtual void notifyRegistered() override { _target.markRegisteredListener(); } + virtual const vespalib::string &getName() const override { return _name; } + virtual const vespalib::string &getDocTypeName() const override { return _docTypeName; } +}; + +struct Fixture +{ + vespalib::ThreadStackExecutor _masterExecutor; + ExecutorThreadService _master; + std::vector<std::shared_ptr<MyTarget>> _targets; + std::shared_ptr<GidToLidChangeHandler> _handler; + + Fixture() + : _masterExecutor(1, 128 * 1024), + _master(_masterExecutor), + _targets(), + _handler(std::make_shared<GidToLidChangeHandler>(&_master)) + { + } + + ~Fixture() + { + close(); + } + + void close() + { + _master.execute(makeLambdaTask([this]() { _handler->close(); })); + _master.sync(); + } + + MyTarget &addTarget() { + _targets.push_back(std::make_shared<MyTarget>()); + return *_targets.back(); + } + + void addListener(std::unique_ptr<IGidToLidChangeListener> listener) { + _handler->addListener(std::move(listener)); + _master.sync(); + } + + void notifyGidToLidChange(GlobalId gid, uint32_t lid) { + _master.execute(makeLambdaTask([this, gid, lid]() { _handler->notifyGidToLidChange(gid, lid); })); + _master.sync(); + } + + void removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) { + _handler->removeListeners(docTypeName, keepNames); + _master.sync(); + } + +}; + +TEST_F("Test that we can register a listener", Fixture) +{ + auto &target = f.addTarget(); + auto listener = std::make_unique<MyListener>(target, "test", "testdoc"); + TEST_DO(target.assertCounts(1, 0, 0, 0)); + f.addListener(std::move(listener)); + TEST_DO(target.assertCounts(1, 1, 0, 0)); + f.notifyGidToLidChange(toGid(doc1), 10); + TEST_DO(target.assertCounts(1, 1, 0, 1)); + f.removeListeners("testdoc", {}); + TEST_DO(target.assertCounts(1, 1, 1, 1)); +} + +TEST_F("Test that we can register multiple listeners", Fixture) +{ + auto &target1 = f.addTarget(); + auto &target2 = f.addTarget(); + auto &target3 = f.addTarget(); + auto listener1 = std::make_unique<MyListener>(target1, "test1", "testdoc"); + auto listener2 = std::make_unique<MyListener>(target2, "test2", "testdoc"); + auto listener3 = std::make_unique<MyListener>(target3, "test3", "testdoc2"); + TEST_DO(target1.assertCounts(1, 0, 0, 0)); + TEST_DO(target2.assertCounts(1, 0, 0, 0)); + TEST_DO(target3.assertCounts(1, 0, 0, 0)); + f.addListener(std::move(listener1)); + f.addListener(std::move(listener2)); + f.addListener(std::move(listener3)); + TEST_DO(target1.assertCounts(1, 1, 0, 0)); + TEST_DO(target2.assertCounts(1, 1, 0, 0)); + TEST_DO(target3.assertCounts(1, 1, 0, 0)); + f.notifyGidToLidChange(toGid(doc1), 10); + TEST_DO(target1.assertCounts(1, 1, 0, 1)); + TEST_DO(target2.assertCounts(1, 1, 0, 1)); + TEST_DO(target3.assertCounts(1, 1, 0, 1)); + f.removeListeners("testdoc", {"test1"}); + TEST_DO(target1.assertCounts(1, 1, 0, 1)); + TEST_DO(target2.assertCounts(1, 1, 1, 1)); + TEST_DO(target3.assertCounts(1, 1, 0, 1)); + f.removeListeners("testdoc", {}); + TEST_DO(target1.assertCounts(1, 1, 1, 1)); + TEST_DO(target2.assertCounts(1, 1, 1, 1)); + TEST_DO(target3.assertCounts(1, 1, 0, 1)); + f.removeListeners("testdoc2", {"test3"}); + TEST_DO(target1.assertCounts(1, 1, 1, 1)); + TEST_DO(target2.assertCounts(1, 1, 1, 1)); + TEST_DO(target3.assertCounts(1, 1, 0, 1)); + f.removeListeners("testdoc2", {"foo"}); + TEST_DO(target1.assertCounts(1, 1, 1, 1)); + TEST_DO(target2.assertCounts(1, 1, 1, 1)); + TEST_DO(target3.assertCounts(1, 1, 1, 1)); +} + +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index fb762bd0d44..418b8079d88 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -5,6 +5,9 @@ #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> +#include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> +#include <vespa/searchlib/common/i_gid_to_lid_mapper.h> +#include <map> #include <vespa/log/log.h> LOG_SETUP("gid_to_lid_change_listener_test"); @@ -28,6 +31,41 @@ vespalib::string doc1("id:test:music::1"); vespalib::string doc2("id:test:music::2"); vespalib::string doc3("id:test:music::3"); +using MockGidToLidMap = std::map<GlobalId, uint32_t>; + +struct MyGidToLidMapper : public search::IGidToLidMapper +{ + const MockGidToLidMap &_map; + MyGidToLidMapper(const MockGidToLidMap &map) + : _map(map) + { + } + virtual uint32_t mapGidToLid(const document::GlobalId &gid) const override { + auto itr = _map.find(gid); + if (itr != _map.end()) { + return itr->second; + } else { + return 0u; + } + } +}; + +struct MyGidToLidMapperFactory : public search::IGidToLidMapperFactory +{ + MockGidToLidMap _map; + + MyGidToLidMapperFactory() + : _map() + { + _map.insert({toGid(doc1), 10}); + _map.insert({toGid(doc2), 17}); + } + + virtual std::unique_ptr<search::IGidToLidMapper> getMapper() const { + return std::make_unique<MyGidToLidMapper>(_map); + } +}; + } struct Fixture @@ -69,13 +107,22 @@ struct Fixture EXPECT_EQUAL(expLid, ref->lid()); } + void assertNoRefLid(uint32_t doc) { + auto ref = getRef(doc); + EXPECT_TRUE(ref == nullptr); + } + void allocListener() { - _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount); + _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount, "test", "testdoc"); } void notifyGidToLidChange(const GlobalId &gid, uint32_t referencedDoc) { _listener->notifyGidToLidChange(gid, referencedDoc); } + + void notifyListenerRegistered() { + _listener->notifyRegistered(); + } }; TEST_F("Test that we can use gid to lid change listener", Fixture) @@ -97,6 +144,31 @@ TEST_F("Test that we can use gid to lid change listener", Fixture) TEST_DO(f.assertRefLid(10, 3)); } +TEST_F("Test that notifyRegistered method in gid to lid change listener works", Fixture) +{ + f.ensureDocIdLimit(6); + f.set(1, toGid(doc1)); + f.set(2, toGid(doc2)); + f.set(3, toGid(doc1)); + f.set(4, toGid(doc3)); + f.commit(); + TEST_DO(f.assertRefLid(0, 1)); + TEST_DO(f.assertRefLid(0, 2)); + TEST_DO(f.assertRefLid(0, 3)); + TEST_DO(f.assertRefLid(0, 4)); + TEST_DO(f.assertNoRefLid(5)); + std::shared_ptr<search::IGidToLidMapperFactory> factory = + std::make_shared<MyGidToLidMapperFactory>(); + f._attr->setGidToLidMapperFactory(factory); + f.allocListener(); + f.notifyListenerRegistered(); + TEST_DO(f.assertRefLid(10, 1)); + TEST_DO(f.assertRefLid(17, 2)); + TEST_DO(f.assertRefLid(10, 3)); + TEST_DO(f.assertRefLid(0, 4)); + TEST_DO(f.assertNoRefLid(5)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt new file mode 100644 index 00000000000..0d80606c22f --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_gid_to_lid_change_registrator_test_app TEST + SOURCES + gid_to_lid_change_registrator_test.cpp + DEPENDS + searchcore_reference + searchcore_server +) +vespa_add_test(NAME searchcore_gid_to_lid_change_registrator_test_app COMMAND searchcore_gid_to_lid_change_registrator_test_app) diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC new file mode 100644 index 00000000000..a5f481dd092 --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/DESC @@ -0,0 +1 @@ +gid_to_lid_change_registrator test. Take a look at gid_to_lid_change_registrator_test.cpp for details. diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES new file mode 100644 index 00000000000..60e12f3f836 --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/FILES @@ -0,0 +1 @@ +gid_to_lid_change_registrator_test.cpp diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp new file mode 100644 index 00000000000..072d618d359 --- /dev/null +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp @@ -0,0 +1,123 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/document/base/globalid.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <map> +#include <vespa/log/log.h> +LOG_SETUP("gid_to_lid_change_registrator_test"); + +namespace proton { + +class MyListener : public IGidToLidChangeListener +{ + vespalib::string _docTypeName; + vespalib::string _name; +public: + MyListener(const vespalib::string &docTypeName, const vespalib::string &name) + : _docTypeName(docTypeName), + _name(name) + { + } + virtual ~MyListener() { } + virtual void notifyGidToLidChange(document::GlobalId, uint32_t) override { } + virtual void notifyRegistered() override { } + virtual const vespalib::string &getName() const override { return _name; } + virtual const vespalib::string &getDocTypeName() const override { return _docTypeName; } +}; + +using AddEntry = std::pair<vespalib::string, vespalib::string>; +using RemoveEntry = std::pair<vespalib::string, std::set<vespalib::string>>; + +class MyHandler : public IGidToLidChangeHandler { + std::vector<AddEntry> _adds; + std::vector<RemoveEntry> _removes; +public: + MyHandler() + : IGidToLidChangeHandler(), + _adds(), + _removes() + { + } + + ~MyHandler() { } + + virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override { + _adds.emplace_back(listener->getDocTypeName(), listener->getName()); + } + + virtual void removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) override { + _removes.emplace_back(docTypeName, keepNames); + } + + void assertAdds(const std::vector<AddEntry> &expAdds) + { + EXPECT_EQUAL(expAdds, _adds); + } + + void assertRemoves(const std::vector<RemoveEntry> &expRemoves) + { + EXPECT_EQUAL(expRemoves, _removes); + } +}; + +struct Fixture +{ + std::shared_ptr<MyHandler> _handler; + + Fixture() + : _handler(std::make_shared<MyHandler>()) + { + } + + ~Fixture() + { + } + + std::unique_ptr<GidToLidChangeRegistrator> + getRegistrator(const vespalib::string &docTypeName) { + return std::make_unique<GidToLidChangeRegistrator>(_handler, docTypeName); + } + + void assertAdds(const std::vector<AddEntry> &expAdds) { + _handler->assertAdds(expAdds); + } + + void assertRemoves(const std::vector<RemoveEntry> &expRemoves) { + _handler->assertRemoves(expRemoves); + } +}; + +TEST_F("Test that we can register a listener", Fixture) +{ + auto registrator = f.getRegistrator("testdoc"); + TEST_DO(f.assertAdds({})); + TEST_DO(f.assertRemoves({})); + registrator->addListener(std::make_unique<MyListener>("testdoc", "f1")); + registrator.reset(); + TEST_DO(f.assertAdds({{"testdoc","f1"}})); + TEST_DO(f.assertRemoves({{"testdoc",{"f1"}}})); +} + +TEST_F("Test that we can register multiple listeners", Fixture) +{ + auto registrator = f.getRegistrator("testdoc"); + TEST_DO(f.assertAdds({})); + TEST_DO(f.assertRemoves({})); + registrator->addListener(std::make_unique<MyListener>("testdoc", "f1")); + registrator->addListener(std::make_unique<MyListener>("testdoc", "f2")); + registrator.reset(); + TEST_DO(f.assertAdds({{"testdoc","f1"},{"testdoc","f2"}})); + TEST_DO(f.assertRemoves({{"testdoc",{"f1","f2"}}})); +} + +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt index 94d7cec68ea..0d0a9593bb9 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt @@ -4,7 +4,9 @@ vespa_add_library(searchcore_reference STATIC document_db_reference_resolver.cpp document_db_referent.cpp document_db_referent_registry.cpp + gid_to_lid_change_handler.cpp gid_to_lid_change_listener.cpp + gid_to_lid_change_registrator.cpp gid_to_lid_mapper.cpp gid_to_lid_mapper_factory.cpp DEPENDS diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp new file mode 100644 index 00000000000..39f4b6b6cd1 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -0,0 +1,113 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "gid_to_lid_change_handler.h" +#include "i_gid_to_lid_change_listener.h" +#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/searchcorespi/index/i_thread_service.h> +#include <vespa/document/base/globalid.h> +#include <assert.h> + +using search::makeLambdaTask; + + +namespace proton { + +GidToLidChangeHandler::GidToLidChangeHandler(searchcorespi::index::IThreadService *master) + : _lock(), + _listeners(), + _master(master) +{ +} + + +GidToLidChangeHandler::~GidToLidChangeHandler() +{ + assert(_master == nullptr); + assert(_listeners.empty()); +} + +void +GidToLidChangeHandler::notifyGidToLidChange(document::GlobalId gid, uint32_t lid) +{ + for (const auto &listener : _listeners) { + listener->notifyGidToLidChange(gid, lid); + } +} + +void +GidToLidChangeHandler::close() +{ + lock_guard guard(_lock); + if (_master != nullptr) { + assert(_master->isCurrentThread()); + _master = nullptr; + _listeners.clear(); + } +} + +void +GidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener> listener) +{ + lock_guard guard(_lock); + if (_master) { + auto self(shared_from_this()); + _master->execute(makeLambdaTask([self,listener(std::move(listener))]() mutable { self->performAddListener(std::move(listener)); })); + } else { + assert(_listeners.empty()); + } +} + + +void +GidToLidChangeHandler::performAddListener(std::unique_ptr<IGidToLidChangeListener> listener) +{ + lock_guard guard(_lock); + if (_master) { + const vespalib::string &docTypeName = listener->getDocTypeName(); + const vespalib::string &name = listener->getName(); + for (const auto &oldlistener : _listeners) { + if (oldlistener->getDocTypeName() == docTypeName && oldlistener->getName() == name) { + return; + } + } + _listeners.emplace_back(std::move(listener)); + _listeners.back()->notifyRegistered(); + } else { + assert(_listeners.empty()); + } +} + +void +GidToLidChangeHandler::removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) +{ + lock_guard guard(_lock); + if (_master) { + auto self(shared_from_this()); + _master->execute(makeLambdaTask([self,docTypeName,keepNames]() mutable { self->performRemoveListener(docTypeName, keepNames); })); + } else { + assert(_listeners.empty()); + } +} + +void +GidToLidChangeHandler::performRemoveListener(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) +{ + lock_guard guard(_lock); + if (_master) { + auto itr = _listeners.begin(); + while (itr != _listeners.end()) { + if (((*itr)->getDocTypeName() == docTypeName) && + (keepNames.find((*itr)->getName()) == keepNames.end())) { + itr = _listeners.erase(itr); + } else { + ++itr; + } + } + } else { + assert(_listeners.empty()); + } +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h new file mode 100644 index 00000000000..1c6fabf3c77 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h @@ -0,0 +1,57 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_gid_to_lid_change_handler.h" +#include <vector> +#include <mutex> + +namespace document { class GlobalId; } +namespace searchcorespi { namespace index { class IThreadService; } } + +namespace proton { + +/* + * Class for registering listeners that get notification when + * gid to lid mapping changes. Also handles notifications which have + * to be executed by master thread service. + */ +class GidToLidChangeHandler : public std::enable_shared_from_this<GidToLidChangeHandler>, + public IGidToLidChangeHandler +{ + using lock_guard = std::lock_guard<std::mutex>; + std::mutex _lock; + std::vector<std::unique_ptr<IGidToLidChangeListener>> _listeners; + searchcorespi::index::IThreadService *_master; + + void performAddListener(std::unique_ptr<IGidToLidChangeListener> listener); + void performRemoveListener(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames); +public: + GidToLidChangeHandler(searchcorespi::index::IThreadService *master); + virtual ~GidToLidChangeHandler(); + + /** + * Notify gid to lid mapping change. Called by master executor. + */ + void notifyGidToLidChange(document::GlobalId gid, uint32_t lid); + + /** + * Close handler, further notifications are blocked. Called by master + * executor. + */ + void close(); + + /* + * Add listener unless a listener with matching cookie already exists. + */ + virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; + + /** + * Remove listener with matching cookie + */ + virtual void removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) override; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index 32b84172c45..7fedb93714c 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -8,10 +8,14 @@ namespace proton { GidToLidChangeListener::GidToLidChangeListener(search::ISequencedTaskExecutor &attributeFieldWriter, std::shared_ptr<search::attribute::ReferenceAttribute> attr, - MonitoredRefCount &refCount) + MonitoredRefCount &refCount, + const vespalib::string &name, + const vespalib::string &docTypeName) : _attributeFieldWriter(attributeFieldWriter), _attr(std::move(attr)), - _refCount(refCount) + _refCount(refCount), + _name(name), + _docTypeName(docTypeName) { _refCount.retain(); } @@ -30,4 +34,26 @@ GidToLidChangeListener::notifyGidToLidChange(document::GlobalId gid, uint32_t li (void) future.get(); } +void +GidToLidChangeListener::notifyRegistered() +{ + std::promise<bool> promise; + std::future<bool> future = promise.get_future(); + _attributeFieldWriter.execute(_attr->getName(), + [this, &promise]() { _attr->notifyGidToLidChangeListenerRegistered(); promise.set_value(true); }); + (void) future.get(); +} + +const vespalib::string & +GidToLidChangeListener::getName() const +{ + return _name; +} + +const vespalib::string & +GidToLidChangeListener::getDocTypeName() const +{ + return _docTypeName; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h index 94db5b4be6c..5b3edcb4ffb 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> +#include "i_gid_to_lid_change_listener.h" #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> @@ -14,18 +14,25 @@ namespace proton { * Class for listening to changes in mapping from gid to lid and updating * reference attribute appropriately. */ -class GidToLidChangeListener +class GidToLidChangeListener : public IGidToLidChangeListener { search::ISequencedTaskExecutor &_attributeFieldWriter; std::shared_ptr<search::attribute::ReferenceAttribute> _attr; MonitoredRefCount &_refCount; + vespalib::string _name; + vespalib::string _docTypeName; public: GidToLidChangeListener(search::ISequencedTaskExecutor &attributeFieldWriter, std::shared_ptr<search::attribute::ReferenceAttribute> attr, - MonitoredRefCount &refCount); + MonitoredRefCount &refCount, + const vespalib::string &name, + const vespalib::string &docTypeName); virtual ~GidToLidChangeListener(); - void notifyGidToLidChange(document::GlobalId gid, uint32_t lid); + virtual void notifyGidToLidChange(document::GlobalId gid, uint32_t lid) override; + virtual void notifyRegistered() override; + virtual const vespalib::string &getName() const override; + virtual const vespalib::string &getDocTypeName() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.cpp new file mode 100644 index 00000000000..99f03d19532 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.cpp @@ -0,0 +1,30 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "gid_to_lid_change_registrator.h" +#include "i_gid_to_lid_change_listener.h" +#include <assert.h> + +namespace proton { + +GidToLidChangeRegistrator::GidToLidChangeRegistrator(std::shared_ptr<IGidToLidChangeHandler> handler, + const vespalib::string &docTypeName) + : _handler(std::move(handler)), + _docTypeName(docTypeName), + _keepNames() +{ +} + +GidToLidChangeRegistrator::~GidToLidChangeRegistrator() +{ + _handler->removeListeners(_docTypeName, _keepNames); +} + +void +GidToLidChangeRegistrator::addListener(std::unique_ptr<IGidToLidChangeListener> listener) +{ + assert(listener->getDocTypeName() == _docTypeName); + _keepNames.insert(listener->getName()); + _handler->addListener(std::move(listener)); +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h new file mode 100644 index 00000000000..0f026dc287d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_registrator.h @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_gid_to_lid_change_handler.h" + +namespace proton { + +/* + * Helper class for registering listeners that get notification when + * gid to lid mapping changes. Will also unregister stale listeners for + * same doctype. + */ +class GidToLidChangeRegistrator +{ + std::shared_ptr<IGidToLidChangeHandler> _handler; + vespalib::string _docTypeName; + std::set<vespalib::string> _keepNames; + +public: + GidToLidChangeRegistrator(std::shared_ptr<IGidToLidChangeHandler> handler, + const vespalib::string &docTypeName); + ~GidToLidChangeRegistrator(); + void addListener(std::unique_ptr<IGidToLidChangeListener> listener); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h new file mode 100644 index 00000000000..b2b682b1d42 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <set> +#include <memory> +#include <vespa/vespalib/stllike/string.h> + +namespace proton { + +class IGidToLidChangeListener; + +/* + * Interface class for registering listeners that get notification when + * gid to lid mapping changes. + */ +class IGidToLidChangeHandler +{ +public: + virtual ~IGidToLidChangeHandler() { } + + virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) = 0; + + virtual void removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) = 0; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h new file mode 100644 index 00000000000..0aef4240aae --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <stdint.h> +#include <vespa/vespalib/stllike/string.h> + +namespace document { class GlobalId; } + +namespace proton { + +/* + * Interface class for listening to changes in mapping from gid to lid + * and updating reference attribute appropriately. + */ +class IGidToLidChangeListener +{ +public: + virtual ~IGidToLidChangeListener() { } + virtual void notifyGidToLidChange(document::GlobalId gid, uint32_t lid) = 0; + virtual void notifyRegistered() = 0; + virtual const vespalib::string &getName() const = 0; + virtual const vespalib::string &getDocTypeName() const = 0; +}; + +} // namespace proton diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index 61ac12e634f..a60cda51703 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -143,6 +143,11 @@ struct Fixture EXPECT_EQUAL(expLid, ref->lid()); } + void assertNoRefLid(uint32_t doc) { + auto ref = getRef(doc); + EXPECT_TRUE(ref == nullptr); + } + void save() { attr().save(); } @@ -179,6 +184,9 @@ struct Fixture void notifyGidToLidChange(const GlobalId &gid, uint32_t referencedDoc) { _attr->notifyGidToLidChange(gid, referencedDoc); } + void notifyGidToLidChangeListenerRegistered() { + _attr->notifyGidToLidChangeListenerRegistered(); + } }; TEST_F("require that we can instantiate reference attribute", Fixture) @@ -309,4 +317,28 @@ TEST_F("require that notifyGidToLidChange works", Fixture) TEST_DO(f.assertRefLid(10, 3)); } +TEST_F("require that notifyGidToLidChangeListenerRegistered works", Fixture) +{ + f.ensureDocIdLimit(6); + f.set(1, toGid(doc1)); + f.set(2, toGid(doc2)); + f.set(3, toGid(doc1)); + f.set(4, toGid(doc3)); + f.commit(); + TEST_DO(f.assertRefLid(0, 1)); + TEST_DO(f.assertRefLid(0, 2)); + TEST_DO(f.assertRefLid(0, 3)); + TEST_DO(f.assertRefLid(0, 4)); + TEST_DO(f.assertNoRefLid(5)); + std::shared_ptr<search::IGidToLidMapperFactory> factory = + std::make_shared<MyGidToLidMapperFactory>(); + f._attr->setGidToLidMapperFactory(factory); + f.notifyGidToLidChangeListenerRegistered(); + TEST_DO(f.assertRefLid(10, 1)); + TEST_DO(f.assertRefLid(17, 2)); + TEST_DO(f.assertRefLid(10, 3)); + TEST_DO(f.assertRefLid(0, 4)); + TEST_DO(f.assertNoRefLid(5)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index b3cfdbde054..479a8a7ce75 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -259,6 +259,18 @@ ReferenceAttribute::notifyGidToLidChange(const GlobalId &gid, DocId referencedLi } } +void +ReferenceAttribute::notifyGidToLidChangeListenerRegistered() +{ + if (_gidToLidMapperFactory) { + std::unique_ptr<IGidToLidMapper> mapperUP = _gidToLidMapperFactory->getMapper(); + const IGidToLidMapper &mapper = *mapperUP; + const auto &store = _store; + const auto saver = _store.getSaver(); + saver.foreach_key([&store,&mapper](EntryRef ref) { const Reference &entry = store.get(ref); entry.setLid(mapper.mapGidToLid(entry.gid())); }); + } +} + IMPLEMENT_IDENTIFIABLE_ABSTRACT(ReferenceAttribute, AttributeVector); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index ebff6c2bff6..670820844a0 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -77,6 +77,7 @@ public: std::shared_ptr<IGidToLidMapperFactory> getGidToLidMapperFactory() const { return _gidToLidMapperFactory; } DocId getReferencedLid(DocId doc) const; void notifyGidToLidChange(const GlobalId &gid, DocId referencedLid); + void notifyGidToLidChangeListenerRegistered(); }; } diff --git a/vespalib/src/vespa/vespalib/test/insertion_operators.h b/vespalib/src/vespa/vespalib/test/insertion_operators.h index ac4fa3541e3..6baeaacb2b8 100644 --- a/vespalib/src/vespa/vespalib/test/insertion_operators.h +++ b/vespalib/src/vespa/vespalib/test/insertion_operators.h @@ -59,5 +59,17 @@ operator<<(std::ostream &os, const std::map<K, V> &map) return os; } +template <typename T, typename U> +std::ostream & +operator<<(std::ostream &os, const std::pair<T,U> &pair) +{ + os << "{"; + os << pair.first; + os << ","; + os << pair.second; + os << "}"; + return os; +} + } // namespace std |