diff options
7 files changed, 340 insertions, 4 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 208be618b1f..0dce35f7eb3 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -122,6 +122,7 @@ vespa_define_module( src/tests/proton/metrics/metrics_engine src/tests/proton/persistenceconformance src/tests/proton/persistenceengine + src/tests/proton/persistenceengine/persistence_handler_map src/tests/proton/proton src/tests/proton/proton_config_fetcher src/tests/proton/proton_configurer diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/CMakeLists.txt b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/CMakeLists.txt new file mode 100644 index 00000000000..ecfe8af5d98 --- /dev/null +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_persistence_handler_map_test_app TEST + SOURCES + persistence_handler_map_test.cpp + DEPENDS + searchcore_persistenceengine + searchcore_pcommon +) +vespa_add_test(NAME searchcore_persistence_handler_map_test_app COMMAND searchcore_persistence_handler_map_test_app) diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp new file mode 100644 index 00000000000..34de8a2e6a3 --- /dev/null +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -0,0 +1,144 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h> +#include <vespa/searchcore/proton/persistenceengine/persistence_handler_map.h> +#include <vespa/vespalib/testkit/testapp.h> + +using namespace document; +using namespace proton; + +using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; + +struct DummyPersistenceHandler : public IPersistenceHandler { + using SP = std::shared_ptr<DummyPersistenceHandler>; + virtual void initialize() override {} + virtual void handlePut(FeedToken, + const storage::spi::Bucket &, + storage::spi::Timestamp, + const document::Document::SP &) override {} + virtual void handleUpdate(FeedToken, + const storage::spi::Bucket &, + storage::spi::Timestamp, + const document::DocumentUpdate::SP &) override {} + virtual void handleRemove(FeedToken, + const storage::spi::Bucket &, + storage::spi::Timestamp, + const document::DocumentId &) override {} + virtual void handleListBuckets(IBucketIdListResultHandler &) override {} + virtual void handleSetClusterState(const storage::spi::ClusterState &, + IGenericResultHandler &) override {} + virtual void handleSetActiveState(const storage::spi::Bucket &, + storage::spi::BucketInfo::ActiveState, + IGenericResultHandler &) override {} + virtual void handleGetBucketInfo(const storage::spi::Bucket &, + IBucketInfoResultHandler &) override {} + virtual void handleCreateBucket(FeedToken, const storage::spi::Bucket &) override {} + virtual void handleDeleteBucket(FeedToken, const storage::spi::Bucket &) override {} + virtual void handleGetModifiedBuckets(IBucketIdListResultHandler &) override {} + virtual void handleSplit(FeedToken, + const storage::spi::Bucket &, + const storage::spi::Bucket &, + const storage::spi::Bucket &) override {} + virtual void handleJoin(FeedToken, + const storage::spi::Bucket &, + const storage::spi::Bucket &, + const storage::spi::Bucket &) override {} + virtual RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency) override { return RetrieversSP(); } + virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &) override { return BucketGuard::UP(); } + virtual void handleListActiveBuckets(IBucketIdListResultHandler &) override {} + virtual void handlePopulateActiveBuckets(document::BucketId::List &, + IGenericResultHandler &) override {} +}; + +BucketSpace space_1(1); +BucketSpace space_2(2); +BucketSpace space_null(3); +DocTypeName type_a("a"); +DocTypeName type_b("b"); +DocTypeName type_c("c"); +DummyPersistenceHandler::SP handler_a(std::make_shared<DummyPersistenceHandler>()); +DummyPersistenceHandler::SP handler_b(std::make_shared<DummyPersistenceHandler>()); +DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>()); +DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>()); + + +void +assertHandler(const IPersistenceHandler::SP &lhs, const IPersistenceHandler::SP &rhs) +{ + EXPECT_EQUAL(lhs.get(), rhs.get()); +} + +void +assertNullHandler(const IPersistenceHandler::SP &handler) +{ + EXPECT_TRUE(handler.get() == nullptr); +} + +void +assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, const HandlerSnapshot::UP &snapshot) +{ + EXPECT_EQUAL(exp.size(), snapshot->size()); + auto &sequence = snapshot->handlers(); + for (size_t i = 0; i < exp.size() && sequence.valid(); ++i, sequence.next()) { + EXPECT_EQUAL(exp[i].get(), sequence.get()); + } +} + +struct Fixture { + PersistenceHandlerMap map; + Fixture() { + TEST_DO(assertNullHandler(map.putHandler(space_1, type_a, handler_a))); + TEST_DO(assertNullHandler(map.putHandler(space_1, type_b, handler_b))); + TEST_DO(assertNullHandler(map.putHandler(space_2, type_c, handler_c))); + } +}; + +TEST_F("require that handlers can be retrieved", Fixture) +{ + TEST_DO(assertHandler(handler_a, f.map.getHandler(space_1, type_a))); + TEST_DO(assertHandler(handler_b, f.map.getHandler(space_1, type_b))); + TEST_DO(assertHandler(handler_c, f.map.getHandler(space_2, type_c))); + TEST_DO(assertNullHandler(f.map.getHandler(space_1, type_c))); + TEST_DO(assertNullHandler(f.map.getHandler(space_null, type_a))); +} + +TEST_F("require that old handler is returned if replaced by new handler", Fixture) +{ + TEST_DO(assertHandler(handler_a, f.map.putHandler(space_1, type_a, handler_a_new))); + TEST_DO(assertHandler(handler_a_new, f.map.getHandler(space_1, type_a))); +} + +TEST_F("require that handler can be removed (and old handler returned)", Fixture) +{ + TEST_DO(assertHandler(handler_a, f.map.removeHandler(space_1, type_a))); + TEST_DO(assertNullHandler(f.map.getHandler(space_1, type_a))); + TEST_DO(assertNullHandler(f.map.removeHandler(space_1, type_c))); +} + +TEST_F("require that handler snapshot can be retrieved for all handlers", Fixture) +{ + TEST_DO(assertSnapshot({handler_c, handler_a, handler_b}, f.map.getHandlerSnapshot())); +} + +TEST_F("require that handler snapshot can be retrieved for given bucket space", Fixture) +{ + TEST_DO(assertSnapshot({handler_a, handler_b}, f.map.getHandlerSnapshot(space_1))); + TEST_DO(assertSnapshot({handler_c}, f.map.getHandlerSnapshot(space_2))); + TEST_DO(assertSnapshot({}, f.map.getHandlerSnapshot(space_null))); +} + +TEST_F("require that handler snapshot can be retrieved for given document type (in bucket space)", Fixture) +{ + // Note: Document id doesn't contain document type -> all handlers returned + TEST_DO(assertSnapshot({handler_a, handler_b}, + f.map.getHandlerSnapshot(space_1, DocumentId("userdoc:namespace:1234:namespace")))); + TEST_DO(assertSnapshot({handler_a}, + f.map.getHandlerSnapshot(space_1, DocumentId("id:namespace:a::doc1")))); + EXPECT_TRUE(f.map.getHandlerSnapshot(space_1, DocumentId("id:namespace:c::doc2")).get() == nullptr); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} + diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index 5b19fa96e4c..6b5d89cbd22 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -43,12 +43,13 @@ public: public: typedef std::unique_ptr<Snapshot> UP; - Snapshot(StdMap &map) : _handlers(), _offset(0) { + Snapshot(const StdMap &map) : _handlers(), _offset(0) { _handlers.reserve(map.size()); - for (MapIterator pos = map.begin(); pos != map.end(); ++pos) { - _handlers.push_back(pos->second); + for (auto itr : map) { + _handlers.push_back(itr.second); } } + Snapshot(std::vector<HandlerSP> &&handlers) : _handlers(std::move(handlers)), _offset(0) {} bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } HandlerSP getSP() const { return _handlers[_offset]; } @@ -162,7 +163,7 @@ public: * @return handler sequence **/ std::unique_ptr<Snapshot> - snapshot() + snapshot() const { return std::unique_ptr<Snapshot>(new Snapshot(_handlers)); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt index eeda9ccfea6..7b0727d5496 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(searchcore_persistenceengine STATIC document_iterator.cpp i_document_retriever.cpp persistenceengine.cpp + persistence_handler_map.cpp transport_latch.cpp DEPENDS ) diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp new file mode 100644 index 00000000000..cf7d027f244 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -0,0 +1,111 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ipersistencehandler.h" +#include "persistence_handler_map.h" + +namespace proton { + +using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; + +PersistenceHandlerMap::PersistenceHandlerMap() + : _map() +{ +} + +IPersistenceHandler::SP +PersistenceHandlerMap::putHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType, + const IPersistenceHandler::SP &handler) +{ + return _map[bucketSpace].putHandler(docType, handler); +} + +IPersistenceHandler::SP +PersistenceHandlerMap::getHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType) const +{ + auto itr = _map.find(bucketSpace); + if (itr != _map.end()) { + return itr->second.getHandler(docType); + } + return IPersistenceHandler::SP(); +} + +IPersistenceHandler::SP +PersistenceHandlerMap::removeHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType) +{ + auto itr = _map.find(bucketSpace); + if (itr != _map.end()) { + return itr->second.removeHandler(docType); + } + return IPersistenceHandler::SP(); +} + +HandlerSnapshot::UP +PersistenceHandlerMap::getHandlerSnapshot() const +{ + std::vector<IPersistenceHandler::SP> handlers; + for (auto spaceItr : _map) { + for (auto handlerItr : spaceItr.second) { + handlers.push_back(handlerItr.second); + } + } + return std::make_unique<HandlerSnapshot> + (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)), + handlers.size()); +} + +namespace { + +struct EmptySequence : public vespalib::Sequence<IPersistenceHandler *> { + virtual bool valid() const override { return false; } + virtual IPersistenceHandler *get() const override { return nullptr; } + virtual void next() override { } + static EmptySequence::UP make() { return std::make_unique<EmptySequence>(); } +}; + +} + +HandlerSnapshot::UP +PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) const +{ + auto itr = _map.find(bucketSpace); + if (itr != _map.end()) { + return std::make_unique<HandlerSnapshot>(itr->second.snapshot(), itr->second.size()); + } + return std::make_unique<HandlerSnapshot>(EmptySequence::make(), 0); +} + +namespace { + +class SequenceOfOne : public vespalib::Sequence<IPersistenceHandler *> { +private: + bool _done; + IPersistenceHandler *_value; +public: + SequenceOfOne(IPersistenceHandler *value) : _done(false), _value(value) {} + + virtual bool valid() const override { return !_done; } + virtual IPersistenceHandler *get() const override { return _value; } + virtual void next() override { _done = true; } + static SequenceOfOne::UP make(IPersistenceHandler *value) { return std::make_unique<SequenceOfOne>(value); } +}; + +} + +HandlerSnapshot::UP +PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace, + const document::DocumentId &id) const +{ + if (!id.hasDocType()) { + return getHandlerSnapshot(bucketSpace); + } + IPersistenceHandler::SP handler = getHandler(bucketSpace, DocTypeName(id.getDocType())); + if (!handler.get()) { + return HandlerSnapshot::UP(); + } + return std::make_unique<HandlerSnapshot>(SequenceOfOne::make(handler.get()), 1); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h new file mode 100644 index 00000000000..8a852066284 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketspace.h> +#include <vespa/searchcore/proton/common/handlermap.hpp> +#include <vespa/vespalib/util/sequence.h> +#include <memory> +#include <unordered_map> + +namespace document { class DocumentId; } + +namespace proton { + +class DocTypeName; +class IPersistenceHandler; + +/** + * Class that maintains a set of PersistenceHandler instances + * and provides mapping from bucket space to the set of handlers in that space. + */ +class PersistenceHandlerMap { +public: + using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>; + + class HandlerSnapshot { + private: + PersistenceHandlerSequence::UP _handlers; + size_t _size; + public: + using UP = std::unique_ptr<HandlerSnapshot>; + HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_) + : _handlers(std::move(handlers_)), + _size(size_) + {} + HandlerSnapshot(const HandlerSnapshot &) = delete; + HandlerSnapshot & operator = (const HandlerSnapshot &) = delete; + + size_t size() const { return _size; } + PersistenceHandlerSequence &handlers() { return *_handlers; } + static PersistenceHandlerSequence::UP release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } + }; + +private: + using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + + struct BucketSpaceHash { + std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } + }; + + std::unordered_map<document::BucketSpace, DocTypeToHandlerMap, BucketSpaceHash> _map; + +public: + PersistenceHandlerMap(); + + PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType, + const PersistenceHandlerSP &handler); + PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType); + PersistenceHandlerSP getHandler(document::BucketSpace bucketSpace, + const DocTypeName &docType) const; + HandlerSnapshot::UP getHandlerSnapshot() const; + HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace, + const document::DocumentId &id) const; +}; + +} |