diff options
13 files changed, 49 insertions, 53 deletions
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 3f60189204b..7876f19b3ba 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -18,8 +18,8 @@ #include <vespa/storage/persistence/bucketownershipnotifier.h> #include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> -#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <vespa/storage/storageserver/statemanager.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/vdslib/state/random.h> diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 80ba4c19384..b393c4a5f8d 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -9,11 +9,13 @@ #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/persistence/spi/test.h> #include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> +#include <vespa/config-stor-filestor.h> #include <thread> using document::DocumentType; diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index 9ac0c7bbfc8..3dd8075176d 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/persistence/spi/test.h> #include <tests/persistence/persistencetestutils.h> diff --git a/storage/src/tests/persistence/provider_error_wrapper_test.cpp b/storage/src/tests/persistence/provider_error_wrapper_test.cpp index 2ba5218003b..d4d26a68844 100644 --- a/storage/src/tests/persistence/provider_error_wrapper_test.cpp +++ b/storage/src/tests/persistence/provider_error_wrapper_test.cpp @@ -3,6 +3,7 @@ #include <vespa/persistence/spi/test.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> +#include <vespa/storage/persistence/provider_error_wrapper.h> using storage::spi::test::makeSpiBucket; diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 228ea29ab42..a952e227b70 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // @author Vegard Sjonfjell -#include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <tests/persistence/persistencetestutils.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/documentapi/messagebus/messages/testandsetcondition.h> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 2c7aada01b1..832e07eaf95 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -2,11 +2,9 @@ #include "filestorhandlerimpl.h" #include "filestormanager.h" - #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/storage/bucketdb/minimumusedbitstracker.h> #include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/vdslib/state/cluster_state_bundle.h> @@ -14,9 +12,12 @@ #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> #include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/state.h> -#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> @@ -139,15 +140,15 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC _sequencedExecutor = vespalib::SequencedTaskExecutor::create(numResponseThreads, 10000, selectSequencer(_config->responseSequencerType)); assert(_sequencedExecutor); LOG(spam, "Setting up the disk"); - for (uint32_t j = 0; j < numThreads; j++) { - _persistenceComponents.push_back(std::make_unique<ServiceLayerComponent>(_compReg, createThreadName(j))); + for (uint32_t i = 0; i < numThreads; i++) { + _persistenceComponents.push_back(std::make_unique<ServiceLayerComponent>(_compReg, createThreadName(i))); _persistenceHandlers.push_back( std::make_unique<PersistenceHandler>(*_sequencedExecutor, *_persistenceComponents.back(), *_config, *_provider, *_filestorHandler, - *_bucketOwnershipNotifier, *_metrics->disks[0]->threads[j])); + *_bucketOwnershipNotifier, *_metrics->disks[0]->threads[i])); _threads.push_back(std::make_unique<PersistenceThread>(*_persistenceHandlers.back(), *_filestorHandler, - j % numStripes, _component)); + i % numStripes, _component)); } } } diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 090dda9f408..13ff1df340b 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -1,37 +1,26 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "persistencehandler.h" -#include <vespa/storage/common/bucketoperationlogger.h> -#include <vespa/document/fieldset/fieldsetrepo.h> -#include <vespa/document/base/exceptions.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> -#include <thread> - #include <vespa/log/log.h> -LOG_SETUP(".persistence.thread"); - -using vespalib::make_string_short::fmt; -using to_str = vespalib::string; +LOG_SETUP(".persistence.persistencehandler"); namespace storage { PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequencedExecutor, - ServiceLayerComponent & component, - const vespa::config::content::StorFilestorConfig & cfg, - spi::PersistenceProvider& provider, - FileStorHandler& filestorHandler, - BucketOwnershipNotifier & bucketOwnershipNotifier, - FileStorThreadMetrics& metrics) + ServiceLayerComponent & component, + const vespa::config::content::StorFilestorConfig & cfg, + spi::PersistenceProvider& provider, + FileStorHandler& filestorHandler, + BucketOwnershipNotifier & bucketOwnershipNotifier, + FileStorThreadMetrics& metrics) : _env(component, filestorHandler, metrics, provider), - _spi(provider), _processAllHandler(_env, provider), - _mergeHandler(_env, _spi, cfg.bucketMergeChunkSize, + _mergeHandler(_env, provider, cfg.bucketMergeChunkSize, cfg.enableMergeLocalNodeChooseDocsOptimalization, cfg.commonMergeChainOptimalizationMinimumSize), - _asyncHandler(_env, _spi, sequencedExecutor), + _asyncHandler(_env, provider, sequencedExecutor), _splitJoinHandler(_env, provider, bucketOwnershipNotifier, cfg.enableMultibitSplitOptimalization), _simpleHandler(_env, provider) { @@ -89,7 +78,7 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr case RecheckBucketInfoCommand::ID: return _splitJoinHandler.handleRecheckBucketInfo(static_cast<RecheckBucketInfoCommand&>(msg), std::move(tracker)); default: - LOG(warning, "Persistence thread received unhandled internal command %s", msg.toString().c_str()); + LOG(warning, "Persistence handler received unhandled internal command %s", msg.toString().c_str()); break; } default: diff --git a/storage/src/vespa/storage/persistence/persistencehandler.h b/storage/src/vespa/storage/persistence/persistencehandler.h index 7453be1173a..2cfac865484 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.h +++ b/storage/src/vespa/storage/persistence/persistencehandler.h @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once @@ -17,6 +17,11 @@ namespace storage { class BucketOwnershipNotifier; +/** + * Handle all messages destined for the persistence layer. The detailed handling + * happens in other handlers, but is forked out after common setup has been done. + * Currently metrics are updated so each thread should have its own instance. + */ class PersistenceHandler : public Types { public: @@ -39,7 +44,6 @@ private: MessageTracker::UP processMessage(api::StorageMessage& msg, MessageTracker::UP tracker) const; PersistenceUtil _env; - spi::PersistenceProvider& _spi; ProcessAllHandler _processAllHandler; MergeHandler _mergeHandler; AsyncHandler _asyncHandler; diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 623cc8a4372..5738e2257ae 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "persistencethread.h" +#include "persistencehandler.h" #include <thread> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index 19eb0811351..9022ab5a862 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -3,11 +3,17 @@ #pragma once #include "diskthread.h" -#include "persistencehandler.h" +#include "types.h" namespace storage { -class BucketOwnershipNotifier; +namespace framework { + class Component; + class Thread; +} + +class PersistenceHandler; +class FileStorHandler; class PersistenceThread final : public DiskThread, public Types { @@ -21,10 +27,10 @@ public: framework::Thread& getThread() override { return *_thread; } private: - PersistenceHandler & _persistenceHandler; - FileStorHandler & _fileStorHandler; - uint32_t _stripeId; - framework::Thread::UP _thread; + PersistenceHandler & _persistenceHandler; + FileStorHandler & _fileStorHandler; + uint32_t _stripeId; + std::unique_ptr<framework::Thread> _thread; void run(framework::ThreadHandle&) override; }; diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index d4108e6322d..5e5682a8bb4 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -13,9 +13,6 @@ #pragma once #include <vespa/persistence/spi/persistenceprovider.h> -#include <vector> -#include <string> -#include <memory> #include <mutex> namespace storage { diff --git a/storageapi/src/vespa/storageapi/message/stat.cpp b/storageapi/src/vespa/storageapi/message/stat.cpp index a971aa0c4d5..8d47b7703dc 100644 --- a/storageapi/src/vespa/storageapi/message/stat.cpp +++ b/storageapi/src/vespa/storageapi/message/stat.cpp @@ -3,8 +3,7 @@ #include "stat.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { IMPLEMENT_COMMAND(StatBucketCommand, StatBucketReply) IMPLEMENT_REPLY(StatBucketReply) @@ -18,7 +17,7 @@ StatBucketCommand::StatBucketCommand(const document::Bucket& bucket, { } -StatBucketCommand::~StatBucketCommand() {} +StatBucketCommand::~StatBucketCommand() = default; void StatBucketCommand::print(std::ostream& out, bool verbose, @@ -102,5 +101,4 @@ operator<<(std::ostream& out, const GetBucketListReply::BucketInfo& instance) return out; } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h index a7d9a30ca6e..e25cb4596c9 100644 --- a/storageapi/src/vespa/storageapi/message/stat.h +++ b/storageapi/src/vespa/storageapi/message/stat.h @@ -5,9 +5,7 @@ #include <vespa/storageapi/messageapi/bucketcommand.h> #include <vespa/storageapi/messageapi/bucketreply.h> -namespace storage { - -namespace api { +namespace storage::api { /** * \class StatBucketCommand @@ -89,5 +87,4 @@ public: std::ostream& operator<<(std::ostream& out, const GetBucketListReply::BucketInfo& instance); -} // api -} // storage +} |