From 27acd0a8d5d678e542c4dc94973cbd616d785128 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Tue, 19 Oct 2021 15:34:11 +0200 Subject: Move async_apply_bucket_diff config to stor-filestor.def. Pass config to merge handler. --- configdefinitions/src/vespa/stor-filestor.def | 4 ++++ storage/src/vespa/storage/config/stor-server.def | 4 ---- .../src/vespa/storage/persistence/filestorage/filestormanager.cpp | 1 - storage/src/vespa/storage/persistence/mergehandler.cpp | 6 ++++-- storage/src/vespa/storage/persistence/mergehandler.h | 4 +++- storage/src/vespa/storage/persistence/persistencehandler.cpp | 3 ++- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 88028cef394..66700eff3e6 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -43,6 +43,10 @@ common_merge_chain_optimalization_minimum_size int default=64 restart ## Note that this will gradually be increased to reach stor-distributormanager:splitsize which is currently at 32M bucket_merge_chunk_size int default=33554432 restart +## If set, portions of apply bucket diff handling will be performed asynchronously +## with persistence thread not waiting for local writes to complete. +async_apply_bucket_diff bool default=false + ## When merging, it is possible to send more metadata than needed in order to ## let local nodes in merge decide which entries fits best to add this time ## based on disk location. Toggle this option on to use it. Note that memory diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index db660fb70d0..6611c3cba91 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -53,10 +53,6 @@ resource_exhaustion_merge_back_pressure_duration_secs double default=30.0 ## a busy-reply that would subsequently be unwound through the entire merge chain. disable_queue_limits_for_chained_merges bool default=false -## If set, portions of apply bucket diff handling will be performed asynchronously -## with persistence thread not waiting for local writes to complete. -async_apply_bucket_diff bool default=false - ## Whether the deadlock detector should be enabled or not. If disabled, it will ## still run, but it will never actually abort the process it is running in. enable_dead_lock_detector bool default=false restart diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index ddcab0f3659..2ffb827accf 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 4b0560424e5..bc7d1d7d282 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -23,13 +23,15 @@ namespace storage { MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, const ClusterContext& cluster_context, const framework::Clock & clock, uint32_t maxChunkSize, - uint32_t commonMergeChainOptimalizationMinimumSize) + uint32_t commonMergeChainOptimalizationMinimumSize, + bool async_apply_bucket_diff) : _clock(clock), _cluster_context(cluster_context), _env(env), _spi(spi), _maxChunkSize(maxChunkSize), - _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize) + _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize), + _async_apply_bucket_diff(async_apply_bucket_diff) { } diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index e3b5b0899f6..0ff8f3c0ef8 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -45,7 +45,8 @@ public: MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, const ClusterContext& cluster_context, const framework::Clock & clock, uint32_t maxChunkSize = 4190208, - uint32_t commonMergeChainOptimalizationMinimumSize = 64); + uint32_t commonMergeChainOptimalizationMinimumSize = 64, + bool async_apply_bucket_diff = false); bool buildBucketInfoList( const spi::Bucket& bucket, @@ -77,6 +78,7 @@ private: spi::PersistenceProvider &_spi; const uint32_t _maxChunkSize; const uint32_t _commonMergeChainOptimalizationMinimumSize; + const bool _async_apply_bucket_diff; /** Returns a reply if merge is complete */ api::StorageReply::SP processBucketMerge(const spi::Bucket& bucket, diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 2106d1385ae..1ef883fc810 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -19,7 +19,8 @@ PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequen _processAllHandler(_env, provider), _mergeHandler(_env, provider, component.cluster_context(), _clock, cfg.bucketMergeChunkSize, - cfg.commonMergeChainOptimalizationMinimumSize), + cfg.commonMergeChainOptimalizationMinimumSize, + cfg.asyncApplyBucketDiff), _asyncHandler(_env, provider, bucketOwnershipNotifier, sequencedExecutor, component.getBucketIdFactory()), _splitJoinHandler(_env, provider, bucketOwnershipNotifier, cfg.enableMultibitSplitOptimalization), _simpleHandler(_env, provider) -- cgit v1.2.3