From 989cd45ef2ebb540128c548eefc79862294fde36 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 20 Oct 2021 11:33:01 +0200 Subject: Add --async-apply-bucket-diff option to vespa-redistribute-bm. --- .../src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp | 8 +++++++- searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp | 3 ++- searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h | 3 +++ searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp index b957226dd8a..f9833b430a0 100644 --- a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp +++ b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp @@ -466,6 +466,7 @@ App::usage() "USAGE:\n"; std::cerr << "vespa-redistribute-bm\n" + "[--async-apply-bucket-diff]\n" "[--bucket-db-stripe-bits bits]\n" "[--client-threads threads]\n" "[--distributor-merge-busy-wait distributor-merge-busy-wait]\n" @@ -481,7 +482,7 @@ App::usage() "[--max-merge-queue-size max-merge-queue-size]\n" "[--max-pending max-pending]\n" "[--max-pending-idealstate-operations max-pending-idealstate-operations]\n" - "[--mbus-distributor-node-max-pending-count] count\n" + "[--mbus-distributor-node-max-pending-count count]\n" "[--mode [grow, shrink, perm-crash, temp-crash, replace]\n" "[--nodes-per-group nodes-per-group]\n" "[--redundancy redundancy]\n" @@ -502,6 +503,7 @@ App::get_options() const char *opt_argument = nullptr; int long_opt_index = 0; static struct option long_opts[] = { + { "async-apply-bucket-diff", 0, nullptr, 0 }, { "bucket-db-stripe-bits", 1, nullptr, 0 }, { "client-threads", 1, nullptr, 0 }, { "distributor-merge-busy-wait", 1, nullptr, 0 }, @@ -532,6 +534,7 @@ App::get_options() { nullptr, 0, nullptr, 0 } }; enum longopts_enum { + LONGOPT_ASYNC_APPLY_BUCKET_DIFF, LONGOPT_BUCKET_DB_STRIPE_BITS, LONGOPT_CLIENT_THREADS, LONGOPT_DISTRIBUTOR_MERGE_BUSY_WAIT, @@ -566,6 +569,9 @@ App::get_options() switch (c) { case 0: switch(long_opt_index) { + case LONGOPT_ASYNC_APPLY_BUCKET_DIFF: + _bm_params.set_async_apply_bucket_diff(true); + break; case LONGOPT_BUCKET_DB_STRIPE_BITS: _bm_params.set_bucket_db_stripe_bits(atoi(opt_argument)); break; diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp index 3ff10b19164..4a3466f1a51 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp @@ -6,7 +6,8 @@ namespace search::bmcluster { BmClusterParams::BmClusterParams() - : _bucket_db_stripe_bits(4), + : _async_apply_bucket_diff(), + _bucket_db_stripe_bits(4), _disable_queue_limits_for_chained_merges(false), // Same default as in stor-server.def _distributor_merge_busy_wait(10), // Same default as stor_distributormanager.def _distributor_stripes(0), diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h index d365a28b0b6..36b4c22f6a8 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h @@ -13,6 +13,7 @@ namespace search::bmcluster { */ class BmClusterParams { + std::optional _async_apply_bucket_diff; uint32_t _bucket_db_stripe_bits; bool _disable_queue_limits_for_chained_merges; uint32_t _distributor_merge_busy_wait; @@ -44,6 +45,7 @@ class BmClusterParams public: BmClusterParams(); ~BmClusterParams(); + const std::optional& get_async_apply_bucket_diff() const noexcept { return _async_apply_bucket_diff; } uint32_t get_bucket_db_stripe_bits() const { return _bucket_db_stripe_bits; } bool get_disable_queue_limits_for_chained_merges() const noexcept { return _disable_queue_limits_for_chained_merges; } uint32_t get_distributor_merge_busy_wait() const { return _distributor_merge_busy_wait; } @@ -73,6 +75,7 @@ public: bool needs_distributor() const { return _enable_distributor || _use_document_api; } bool needs_message_bus() const { return _use_message_bus || _use_document_api; } bool needs_service_layer() const { return _enable_service_layer || _enable_distributor || _use_storage_chain || _use_message_bus || _use_document_api; } + void set_async_apply_bucket_diff(bool value) { _async_apply_bucket_diff = value; } void set_bucket_db_stripe_bits(uint32_t value) { _bucket_db_stripe_bits = value; } void set_disable_queue_limits_for_chained_merges(bool value) { _disable_queue_limits_for_chained_merges = value; } void set_distributor_merge_busy_wait(uint32_t value) { _distributor_merge_busy_wait = value; } diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 545fb08c762..0882153edd6 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -382,6 +382,9 @@ struct ServiceLayerConfigSet : public StorageConfigSet stor_bucket_init(), stor_visitor() { + if (params.get_async_apply_bucket_diff().has_value()) { + stor_filestor.asyncApplyBucketDiff = params.get_async_apply_bucket_diff().value(); + } stor_filestor.numResponseThreads = params.get_response_threads(); stor_filestor.numNetworkThreads = params.get_rpc_network_threads(); stor_filestor.useAsyncMessageHandlingOnSchedule = params.get_use_async_message_handling_on_schedule(); -- cgit v1.2.3