From 43b5a800afbda9b49845f3cf67f155a7f4a22c5c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 2 Feb 2024 13:36:32 +0000 Subject: GC maxpendingidealstateoperations which has not been wired in for a long time. --- .../src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp | 5 ----- searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp | 1 - searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h | 3 --- searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp | 1 - storage/src/tests/distributor/top_level_distributor_test.cpp | 1 - storage/src/vespa/storage/config/distributorconfiguration.cpp | 8 ++------ storage/src/vespa/storage/config/distributorconfiguration.h | 1 - storage/src/vespa/storage/config/stor-distributormanager.def | 3 --- 8 files changed, 2 insertions(+), 21 deletions(-) diff --git a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp index 859fc407945..d0b1d69ba47 100644 --- a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp +++ b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include #include #include #include @@ -540,7 +539,6 @@ App::get_options(int argc, char **argv) LONGOPT_MAX_MERGES_PER_NODE, LONGOPT_MAX_MERGE_QUEUE_SIZE, LONGOPT_MAX_PENDING, - LONGOPT_MAX_PENDING_IDEALSTATE_OPERATIONS, LONGOPT_MBUS_DISTRIBUTOR_NODE_MAX_PENDING_COUNT, LONGOPT_MODE, LONGOPT_NODES_PER_GROUP, @@ -600,9 +598,6 @@ App::get_options(int argc, char **argv) case LONGOPT_MAX_PENDING: _bm_params.set_max_pending(atoi(optarg)); break; - case LONGOPT_MAX_PENDING_IDEALSTATE_OPERATIONS: - _bm_params.set_max_pending_idealstate_operations(atoi(optarg)); - break; case LONGOPT_MBUS_DISTRIBUTOR_NODE_MAX_PENDING_COUNT: _bm_params.set_mbus_distributor_node_max_pending_count(atoi(optarg)); break; diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp index 0310bdc7319..12f90523760 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp @@ -18,7 +18,6 @@ BmClusterParams::BmClusterParams() _indexing_sequencer(), _max_merges_per_node(16), // Same default as in stor-server.def _max_merge_queue_size(1024), // Same default as in stor-server.def - _max_pending_idealstate_operations(100), // Same default as in stor-distributormanager.def _mbus_distributor_node_max_pending_count(), _num_nodes(1), _nodes_per_group(1), diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h index 13cd9e31604..585d6426b87 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h @@ -25,7 +25,6 @@ class BmClusterParams vespalib::string _indexing_sequencer; uint32_t _max_merges_per_node; uint32_t _max_merge_queue_size; - uint32_t _max_pending_idealstate_operations; std::optional _mbus_distributor_node_max_pending_count; uint32_t _num_nodes; uint32_t _nodes_per_group; @@ -54,7 +53,6 @@ public: const vespalib::string & get_indexing_sequencer() const { return _indexing_sequencer; } uint32_t get_max_merges_per_node() const noexcept { return _max_merges_per_node; } uint32_t get_max_merge_queue_size() const noexcept { return _max_merge_queue_size; } - uint32_t get_max_pending_idealstate_operations() const noexcept { return _max_pending_idealstate_operations; } const std::optional& get_mbus_distributor_node_max_pending_count() const noexcept { return _mbus_distributor_node_max_pending_count; } uint32_t get_nodes_per_group() const noexcept { return _nodes_per_group; } uint32_t get_num_nodes() const { return _num_nodes; } @@ -83,7 +81,6 @@ public: void set_indexing_sequencer(vespalib::stringref sequencer) { _indexing_sequencer = sequencer; } void set_max_merges_per_node(uint32_t value) { _max_merges_per_node = value; } void set_max_merge_queue_size(uint32_t value) { _max_merge_queue_size = value; } - void set_max_pending_idealstate_operations(uint32_t value) { _max_pending_idealstate_operations = value; } void set_mbus_distributor_node_max_pending_count(int32_t value) { _mbus_distributor_node_max_pending_count = value; } void set_nodes_per_group(uint32_t value); void set_redundancy(uint32_t value) { _redundancy = value; } diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index ede5728026e..9bc9a47c59d 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -386,7 +386,6 @@ struct DistributorConfigSet : public StorageConfigSet stor_visitordispatcher() { stor_distributormanager.inhibitMergeSendingOnBusyNodeDurationSec = params.get_distributor_merge_busy_wait(); - stor_distributormanager.maxpendingidealstateoperations = params.get_max_pending_idealstate_operations(); stor_distributormanager.numDistributorStripes = params.get_distributor_stripes(); } diff --git a/storage/src/tests/distributor/top_level_distributor_test.cpp b/storage/src/tests/distributor/top_level_distributor_test.cpp index 4e44abe71fb..15905586633 100644 --- a/storage/src/tests/distributor/top_level_distributor_test.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test.cpp @@ -290,7 +290,6 @@ TEST_F(TopLevelDistributorTest, metric_update_hook_updates_pending_maintenance_m setup_distributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); // To ensure we count all operations, not just those fitting within the pending window. auto cfg = current_distributor_config(); - cfg.maxpendingidealstateoperations = 1; // FIXME STRIPE this does not actually seem to be used...! reconfigure(cfg); // 1 bucket must be merged, 1 must be split, 1 should be activated. diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 443dcc32174..cc399d6b4eb 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -20,7 +20,6 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component) _byteCountJoinLimit(0), _docCountJoinLimit(0), _minimalBucketSplit(16), - _maxIdealStateOperations(100), _maxNodesPerMerge(16), _max_consecutively_inhibited_maintenance_ticks(20), _max_activation_inhibited_out_of_sync_groups(0), @@ -104,7 +103,6 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); } - _maxIdealStateOperations = config.maxpendingidealstateoperations; _byteCountSplitLimit = config.splitsize; _docCountSplitLimit = config.splitcount; _byteCountJoinLimit = config.joinsize; @@ -159,16 +157,14 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist LOG(debug, "Distributor now using new configuration parameters. Split limits: %d docs/%d bytes. " "Join limits: %d docs/%d bytes. Minimal bucket split %d. " - "Documents to garbage collect: %s (check every %d seconds). " - "Maximum pending ideal state operations: %d", + "Documents to garbage collect: %s (check every %d seconds). ", (int)_docCountSplitLimit, (int)_byteCountSplitLimit, (int)_docCountJoinLimit, (int)_byteCountJoinLimit, (int)_minimalBucketSplit, _garbageCollectionSelection.c_str(), - (int)vespalib::to_s(_garbageCollectionInterval), - (int)_maxIdealStateOperations); + (int)vespalib::to_s(_garbageCollectionInterval)); } void diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index 5b1e1c0af4c..d2fa3f2c35a 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -267,7 +267,6 @@ private: uint32_t _byteCountJoinLimit; uint32_t _docCountJoinLimit; uint32_t _minimalBucketSplit; - uint32_t _maxIdealStateOperations; uint32_t _maxNodesPerMerge; uint32_t _max_consecutively_inhibited_maintenance_ticks; uint32_t _max_activation_inhibited_out_of_sync_groups; diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index cf6e65c13bb..a9a31977e87 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -1,9 +1,6 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.config.content.core -## Maximum number of ideal state operations scheduled by a distributor. -maxpendingidealstateoperations int default=100 - ## The total size of unique documents in a bucket before we split it due to ## being too big. By default this is now 16 MB. Should be kept in sync with stor-filestor.def:bucket_merge_chunk_size. splitsize int default=16772216 -- cgit v1.2.3 From 88d5984a2fea5054176669d5626f0983e75cb5dc Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 2 Feb 2024 15:36:04 +0000 Subject: Must also gc param description in parallell array. --- searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp index d0b1d69ba47..444c1157cc4 100644 --- a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp +++ b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp @@ -510,7 +510,6 @@ App::get_options(int argc, char **argv) { "max-merges-per-node", 1, nullptr, 0 }, { "max-merge-queue-size", 1, nullptr, 0 }, { "max-pending", 1, nullptr, 0 }, - { "max-pending-idealstate-operations", 1, nullptr, 0 }, { "mbus-distributor-node-max-pending-count", 1, nullptr, 0 }, { "mode", 1, nullptr, 0 }, { "nodes-per-group", 1, nullptr, 0 }, -- cgit v1.2.3