From ae2e84220d67f6c554a637fbf1e21aad78ffb5be Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 1 Feb 2024 15:29:21 +0000 Subject: GC void config from stor-visitor.def --- .../src/vespa/storage/visiting/stor-visitor.def | 24 ----- storage/src/vespa/storage/visiting/visitor.cpp | 4 +- .../src/vespa/storage/visiting/visitormanager.cpp | 6 -- .../src/vespa/storage/visiting/visitorthread.cpp | 103 +++++---------------- storage/src/vespa/storage/visiting/visitorthread.h | 6 -- 5 files changed, 25 insertions(+), 118 deletions(-) (limited to 'storage/src/vespa') diff --git a/storage/src/vespa/storage/visiting/stor-visitor.def b/storage/src/vespa/storage/visiting/stor-visitor.def index a8da6ee5032..752b4ce39df 100644 --- a/storage/src/vespa/storage/visiting/stor-visitor.def +++ b/storage/src/vespa/storage/visiting/stor-visitor.def @@ -5,35 +5,11 @@ namespace=vespa.config.content.core ## Keep in sync with #stor-filestor:num_visitor_threads visitorthreads int default=16 restart -## Default timeout of visitors that loses contact with client (in seconds) -disconnectedvisitortimeout int default=0 restart - -## Time period (in seconds) in which to ignore requests to visitors that doesnt -## exist anymore. (Normal for visitors to get some messages right after -## aborting, logging them as faults instead after this timeout has passed.) -ignorenonexistingvisitortimelimit int default=300 restart - ## The number of buckets that are visited in parallel in a visitor visiting ## multiple buckets. Default is 8, meaning if you send a create visitor to visit ## 100 buckets, 8 of them will be visited in parallel. defaultparalleliterators int default=8 -## Default number of maximum client replies pending. -defaultpendingmessages int default=32 - -## Default size of docblocks used to transfer visitor data. -defaultdocblocksize int default=4190208 - -## Default docblock timeout in ms used to transfer visitor data. -## Currently defaults to a day. This is to avoid slow visitor target problems, -## getting data resent faster than it can process, and since there are very few -## reasons to actually time out -defaultdocblocktimeout int default=180000 - -## Default timeout of visitor info messages: Progress and error reports. -## If these time out, the visitor will be aborted on the storage node. -defaultinfotimeout int default=60000 - ## Max concurrent visitors (legacy) maxconcurrentvisitors int default=64 diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 6904ecd1450..405ca292990 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -192,6 +192,8 @@ Visitor::Visitor(StorageComponent& component) _hitCounter(), _trace(DEFAULT_TRACE_MEMORY_LIMIT), _messageHandler(nullptr), + _messageSession(), + _documentPriority(documentapi::Priority::PRI_NORMAL_1), _id(), _controlDestination(), _dataDestination(), @@ -990,7 +992,7 @@ Visitor::getStatus(std::ostream& out, bool verbose) const != _visitorTarget._pendingMessages.end()) { out << "pending"; - }; + } auto queued = idToSendTime.find(idAndMeta.first); if (queued != idToSendTime.end()) { out << "Scheduled for sending at timestamp " diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index dc1635bc4b1..55948fb47cb 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -8,7 +8,6 @@ #include "recoveryvisitor.h" #include "reindexing_visitor.h" #include -#include #include #include #include @@ -117,11 +116,6 @@ void VisitorManager::on_configure(const vespa::config::content::core::StorVisitorConfig& config) { std::lock_guard sync(_visitorLock); - if (config.defaultdocblocksize % 512 != 0) { - throw config::InvalidConfigException( - "The default docblock size needs to be a multiple of the " - "disk block size. (512b)"); - } // Do some sanity checking of input. Cannot haphazardly mix and match // old and new max concurrency config values diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 0de954c47ed..57198f6761a 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -21,6 +21,9 @@ using storage::api::ReturnCode; namespace storage { +constexpr uint32_t DEFAULT_PENDING_MESSAGES = 32; +constexpr uint32_t DEFAULT_DOCBLOCK_SIZE = 4_Mi; + VisitorThread::Event::Event(Event&& other) noexcept : _visitorId(other._visitorId), _message(std::move(other._message)), @@ -82,15 +85,9 @@ VisitorThread::VisitorThread(uint32_t threadIndex, _messageSender(sender), _metrics(metrics), _threadIndex(threadIndex), - _disconnectedVisitorTimeout(0), // Need config to set values - _ignoreNonExistingVisitorTimeLimit(0), _defaultParallelIterators(0), _iteratorsPerBucket(1), - _defaultPendingMessages(0), - _defaultDocBlockSize(0), _visitorMemoryUsageLimit(UINT32_MAX), - _defaultDocBlockTimeout(180s), - _defaultVisitorInfoTimeout(60s), _timeBetweenTicks(1000), _component(componentRegister, getThreadName(threadIndex)), _messageSessionFactory(messageSessionFac), @@ -402,11 +399,9 @@ validateDocumentSelection(const document::DocumentTypeRepo& repo, } bool -VisitorThread::onCreateVisitor( - const std::shared_ptr& cmd) +VisitorThread::onCreateVisitor(const std::shared_ptr& cmd) { metrics::MetricTimer visitorTimer; - assert(_defaultDocBlockSize); // Ensure we've gotten a config assert(_currentlyRunningVisitor == _visitors.end()); ReturnCode result(ReturnCode::OK); std::unique_ptr docSelection; @@ -437,7 +432,7 @@ VisitorThread::onCreateVisitor( if (cmd->getMaximumPendingReplyCount() != 0) { visitor->setMaxPending(cmd->getMaximumPendingReplyCount()); } else { - visitor->setMaxPending(_defaultPendingMessages); + visitor->setMaxPending(DEFAULT_PENDING_MESSAGES); } visitor->setFieldSet(cmd->getFieldSet()); @@ -449,11 +444,11 @@ VisitorThread::onCreateVisitor( visitor->setMaxParallel(_defaultParallelIterators); visitor->setMaxParallelPerBucket(_iteratorsPerBucket); - visitor->setDocBlockSize(_defaultDocBlockSize); + visitor->setDocBlockSize(DEFAULT_DOCBLOCK_SIZE); visitor->setMemoryUsageLimit(_visitorMemoryUsageLimit); - visitor->setDocBlockTimeout(_defaultDocBlockTimeout); - visitor->setVisitorInfoTimeout(_defaultVisitorInfoTimeout); + visitor->setDocBlockTimeout(180s); + visitor->setVisitorInfoTimeout(60s); visitor->setOwnNodeIndex(_component.getIndex()); visitor->setBucketSpace(cmd->getBucketSpace()); @@ -546,70 +541,26 @@ VisitorThread::onInternal(const std::shared_ptr& cmd) { auto& pcmd = dynamic_cast(*cmd); const vespa::config::content::core::StorVisitorConfig& config(pcmd.getConfig()); - if (_defaultDocBlockSize != 0) { // Live update - LOG(config, "Updating visitor thread configuration in visitor " - "thread %u: " - "Current config(disconnectedVisitorTimeout %u," - " ignoreNonExistingVisitorTimeLimit %u," - " defaultParallelIterators %u," - " iteratorsPerBucket %u," - " defaultPendingMessages %u," - " defaultDocBlockSize %u," - " visitorMemoryUsageLimit %u," - " defaultDocBlockTimeout %" PRIu64 "," - " defaultVisitorInfoTimeout %" PRIu64 ") " - "New config(disconnectedVisitorTimeout %u," - " ignoreNonExistingVisitorTimeLimit %u," - " defaultParallelIterators %u," - " defaultPendingMessages %u," - " defaultDocBlockSize %u," - " visitorMemoryUsageLimit %u," - " defaultDocBlockTimeout %u," - " defaultVisitorInfoTimeout %u) ", - _threadIndex, - _disconnectedVisitorTimeout, - _ignoreNonExistingVisitorTimeLimit, - _defaultParallelIterators, - _iteratorsPerBucket, - _defaultPendingMessages, - _defaultDocBlockSize, - _visitorMemoryUsageLimit, - vespalib::count_ms(_defaultDocBlockTimeout), - vespalib::count_ms(_defaultVisitorInfoTimeout), - config.disconnectedvisitortimeout, - config.ignorenonexistingvisitortimelimit, - config.defaultparalleliterators, - config.defaultpendingmessages, - config.defaultdocblocksize, - config.visitorMemoryUsageLimit, - config.defaultdocblocktimeout, - config.defaultinfotimeout - ); - } - _disconnectedVisitorTimeout = config.disconnectedvisitortimeout; - _ignoreNonExistingVisitorTimeLimit = config.ignorenonexistingvisitortimelimit; + LOG(config, "Updating visitor thread configuration in visitor " + "thread %u: " + "Current config(defaultParallelIterators %u," + " iteratorsPerBucket %u," + " visitorMemoryUsageLimit %u)" + "New config(defaultParallelIterators %u," + " visitorMemoryUsageLimit %u)", + _threadIndex, + _defaultParallelIterators, + _iteratorsPerBucket, + _visitorMemoryUsageLimit, + config.defaultparalleliterators, + config.visitorMemoryUsageLimit + ); _defaultParallelIterators = config.defaultparalleliterators; - _defaultPendingMessages = config.defaultpendingmessages; - _defaultDocBlockSize = config.defaultdocblocksize; _visitorMemoryUsageLimit = config.visitorMemoryUsageLimit; - _defaultDocBlockTimeout = std::chrono::milliseconds(config.defaultdocblocktimeout); - _defaultVisitorInfoTimeout = std::chrono::milliseconds(config.defaultinfotimeout); if (_defaultParallelIterators < 1) { LOG(config, "Cannot use value of defaultParallelIterators < 1"); _defaultParallelIterators = 1; } - if (_defaultPendingMessages < 1) { - LOG(config, "Cannot use value of defaultPendingMessages < 1"); - _defaultPendingMessages = 1; - } - if (_defaultDocBlockSize < 1024) { - LOG(config, "Refusing to use default block size less than 1k"); - _defaultDocBlockSize = 1024; - } - if (_defaultDocBlockTimeout < 1ms) { - LOG(config, "Cannot use value of defaultDocBlockTimeout < 1"); - _defaultDocBlockTimeout = 1ms; - } break; } case RequestStatusPage::ID: @@ -695,20 +646,10 @@ VisitorThread::getStatus(vespalib::asciistream& out, out << "

Current queue size: " << _queue.size() << "

\n"; out << "

Config:

\n" << "\n" - << "\n" - << "\n" << "\n" << "\n" - << "\n" - << "\n" - << "\n" << "\n" << "
ParameterValue
Disconnected visitor timeout" - << _disconnectedVisitorTimeout << "
Ignore non-existing visitor timelimit" - << _ignoreNonExistingVisitorTimeLimit << "
Default parallel iterators" << _defaultParallelIterators << "
Iterators per bucket" << _iteratorsPerBucket << "
Default pending messages" - << _defaultPendingMessages << "
Default DocBlock size" - << _defaultDocBlockSize << "
Default DocBlock timeout (ms)" - << vespalib::count_ms(_defaultDocBlockTimeout) << "
Visitor memory usage limit" << _visitorMemoryUsageLimit << "
\n"; diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 4463a62fdd9..034c726e947 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -75,15 +75,9 @@ class VisitorThread : public framework::Runnable, VisitorMessageHandler& _messageSender; VisitorThreadMetrics& _metrics; uint32_t _threadIndex; - uint32_t _disconnectedVisitorTimeout; - uint32_t _ignoreNonExistingVisitorTimeLimit; uint32_t _defaultParallelIterators; uint32_t _iteratorsPerBucket; - uint32_t _defaultPendingMessages; - uint32_t _defaultDocBlockSize; uint32_t _visitorMemoryUsageLimit; - vespalib::duration _defaultDocBlockTimeout; - vespalib::duration _defaultVisitorInfoTimeout; std::atomic _timeBetweenTicks; StorageComponent _component; std::unique_ptr _thread; -- cgit v1.2.3