diff options
21 files changed, 171 insertions, 140 deletions
diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h index b3627d99d31..95737e9d079 100644 --- a/fastos/src/vespa/fastos/thread.h +++ b/fastos/src/vespa/fastos/thread.h @@ -381,7 +381,7 @@ public: /** * Returns the id of this thread. */ - virtual FastOS_ThreadId GetThreadId ()=0; + virtual FastOS_ThreadId GetThreadId () const noexcept = 0; }; diff --git a/fastos/src/vespa/fastos/unix_thread.cpp b/fastos/src/vespa/fastos/unix_thread.cpp index a45d90426ef..aac4c4d189c 100644 --- a/fastos/src/vespa/fastos/unix_thread.cpp +++ b/fastos/src/vespa/fastos/unix_thread.cpp @@ -102,7 +102,7 @@ FastOS_UNIX_Thread::~FastOS_UNIX_Thread() } } -FastOS_ThreadId FastOS_UNIX_Thread::GetThreadId () +FastOS_ThreadId FastOS_UNIX_Thread::GetThreadId () const noexcept { return _handle; } diff --git a/fastos/src/vespa/fastos/unix_thread.h b/fastos/src/vespa/fastos/unix_thread.h index 18e840183cb..b2f666060ca 100644 --- a/fastos/src/vespa/fastos/unix_thread.h +++ b/fastos/src/vespa/fastos/unix_thread.h @@ -36,7 +36,7 @@ public: ~FastOS_UNIX_Thread(); - FastOS_ThreadId GetThreadId () override; + FastOS_ThreadId GetThreadId () const noexcept override; static bool CompareThreadIds (FastOS_ThreadId a, FastOS_ThreadId b); static FastOS_ThreadId GetCurrentThreadId (); }; diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 386ceb4eeda..333e79216ad 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -56,6 +56,7 @@ App::setupSignals() SIG::PIPE.ignore(); SIG::INT.hook(); SIG::TERM.hook(); + SIG::enable_cross_thread_stack_tracing(); } void diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index 00be2929229..36da2ed0333 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -65,17 +65,17 @@ disable_queue_limits_for_chained_merges bool default=true ## Whether the deadlock detector should be enabled or not. If disabled, it will ## still run, but it will never actually abort the process it is running in. -enable_dead_lock_detector bool default=false restart +enable_dead_lock_detector bool default=false ## Whether to enable deadlock detector warnings in log or not. If enabled, ## warnings will be written even if dead lock detecting is not enabled. -enable_dead_lock_detector_warnings bool default=true restart +enable_dead_lock_detector_warnings bool default=true ## Each thread registers how often it will at minimum register ticks (given that ## the system is not overloaded. If you are running Vespa on overloaded nodes, ## you can use this slack timeout to add to the thread timeouts in order to ## allow for more slack before dead lock detector kicks in. The value is in seconds. -dead_lock_detector_timeout_slack double default=240 restart +dead_lock_detector_timeout_slack double default=240 ## If set to 0, storage will attempt to auto-detect the number of VDS mount ## points to use. If set to a number, force this number. This number only makes diff --git a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.cpp b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.cpp index 5353335aae8..110de6bc6af 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.cpp @@ -9,10 +9,10 @@ LOG_SETUP(".deadlock.killer"); namespace storage { void RealAppKiller::kill() { - LOG(info, "Aborting the server to dump core, as we're " - "most likely deadlocked and want a core file " - "to view the stack traces."); - LOG_ABORT("should not be reached"); + LOG(error, "One or more threads have failed internal liveness checks; aborting process. " + "A core dump will be generated (if enabled by the kernel). " + "Please report this to the Vespa team at https://github.com/vespa-engine/vespa/issues"); + abort(); } } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index c49a2961a3e..5acc2e4d14d 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -4,6 +4,7 @@ #include "htmltable.h" #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/content_bucket_space_repo.h> +#include <vespa/storageframework/generic/thread/thread.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/bufferedlogger.h> @@ -22,8 +23,7 @@ DeadLockDetector::DeadLockDetector(StorageComponentRegister& compReg, AppKiller: _enableWarning(true), _enableShutdown(false), _processSlack(30s), - _waitSlack(5s), - _reportedBucketDBLocksAtState(OK) + _waitSlack(5s) { auto* dComp(dynamic_cast<DistributorComponentRegister*>(&compReg)); if (dComp) { @@ -49,19 +49,21 @@ DeadLockDetector::~DeadLockDetector() void DeadLockDetector::enableWarning(bool enable) { - if (enable == _enableWarning) return; - LOG(debug, "%s dead lock detection warnings", - enable ? "Enabling" : "Disabling"); - _enableWarning = enable; + if (enable == warning_enabled_relaxed()) { + return; + } + LOG(debug, "%s dead lock detection warnings", enable ? "Enabling" : "Disabling"); + _enableWarning.store(enable, std::memory_order_relaxed); } void DeadLockDetector::enableShutdown(bool enable) { - if (enable == _enableShutdown) return; - LOG(debug, "%s dead lock detection", - enable ? "Enabling" : "Disabling"); - _enableShutdown = enable; + if (enable == shutdown_enabled_relaxed()) { + return; + } + LOG(debug, "%s dead lock detection", enable ? "Enabling" : "Disabling"); + _enableShutdown.store(enable, std::memory_order_relaxed); } namespace { @@ -76,14 +78,11 @@ namespace { { } - void visitThread(const vespalib::string& id, - const framework::ThreadProperties& p, - const framework::ThreadTickData& td) override - { - if (_states.find(id) == _states.end()) { - _states[id] = DeadLockDetector::OK; + void visitThread(const framework::Thread& thread) override { + if (_states.find(thread.getId()) == _states.end()) { + _states[thread.getId()] = DeadLockDetector::OK; } - _visitor.visitThread(id, p, td, _states[id]); + _visitor.visitThread(thread, _states[thread.getId()]); } }; } @@ -145,25 +144,26 @@ namespace { ThreadChecker(DeadLockDetector& d, vespalib::steady_time time) : _detector(d), _currentTime(time) {} - void visitThread(const vespalib::string& id, - const framework::ThreadProperties& tp, - const framework::ThreadTickData& tick, + void visitThread(const framework::Thread& thread, DeadLockDetector::State& state) override { - // In case we just got a new tick, ignore the thread + const auto& id = thread.getId(); + const auto& tp = thread.getProperties(); + const auto tick = thread.getTickData(); + // In case we just got a new tick, ignore the thread if (tick._lastTick > _currentTime) return; - // If thread is already in halted state, ignore it. + // If thread is already in halted state, ignore it. if (state == DeadLockDetector::HALTED) return; if (_detector.isAboveFailThreshold(_currentTime, tp, tick)) { state = DeadLockDetector::HALTED; - _detector.handleDeadlock(_currentTime, id, tp, tick, false); + _detector.handleDeadlock(_currentTime, thread, id, tp, tick, false); } else if (_detector.isAboveWarnThreshold(_currentTime, tp, tick)) { state = DeadLockDetector::WARNED; - _detector.handleDeadlock(_currentTime, id, tp, tick, true); + _detector.handleDeadlock(_currentTime, thread, id, tp, tick, true); } else if (state != DeadLockDetector::OK) { vespalib::asciistream ost; - ost << "Thread " << id << " has registered tick again.\n"; + ost << "Thread " << id << " has registered tick again."; LOGBP(info, "%s", ost.str().data()); state = DeadLockDetector::OK; } @@ -173,6 +173,7 @@ namespace { void DeadLockDetector::handleDeadlock(vespalib::steady_time currentTime, + const framework::Thread& deadlocked_thread, const vespalib::string& id, const framework::ThreadProperties&, const framework::ThreadTickData& tick, @@ -182,40 +183,30 @@ DeadLockDetector::handleDeadlock(vespalib::steady_time currentTime, error << "Thread " << id << " has gone " << vespalib::count_ms(currentTime - tick._lastTick) << " milliseconds without registering a tick."; + const bool shutdown_enabled = shutdown_enabled_relaxed(); + const bool warning_enabled = warning_enabled_relaxed(); if (!warnOnly) { - if (_enableShutdown && !warnOnly) { - error << " Restarting process due to deadlock."; + if (shutdown_enabled) { + error << " Restarting process due to presumed internal deadlock."; } else { - error << " Would have restarted process due to " - << "deadlock if shutdown had been enabled."; + error << " Would have restarted process due to deadlock if shutdown had been enabled."; } } else { - error << " Global slack not expended yet. Warning for now."; + // TODO would ideally print thread ID here, but it's not well-defined how to print a pthread_t... + error << " Global slack not expended yet. Warning for now. Attempting to dump stack of thread...\n"; + error << deadlocked_thread.get_live_thread_stack_trace(); } if (warnOnly) { - if (_enableWarning) { - LOGBT(warning, "deadlockw-" + id, "%s", - error.str().data()); - if (_reportedBucketDBLocksAtState != WARNED) { - _reportedBucketDBLocksAtState = WARNED; - LOG(info, "Locks in bucket database at deadlock time:\n%s", - getBucketLockInfo().c_str()); - } + if (warning_enabled) { + LOGBT(warning, "deadlockw-" + id, "%s", vespalib::string(error.str()).c_str()); } return; } else { - if (_enableShutdown || _enableWarning) { - LOGBT(error, "deadlock-" + id, "%s", - error.str().data()); + if (shutdown_enabled || warning_enabled) { + LOGBT(error, "deadlock-" + id, "%s", vespalib::string(error.str()).c_str()); } } - if (!_enableShutdown) return; - if (_reportedBucketDBLocksAtState != HALTED) { - _reportedBucketDBLocksAtState = HALTED; - LOG(info, "Locks in bucket database at deadlock time:" - "\n%s", getBucketLockInfo().c_str()); - } - if (_enableShutdown) { + if (shutdown_enabled) { _killer->kill(); } } @@ -278,13 +269,13 @@ namespace { return ost.str(); } - void visitThread(const vespalib::string& id, - const framework::ThreadProperties& tp, - const framework::ThreadTickData& tick, + void visitThread(const framework::Thread& thread, DeadLockDetector::State& /*state*/) override { - _table._table.addRow(id); + _table._table.addRow(thread.getId()); uint32_t i = _table._table.getRowCount() - 1; + const auto& tp = thread.getProperties(); + const auto tick = thread.getTickData(); _table._msSinceLastTick[i] = vespalib::count_ms(_time - tick._lastTick); _table._maxProcTickTime[i] = vespalib::count_ms(tp.getMaxProcessTime()); _table._maxWaitTickTime[i] = vespalib::count_ms(tp.getWaitTime()); @@ -315,7 +306,7 @@ DeadLockDetector::reportHtmlStatus(std::ostream& os, << " appear before this slack time is expendede on top of the per" << " thread value.\n" << "</p>\n"; - if (_enableShutdown) { + if (shutdown_enabled_relaxed()) { out << "<p>The deadlock detector is enabled and will kill the process " << "if a deadlock is detected</p>\n"; } else { diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h index 1ef70063206..e2a2278a727 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h @@ -17,12 +17,13 @@ #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/thread/threadpool.h> -#include <map> #include <atomic> - +#include <map> namespace storage { +namespace framework { class Thread; } + struct DeadLockDetector : private framework::Runnable, private framework::HtmlStatusReporter { @@ -32,8 +33,8 @@ struct DeadLockDetector : private framework::Runnable, AppKiller::UP killer = std::make_unique<RealAppKiller>()); ~DeadLockDetector() override; - void enableWarning(bool enable); - void enableShutdown(bool enable); + void enableWarning(bool enable); // Thread-safe + void enableShutdown(bool enable); // Thread-safe // There are no data read/write dependencies on neither _processSlack // nor _waitSlack so relaxed ops suffice. void setProcessSlack(vespalib::duration slack) { @@ -49,14 +50,11 @@ struct DeadLockDetector : private framework::Runnable, return _waitSlack.load(std::memory_order_relaxed); } - // These utility functions are public as internal anonymous classes are - // using them. Can also be useful for whitebox testing. + // These utility functions are public as internal anonymous classes are + // using them. Can also be useful for whitebox testing. struct ThreadVisitor { virtual ~ThreadVisitor() = default; - virtual void visitThread(const vespalib::string& id, - const framework::ThreadProperties&, - const framework::ThreadTickData&, - State& state) = 0; + virtual void visitThread(const framework::Thread& thread, State& state) = 0; }; void visitThreads(ThreadVisitor&) const; @@ -67,21 +65,29 @@ struct DeadLockDetector : private framework::Runnable, const framework::ThreadProperties& tp, const framework::ThreadTickData& tick) const; void handleDeadlock(vespalib::steady_time currentTime, + const framework::Thread& deadlocked_thread, const vespalib::string& id, const framework::ThreadProperties& tp, const framework::ThreadTickData& tick, bool warnOnly); + // Note: returned value may change between calls due to reconfiguration by other threads + [[nodiscard]] bool warning_enabled_relaxed() const noexcept { + return _enableWarning.load(std::memory_order_relaxed); + } + [[nodiscard]] bool shutdown_enabled_relaxed() const noexcept { + return _enableShutdown.load(std::memory_order_relaxed); + } + private: AppKiller::UP _killer; mutable std::map<vespalib::string, State> _states; mutable std::mutex _lock; std::condition_variable _cond; - bool _enableWarning; - bool _enableShutdown; + std::atomic<bool> _enableWarning; + std::atomic<bool> _enableShutdown; std::atomic<vespalib::duration> _processSlack; std::atomic<vespalib::duration> _waitSlack; - State _reportedBucketDBLocksAtState; DistributorComponent::UP _dComponent; ServiceLayerComponent::UP _slComponent; StorageComponent* _component; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 588ce356503..a4701d9f396 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -273,7 +273,6 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) // we want to handle. if (_newServerConfig) { - bool updated = false; StorServerConfigBuilder oldC(*_serverConfig); StorServerConfig& newC(*_newServerConfig); DIFFERWARN(rootFolder, "Cannot alter root folder of node live"); @@ -282,7 +281,10 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) DIFFERWARN(isDistributor, "Cannot alter role of node live"); _serverConfig = std::make_unique<StorServerConfig>(oldC); _newServerConfig.reset(); - (void)updated; + _deadLockDetector->enableWarning(_serverConfig->enableDeadLockDetectorWarnings); + _deadLockDetector->enableShutdown(_serverConfig->enableDeadLockDetector); + _deadLockDetector->setProcessSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); + _deadLockDetector->setWaitSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); } if (_newDistributionConfig) { StorDistributionConfigBuilder oldC(*_distributionConfig); diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index b8ef8e4610b..179f315b65a 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -4,6 +4,7 @@ #include "threadpoolimpl.h" #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/vespalib/util/atomic.h> +#include <vespa/vespalib/util/signalhandler.h> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".framework.thread.impl"); @@ -78,6 +79,12 @@ ThreadImpl::join() _thread.join(); } +vespalib::string +ThreadImpl::get_live_thread_stack_trace() const +{ + return vespalib::SignalHandler::get_cross_thread_stack_trace(_thread.native_thread_id()); +} + void ThreadImpl::registerTick(CycleType cycleType, vespalib::steady_time now) { diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index 46a9412bf67..8bbf64efcc4 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h @@ -13,7 +13,7 @@ namespace storage::framework::defaultimplementation { struct ThreadPoolImpl; -class ThreadImpl : public Thread +class ThreadImpl final : public Thread { struct BackendThread : public document::Runnable { ThreadImpl& _impl; @@ -61,12 +61,15 @@ public: ThreadImpl(ThreadPoolImpl&, Runnable&, vespalib::stringref id, vespalib::duration waitTime, vespalib::duration maxProcessTime, int ticksBeforeWait, std::optional<vespalib::CpuUsage::Category> cpu_category); - ~ThreadImpl(); + ~ThreadImpl() override; bool interrupted() const override; bool joined() const override; void interrupt() override; void join() override; + + vespalib::string get_live_thread_stack_trace() const override; + void registerTick(CycleType, vespalib::steady_time) override; vespalib::duration getWaitTime() const override { return _properties.getWaitTime(); @@ -76,8 +79,8 @@ public: } void setTickData(const ThreadTickData&); - ThreadTickData getTickData() const; - const ThreadProperties& getProperties() const { return _properties; } + ThreadTickData getTickData() const override; + const ThreadProperties& getProperties() const override { return _properties; } }; } diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index 7711eddf51c..6ac4e214e5d 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp @@ -4,7 +4,9 @@ #include "threadimpl.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <thread> + #include <vespa/log/log.h> LOG_SETUP(".storageframework.thread_pool_impl"); @@ -52,9 +54,7 @@ ThreadPoolImpl::startThread(Runnable& runnable, vespalib::stringref id, vespalib std::optional<vespalib::CpuUsage::Category> cpu_category) { std::lock_guard lock(_threadVectorLock); - if (_stopping) { - throw IllegalStateException("Threadpool is stopping", VESPA_STRLOC); - } + assert(!_stopping); auto thread = std::make_unique<ThreadImpl>(*this, runnable, id, waitTime, maxProcessTime, ticksBeforeWait, cpu_category); _threads.push_back(thread.get()); return thread; @@ -65,7 +65,7 @@ ThreadPoolImpl::visitThreads(ThreadVisitor& visitor) const { std::lock_guard lock(_threadVectorLock); for (const ThreadImpl * thread : _threads) { - visitor.visitThread(thread->getId(), thread->getProperties(), thread->getTickData()); + visitor.visitThread(*thread); } } diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index d6053a2f128..dcdc0981fe4 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -9,7 +9,7 @@ namespace storage::framework::defaultimplementation { class ThreadImpl; -struct ThreadPoolImpl : public ThreadPool +struct ThreadPoolImpl final : public ThreadPool { FastOS_ThreadPool _backendThreadPool; std::vector<ThreadImpl*> _threads; diff --git a/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt b/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt index a392725b771..fc7796f0dd8 100644 --- a/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_library(storageframework_thread OBJECT SOURCES thread.cpp - threadpool.cpp + thread_properties.cpp tickingthread.cpp DEPENDS ) diff --git a/storage/src/vespa/storageframework/generic/thread/thread.h b/storage/src/vespa/storageframework/generic/thread/thread.h index c17638a0d42..444376b87a5 100644 --- a/storage/src/vespa/storageframework/generic/thread/thread.h +++ b/storage/src/vespa/storageframework/generic/thread/thread.h @@ -13,10 +13,19 @@ #pragma once #include "runnable.h" +#include "thread_properties.h" #include <condition_variable> namespace storage::framework { +/** Data kept on each thread due to the registerTick functionality. */ +struct ThreadTickData { + CycleType _lastTickType; + vespalib::steady_time _lastTick; + vespalib::duration _maxProcessingTimeSeen; + vespalib::duration _maxWaitTimeSeen; +}; + class Thread : public ThreadHandle { vespalib::string _id; @@ -45,6 +54,11 @@ public: */ virtual void join() = 0; + virtual ThreadTickData getTickData() const = 0; + virtual const ThreadProperties& getProperties() const = 0; + + virtual vespalib::string get_live_thread_stack_trace() const = 0; + /** * Utility function to interrupt and join a thread, possibly broadcasting * through a monitor after the signalling face. diff --git a/storage/src/vespa/storageframework/generic/thread/threadpool.cpp b/storage/src/vespa/storageframework/generic/thread/thread_properties.cpp index 480e42c91ef..0822c53408b 100644 --- a/storage/src/vespa/storageframework/generic/thread/threadpool.cpp +++ b/storage/src/vespa/storageframework/generic/thread/thread_properties.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "threadpool.h" +#include "thread_properties.h" namespace storage::framework { diff --git a/storage/src/vespa/storageframework/generic/thread/thread_properties.h b/storage/src/vespa/storageframework/generic/thread/thread_properties.h new file mode 100644 index 00000000000..2f45cf331e8 --- /dev/null +++ b/storage/src/vespa/storageframework/generic/thread/thread_properties.h @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageframework/generic/clock/time.h> +#include <cstdint> + +namespace storage::framework { + +/** + * Each thread may have different properties, as to how long they wait between + * ticks and how long they're supposed to use processing between ticks. To be + * able to specify this per thread, a set of properties can be set by each + * thread. + */ +class ThreadProperties { +private: + /** + * Time this thread should maximum use to process before a tick is + * registered. (Including wait time if wait time is not set) + */ + vespalib::duration _maxProcessTime; + /** + * Time this thread will wait in a non-interrupted wait cycle. + * Used in cases where a wait cycle is registered. As long as no other + * time consuming stuff is done in a wait cycle, you can just use the + * wait time here. The deadlock detector should add a configurable + * global time period before flagging deadlock anyways. + */ + vespalib::duration _waitTime; + /** + * Number of ticks to be done before a wait. + */ + uint32_t _ticksBeforeWait; + +public: + ThreadProperties(vespalib::duration waitTime, + vespalib::duration maxProcessTime, + int ticksBeforeWait); + + vespalib::duration getMaxProcessTime() const { return _maxProcessTime; } + vespalib::duration getWaitTime() const { return _waitTime; } + int getTicksBeforeWait() const { return _ticksBeforeWait; } + + vespalib::duration getMaxCycleTime() const { + return std::max(_maxProcessTime, _waitTime); + } +}; + +} diff --git a/storage/src/vespa/storageframework/generic/thread/threadpool.h b/storage/src/vespa/storageframework/generic/thread/threadpool.h index 7607932e079..927a6ca35df 100644 --- a/storage/src/vespa/storageframework/generic/thread/threadpool.h +++ b/storage/src/vespa/storageframework/generic/thread/threadpool.h @@ -22,60 +22,10 @@ namespace storage::framework { -/** - * Each thread may have different properties, as to how long they wait between - * ticks and how long they're supposed to use processing between ticks. To be - * able to specify this per thread, a set of properties can be set by each - * thread. - */ -class ThreadProperties { -private: - /** - * Time this thread should maximum use to process before a tick is - * registered. (Including wait time if wait time is not set) - */ - vespalib::duration _maxProcessTime; - /** - * Time this thread will wait in a non-interrupted wait cycle. - * Used in cases where a wait cycle is registered. As long as no other - * time consuming stuff is done in a wait cycle, you can just use the - * wait time here. The deadlock detector should add a configurable - * global time period before flagging deadlock anyways. - */ - vespalib::duration _waitTime; - /** - * Number of ticks to be done before a wait. - */ - uint32_t _ticksBeforeWait; - - public: - ThreadProperties(vespalib::duration waitTime, - vespalib::duration maxProcessTime, - int ticksBeforeWait); - - vespalib::duration getMaxProcessTime() const { return _maxProcessTime; } - vespalib::duration getWaitTime() const { return _waitTime; } - int getTicksBeforeWait() const { return _ticksBeforeWait; } - - vespalib::duration getMaxCycleTime() const { - return std::max(_maxProcessTime, _waitTime); - } -}; - -/** Data kept on each thread due to the registerTick functinality. */ -struct ThreadTickData { - CycleType _lastTickType; - vespalib::steady_time _lastTick; - vespalib::duration _maxProcessingTimeSeen; - vespalib::duration _maxWaitTimeSeen; -}; - /** Interface used to access data for the existing threads. */ struct ThreadVisitor { virtual ~ThreadVisitor() = default; - virtual void visitThread(const vespalib::string& id, - const ThreadProperties&, - const ThreadTickData&) = 0; + virtual void visitThread(const Thread& thread) = 0; }; struct ThreadPool { diff --git a/storageserver/src/apps/storaged/storage.cpp b/storageserver/src/apps/storaged/storage.cpp index 87c03714492..d2a78657881 100644 --- a/storageserver/src/apps/storaged/storage.cpp +++ b/storageserver/src/apps/storaged/storage.cpp @@ -209,6 +209,7 @@ int StorageApp::main(int argc, char **argv) int main(int argc, char **argv) { vespalib::SignalHandler::PIPE.ignore(); + vespalib::SignalHandler::enable_cross_thread_stack_tracing(); storage::StorageApp app; storage::sigtramp = &app; int retval = app.main(argc,argv); diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.cpp b/vespalib/src/vespa/vespalib/util/document_runnable.cpp index d7534514f41..c0af72dbbb1 100644 --- a/vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ b/vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -72,6 +72,11 @@ bool Runnable::join() const return true; } +FastOS_ThreadId Runnable::native_thread_id() const noexcept +{ + return GetThread()->GetThreadId(); +} + void Runnable::Run(FastOS_ThreadInterface*, void*) { { diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.h b/vespalib/src/vespa/vespalib/util/document_runnable.h index 5ca344ea7ef..89388bac34c 100644 --- a/vespalib/src/vespa/vespalib/util/document_runnable.h +++ b/vespalib/src/vespa/vespalib/util/document_runnable.h @@ -88,6 +88,8 @@ public: * Checks if runnable is running or not. (Started is considered running) */ [[nodiscard]] bool running() const noexcept; + + FastOS_ThreadId native_thread_id() const noexcept; }; } |