diff options
author | Arne Juul <arnej@yahooinc.com> | 2023-01-05 15:20:53 +0000 |
---|---|---|
committer | Arne Juul <arnej@yahooinc.com> | 2023-01-05 15:20:53 +0000 |
commit | 57bd3821ade70a0bd52a09b73f5e578841ef4b54 (patch) | |
tree | 01209314bcca50380a7736cfea718128bfa6e78c /slobrok | |
parent | 6cbf34743397d5039dcff4c19893401cfd471cd1 (diff) |
add metric for time since last full consensus
Diffstat (limited to 'slobrok')
-rw-r--r-- | slobrok/src/vespa/slobrok/server/exchange_manager.cpp | 18 | ||||
-rw-r--r-- | slobrok/src/vespa/slobrok/server/exchange_manager.h | 1 | ||||
-rw-r--r-- | slobrok/src/vespa/slobrok/server/metrics_producer.cpp | 3 | ||||
-rw-r--r-- | slobrok/src/vespa/slobrok/server/rpchooks.h | 7 | ||||
-rw-r--r-- | slobrok/src/vespa/slobrok/server/sbenv.h | 7 |
5 files changed, 32 insertions, 4 deletions
diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index 45bf22cd7a0..98b21391e9d 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -11,11 +11,19 @@ LOG_SETUP(".slobrok.server.exchange_manager"); namespace slobrok { +namespace { +unsigned long now() { + using namespace std::chrono; + return duration_cast<seconds>(steady_clock::now().time_since_epoch()).count(); +} +} + //----------------------------------------------------------------------------- ExchangeManager::ExchangeManager(SBEnv &env) : _partners(), - _env(env) + _env(env), + _lastFullConsensusTime(now()) { } @@ -102,6 +110,7 @@ ExchangeManager::diffLists(const ServiceMappingList &lhs, const ServiceMappingLi void ExchangeManager::healthCheck() { + bool someBad = false; auto newWorldList = env().consensusMap().currentConsensus(); for (const auto & [ name, partner ] : _partners) { partner->maybeStartFetch(); @@ -112,9 +121,16 @@ ExchangeManager::healthCheck() if (! diff.empty()) { LOG(warning, "Diff from consensus map to peer slobrok mirror: %s", diff.c_str()); + someBad = true; } } } + if (someBad) { + _env.setConsensusTime(now() - _lastFullConsensusTime); + } else { + _lastFullConsensusTime = now(); + _env.setConsensusTime(0); + } LOG(debug, "ExchangeManager::healthCheck for %ld partners", _partners.size()); } diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.h b/slobrok/src/vespa/slobrok/server/exchange_manager.h index c59d14c90c9..eeb2cc539ce 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.h +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.h @@ -73,6 +73,7 @@ private: }; SBEnv &_env; + unsigned long _lastFullConsensusTime; vespalib::string diffLists(const ServiceMappingList &lhs, const ServiceMappingList &rhs); diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp index 52c7382fc6e..d4ab22981f1 100644 --- a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp +++ b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp @@ -96,6 +96,9 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr, snapshot.addCount("slobrok.requests.admin", "count of administrative requests received", curr.adminReqs - prev.adminReqs); + snapshot.addCount("slobrok.missing.consensus", + "number of seconds without full consensus with all other brokers", + curr.missingConsensusTime); return snapshot.asString(); } diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h index 0919aebca91..d98cd88a081 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.h +++ b/slobrok/src/vespa/slobrok/server/rpchooks.h @@ -35,7 +35,8 @@ public: unsigned long doRemoveReqs; unsigned long adminReqs; unsigned long otherReqs; - static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0}; } + unsigned long missingConsensusTime; + static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0,0}; } }; private: @@ -54,7 +55,9 @@ public: void reportMetrics(); const Metrics& getMetrics() const { return _cnts; } void countFailedHeartbeat() { _cnts.heartBeatFails++; } - + void setConsensusTime(unsigned long value) { + _cnts.missingConsensusTime = value; + } private: void rpc_registerRpcServer(FRT_RPCRequest *req); void rpc_unregisterRpcServer(FRT_RPCRequest *req); diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index 653d4b75818..212c163d0cc 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -105,7 +105,12 @@ public: OkState addPeer(const std::string& name, const std::string& spec); OkState removePeer(const std::string& name, const std::string& spec); - void countFailedHeartbeat() { _rpcHooks.countFailedHeartbeat(); } + void countFailedHeartbeat() { + _rpcHooks.countFailedHeartbeat(); + } + void setConsensusTime(unsigned long value) { + _rpcHooks.setConsensusTime(value); + } }; } // namespace slobrok |