add metric for time since last full consensus

5 files changed, 32 insertions, 4 deletions

diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp
index 45bf22cd7a0..98b21391e9d 100644
--- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp
+++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp
@@ -11,11 +11,19 @@ LOG_SETUP(".slobrok.server.exchange_manager");
 
 namespace slobrok {
 
+namespace {
+unsigned long now() {
+    using namespace std::chrono;
+    return duration_cast<seconds>(steady_clock::now().time_since_epoch()).count();
+}
+}
+
 //-----------------------------------------------------------------------------
 
 ExchangeManager::ExchangeManager(SBEnv &env)
     : _partners(),
-      _env(env)
+      _env(env),
+      _lastFullConsensusTime(now())
 {
 }
 
@@ -102,6 +110,7 @@ ExchangeManager::diffLists(const ServiceMappingList &lhs, const ServiceMappingLi
 void
 ExchangeManager::healthCheck()
 {
+    bool someBad = false;
     auto newWorldList = env().consensusMap().currentConsensus();
     for (const auto & [ name, partner ] : _partners) {
         partner->maybeStartFetch();
@@ -112,9 +121,16 @@ ExchangeManager::healthCheck()
             if (! diff.empty()) {
                 LOG(warning, "Diff from consensus map to peer slobrok mirror: %s",
                     diff.c_str());
+                someBad = true;
             }
         }
     }
+    if (someBad) {
+        _env.setConsensusTime(now() - _lastFullConsensusTime);
+    } else {
+        _lastFullConsensusTime = now();
+        _env.setConsensusTime(0);
+    }
     LOG(debug, "ExchangeManager::healthCheck for %ld partners", _partners.size());
 }
 
diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.h b/slobrok/src/vespa/slobrok/server/exchange_manager.h
index c59d14c90c9..eeb2cc539ce 100644
--- a/slobrok/src/vespa/slobrok/server/exchange_manager.h
+++ b/slobrok/src/vespa/slobrok/server/exchange_manager.h
@@ -73,6 +73,7 @@ private:
     };
 
     SBEnv             &_env;
+    unsigned long      _lastFullConsensusTime;
 
     vespalib::string diffLists(const ServiceMappingList &lhs, const ServiceMappingList &rhs);
 
diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
index 52c7382fc6e..d4ab22981f1 100644
--- a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
+++ b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
@@ -96,6 +96,9 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr,
     snapshot.addCount("slobrok.requests.admin",
              "count of administrative requests received",
              curr.adminReqs - prev.adminReqs);
+    snapshot.addCount("slobrok.missing.consensus",
+             "number of seconds without full consensus with all other brokers",
+             curr.missingConsensusTime);
     return snapshot.asString();
 }
 
diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h
index 0919aebca91..d98cd88a081 100644
--- a/slobrok/src/vespa/slobrok/server/rpchooks.h
+++ b/slobrok/src/vespa/slobrok/server/rpchooks.h
@@ -35,7 +35,8 @@ public:
         unsigned long doRemoveReqs;
         unsigned long adminReqs;
         unsigned long otherReqs;
-        static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0}; }
+        unsigned long missingConsensusTime;
+        static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0,0}; }
     };
 
 private:
@@ -54,7 +55,9 @@ public:
     void reportMetrics();
     const Metrics& getMetrics() const { return _cnts; }
     void countFailedHeartbeat() { _cnts.heartBeatFails++; }
-
+    void setConsensusTime(unsigned long value) {
+        _cnts.missingConsensusTime = value;
+    }
 private:
     void rpc_registerRpcServer(FRT_RPCRequest *req);
     void rpc_unregisterRpcServer(FRT_RPCRequest *req);
diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h
index 653d4b75818..212c163d0cc 100644
--- a/slobrok/src/vespa/slobrok/server/sbenv.h
+++ b/slobrok/src/vespa/slobrok/server/sbenv.h
@@ -105,7 +105,12 @@ public:
     OkState addPeer(const std::string& name, const std::string& spec);
     OkState removePeer(const std::string& name, const std::string& spec);
 
-    void countFailedHeartbeat() { _rpcHooks.countFailedHeartbeat(); }
+    void countFailedHeartbeat() {
+        _rpcHooks.countFailedHeartbeat();
+    }
+    void setConsensusTime(unsigned long value) {
+        _rpcHooks.setConsensusTime(value);
+    }
 };
 
 } // namespace slobrok