count failed heartbeats

5 files changed, 12 insertions, 3 deletions

diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
index 09af3eca194..d566e757248 100644
--- a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
+++ b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp
@@ -74,6 +74,9 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr,
              uint32_t prevTime, uint32_t currTime)
 {
     MetricSnapshot snapshot(prevTime, currTime);
+    snapshot.addCount("slobrok.heartbeats.failed",
+             "count of failed heartbeat requests",
+             curr.heartBeatFails - prev.heartBeatFails);
     snapshot.addCount("slobrok.requests.register",
              "count of register requests received",
              curr.registerReqs - prev.registerReqs);
@@ -92,7 +95,7 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr,
 MetricsProducer::MetricsProducer(const RPCHooks &hooks,
                                                FNET_Transport &transport)
     : _rpcHooks(hooks),
-      _lastMetrics{ 0, 0, 0, 0, 0, 0, 0},
+      _lastMetrics(RPCHooks::Metrics::zero()),
       _producer(),
       _startTime(time(NULL)),
       _lastSnapshotStart(_startTime),
@@ -114,7 +117,7 @@ MetricsProducer::getTotalMetrics(const vespalib::string &)
 {
     uint32_t now = time(NULL);
     RPCHooks::Metrics current = _rpcHooks.getMetrics();
-    RPCHooks::Metrics start{0, 0, 0, 0, 0, 0, 0};
+    RPCHooks::Metrics start = RPCHooks::Metrics::zero();
     return makeSnapshot(start, current, _startTime, now);
 }
 
diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
index f8d0a4a3897..a4cb5c067ac 100644
--- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
+++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
@@ -310,6 +310,7 @@ RpcServerManager::PerformTask()
 void
 RpcServerManager::notifyFailedRpcSrv(ManagedRpcServer *rpcsrv, std::string errmsg)
 {
+    _env.countFailedHeartbeat();
     bool logged = false;
     NamedService *old = _rpcsrvmap.lookup(rpcsrv->getName());
     if (old == rpcsrv) {
diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp
index 2eeafe7bbda..a79c0883cef 100644
--- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp
+++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp
@@ -46,7 +46,7 @@ RPCHooks::RPCHooks(SBEnv &env,
                    RpcServerMap& rpcsrvmap,
                    RpcServerManager& rpcsrvman)
     : _env(env), _rpcsrvmap(rpcsrvmap), _rpcsrvmanager(rpcsrvman),
-      _cnts{0, 0, 0, 0, 0, 0, 0},
+      _cnts(Metrics::zero()),
       _m_reporter(nullptr)
 {
 }
@@ -60,6 +60,7 @@ RPCHooks::~RPCHooks()
 void
 RPCHooks::reportMetrics()
 {
+    EV_COUNT("heartbeats_failed", _cnts.heartBeatFails);
     EV_COUNT("register_reqs", _cnts.registerReqs);
     EV_COUNT("mirror_reqs", _cnts.mirrorReqs);
     EV_COUNT("wantadd_reqs", _cnts.wantAddReqs);
diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h
index 0f560d62975..212f997a368 100644
--- a/slobrok/src/vespa/slobrok/server/rpchooks.h
+++ b/slobrok/src/vespa/slobrok/server/rpchooks.h
@@ -22,6 +22,7 @@ class RPCHooks : public FRT_Invokable
 public:
 
     struct Metrics {
+        unsigned long heartBeatFails;
         unsigned long registerReqs;
         unsigned long mirrorReqs;
         unsigned long wantAddReqs;
@@ -29,6 +30,7 @@ public:
         unsigned long doRemoveReqs;
         unsigned long adminReqs;
         unsigned long otherReqs;
+        static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0}; }
     };
 
 private:
@@ -51,6 +53,7 @@ public:
     void initRPC(FRT_Supervisor *supervisor);
     void reportMetrics();
     const Metrics& getMetrics() const { return _cnts; }
+    void countFailedHeartbeat() { _cnts.heartBeatFails++; }
 
 private:
     void rpc_lookupRpcServer(FRT_RPCRequest *req);
diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h
index 009fa6c6fa7..def29359379 100644
--- a/slobrok/src/vespa/slobrok/server/sbenv.h
+++ b/slobrok/src/vespa/slobrok/server/sbenv.h
@@ -85,6 +85,7 @@ public:
     OkState addPeer(const std::string& name, const std::string &spec);
     OkState removePeer(const std::string& name, const std::string &spec);
 
+    void countFailedHeartbeat() { _rpcHooks.countFailedHeartbeat(); }
 };
 
 } // namespace slobrok