Add heart beat age metric.

5 files changed, 26 insertions, 1 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
index e032cc1cef6..b5c6df972a8 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
@@ -272,6 +272,7 @@ DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docType
       bucketMove(this),
       totalMemoryUsage(this),
       totalDiskUsage("disk_usage", {}, "The total disk usage (in bytes) for this document db", this),
+      heart_beat_age("heart_beat_age", {}, "How long ago heart beat maintenace job was run", this),
       maxNumThreads(maxNumThreads_)
 {
 }
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
index c1bc5633d19..7f2e095a39b 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
@@ -205,6 +205,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
     BucketMoveMetrics bucketMove;
     MemoryUsageMetrics totalMemoryUsage;
     metrics::LongValueMetric totalDiskUsage;
+    metrics::DoubleValueMetric heart_beat_age;
     size_t maxNumThreads;
 
     DocumentDBTaggedMetrics(const vespalib::string &docTypeName, size_t maxNumThreads_);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index fadd6fd0726..6897a182862 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -1077,6 +1077,13 @@ DocumentDB::updateMetrics(const metrics::MetricLockGuard & guard)
     }
     _metricsUpdater.updateMetrics(guard, _metrics);
     _maintenanceController.updateMetrics(_metrics);
+    auto heart_beat_time = _feedHandler->get_heart_beat_time();
+    if (heart_beat_time != vespalib::steady_time()) {
+        vespalib::steady_time now = vespalib::steady_clock::now();
+        _metrics.heart_beat_age.set(vespalib::to_s(now - heart_beat_time));
+    } else {
+        _metrics.heart_beat_age.set(0.0);
+    }
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index 1a97972645f..95724237604 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -435,7 +435,9 @@ FeedHandler::FeedHandler(IThreadingService &writeService,
       _bucketDBHandler(nullptr),
       _syncLock(),
       _syncedSerialNum(0),
-      _allowSync(false)
+      _allowSync(false),
+      _heart_beat_time_lock(),
+      _heart_beat_time()
 { }
 
 
@@ -764,6 +766,10 @@ void
 FeedHandler::heartBeat()
 {
     assert(_writeService.master().isCurrentThread());
+    {
+        std::lock_guard guard(_heart_beat_time_lock);
+        _heart_beat_time = vespalib::steady_clock::now();
+    }
     _activeFeedView->heartBeat(_serialNum);
 }
 
@@ -824,4 +830,11 @@ FeedHandler::syncTls(SerialNum syncTo)
     }
 }
 
+vespalib::steady_time
+FeedHandler::get_heart_beat_time() const
+{
+    std::lock_guard guard(_heart_beat_time_lock);
+    return _heart_beat_time;
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
index 6a014c761cc..bdcc4afba99 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
@@ -96,6 +96,8 @@ private:
     std::mutex                             _syncLock;
     SerialNum                              _syncedSerialNum; 
     bool                                   _allowSync; // Sanity check
+    mutable std::mutex                     _heart_beat_time_lock;
+    vespalib::steady_time                  _heart_beat_time;
 
     /**
      * Delayed handling of feed operations, in master write thread.
@@ -245,6 +247,7 @@ public:
     [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override;
     [[nodiscard]] CommitResult storeOperationSync(const FeedOperation & op);
     void considerDelayedPrune();
+    vespalib::steady_time get_heart_beat_time() const;
 };
 
 } // namespace proton