Add metrics around bucket DB pruning and merging phases of state transitions

3 files changed, 21 insertions, 3 deletions

diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
index e9595b4a960..7b2b2561f91 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
@@ -234,7 +234,9 @@ BucketDBUpdater::onSetSystemState(
         return false;
     }
     ensureTransitionTimerStarted();
-
+    // Separate timer since _transitionTimer might span multiple pending states.
+    framework::MilliSecTimer process_timer(_distributorComponent.getClock());
+    
     removeSuperfluousBuckets(cmd->getClusterStateBundle());
     replyToPreviousPendingClusterStateIfAny();
 
@@ -255,6 +257,9 @@ BucketDBUpdater::onSetSystemState(
             _distributorComponent.getUniqueTimestamp());
     _outdatedNodesMap = _pendingClusterState->getOutdatedNodesMap();
 
+    _distributorComponent.getDistributor().getMetrics().set_cluster_state_processing_time.addValue(
+            process_timer.getElapsedTimeAsDouble());
+
     if (isPendingClusterStateCompleted()) {
         processCompletedPendingClusterState();
     }
@@ -579,6 +584,8 @@ BucketDBUpdater::processCompletedPendingClusterState()
 void
 BucketDBUpdater::activatePendingClusterState()
 {
+    framework::MilliSecTimer process_timer(_distributorComponent.getClock());
+
     _pendingClusterState->mergeIntoBucketDatabases();
 
     if (_pendingClusterState->isVersionedTransition()) {
@@ -601,6 +608,9 @@ BucketDBUpdater::activatePendingClusterState()
     sendAllQueuedBucketRechecks();
     completeTransitionTimer();
     clearReadOnlyBucketRepoDatabases();
+
+    _distributorComponent.getDistributor().getMetrics().activate_cluster_state_processing_time.addValue(
+            process_timer.getElapsedTimeAsDouble());
 }
 
 void
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
index 83923a1f00e..244406ca6fb 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
@@ -24,6 +24,12 @@ DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt)
               "state transition is preempted before completing, its elapsed "
               "time is counted as part of the total time spent for the final, "
               "completed state transition", this),
+      set_cluster_state_processing_time("set_cluster_state_processing_time", {},
+              "Elapsed time where the distributor thread is blocked on processing "
+              "its bucket database upon receiving a new cluster state", this),
+      activate_cluster_state_processing_time("activate_cluster_state_processing_time", {},
+              "Elapsed time where the distributor thread is blocked on merging pending "
+              "bucket info into its bucket database upon activating a cluster state", this),
       recoveryModeTime("recoverymodeschedulingtime", {},
               "Time spent scheduling operations in recovery mode "
               "after receiving new cluster state", this),
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h
index dfe976a89ab..1e4730b8de6 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.h
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.h
@@ -23,12 +23,14 @@ public:
     metrics::LoadMetric<PersistenceOperationMetricSet> getbucketlists;
     metrics::LoadMetric<VisitorMetricSet> visits;
     metrics::DoubleAverageMetric stateTransitionTime;
+    metrics::DoubleAverageMetric set_cluster_state_processing_time;
+    metrics::DoubleAverageMetric activate_cluster_state_processing_time;
     metrics::DoubleAverageMetric recoveryModeTime;
     metrics::LongValueMetric docsStored;
     metrics::LongValueMetric bytesStored;
 
-    DistributorMetricSet(const metrics::LoadTypeSet& lt);
-    ~DistributorMetricSet();
+    explicit DistributorMetricSet(const metrics::LoadTypeSet& lt);
+    ~DistributorMetricSet() override;
 };
 
 }