Add metrics that offer insight into how throttling affects scheduling

Adds the following stripe-level metrics:
 * `throttled_rpc_direct_dispatches` - number of times an RPC thread
   could not directly dispatch an async operation directly to Proton
   because it was disallowed by the throttle policy.
 * `throttled_persistence_thread_polls` - number of times a persistence
   thread could not immediately dispatch a queued async operation because
   it was disallowed by the throttle policy.
 * `timeouts_waiting_for_throttle_token` - number of times a persistence
   thread timed out waiting for an available throttle policy token.

3 files changed, 21 insertions, 3 deletions

diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
index 5e0ea0359dc..2ccbc7a85ef 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
@@ -940,7 +940,10 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout)
             } else if (should_throttle_op && !throttle_token.valid()) {
                 // Important: _non-blocking_ attempt at getting a throttle token.
                 throttle_token = _owner.operation_throttler().try_acquire_one();
-                was_throttled = !throttle_token.valid();
+                if (!throttle_token.valid()) {
+                    was_throttled = true;
+                    _metrics->throttled_persistence_thread_polls.inc();
+                }
             }
             if (!should_throttle_op || throttle_token.valid()) {
                 return getMessage(guard, idx, iter, std::move(throttle_token));
@@ -956,10 +959,11 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout)
                 // prevents RPC threads from pushing onto the queue.
                 guard.unlock();
                 throttle_token = _owner.operation_throttler().blocking_acquire_one(timeout);
+                guard.lock();
                 if (!throttle_token.valid()) {
+                    _metrics->timeouts_waiting_for_throttle_token.inc();
                     return {}; // Already exhausted our timeout window.
                 }
-                guard.lock();
             }
         }
     }
@@ -984,6 +988,8 @@ FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard)
         auto throttle_token = _owner.operation_throttler().try_acquire_one();
         if (throttle_token.valid()) {
             return getMessage(guard, idx, iter, std::move(throttle_token));
+        } else {
+            _metrics->throttled_rpc_direct_dispatches.inc();
         }
     }
     return {};
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp
index a98077da57a..6cb76c32997 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp
@@ -190,7 +190,16 @@ FileStorThreadMetrics::~FileStorThreadMetrics() = default;
 
 FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std::string& description)
     : MetricSet(name, {{"partofsum"}}, description),
-      averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this)
+      averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this),
+      throttled_rpc_direct_dispatches("throttled_rpc_direct_dispatches", {},
+                                      "Number of times an RPC thread could not directly dispatch an async operation "
+                                      "directly to Proton because it was disallowed by the throttle policy", this),
+      throttled_persistence_thread_polls("throttled_persistence_thread_polls", {},
+                                         "Number of times a persistence thread could not immediately dispatch a "
+                                         "queued async operation because it was disallowed by the throttle policy", this),
+      timeouts_waiting_for_throttle_token("timeouts_waiting_for_throttle_token", {},
+                                          "Number of times a persistence thread timed out waiting for an available "
+                                          "throttle policy token", this)
 {
 }
 
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h
index d8135c9aeca..1bcbacd2ca6 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h
@@ -131,6 +131,9 @@ class FileStorStripeMetrics : public metrics::MetricSet
 public:
     using SP = std::shared_ptr<FileStorStripeMetrics>;
     metrics::DoubleAverageMetric averageQueueWaitingTime;
+    metrics::LongCountMetric throttled_rpc_direct_dispatches;
+    metrics::LongCountMetric throttled_persistence_thread_polls;
+    metrics::LongCountMetric timeouts_waiting_for_throttle_token;
     FileStorStripeMetrics(const std::string& name, const std::string& description);
     ~FileStorStripeMetrics() override;
 };