From 9bad60ef6d692745fbbf98338dfb17751f47dac3 Mon Sep 17 00:00:00 2001
From: Tor Brede Vekterli <vekterli@yahooinc.com>
Date: Fri, 17 Feb 2023 10:22:38 +0000
Subject: Add metrics tracking failed RPC and status page capability checks

---
 .../vespa/storage/frameworkimpl/status/statuswebserver.cpp |  2 ++
 .../storageserver/tls_statistics_metrics_wrapper.cpp       | 14 +++++++++++++-
 .../storage/storageserver/tls_statistics_metrics_wrapper.h |  4 ++++
 3 files changed, 19 insertions(+), 1 deletion(-)

(limited to 'storage')

diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
index 8690f6e122d..0b4e32d637d 100644
--- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
+++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
@@ -10,6 +10,7 @@
 #include <vespa/vespalib/component/vtag.h>
 #include <vespa/vespalib/net/connection_auth_context.h>
 #include <vespa/vespalib/net/crypto_engine.h>
+#include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/config/subscription/configuri.h>
 #include <vespa/config/helper/configfetcher.hpp>
 #include <functional>
@@ -203,6 +204,7 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, vespalib::Por
             if (auth_ctx.capabilities().contains_all(reporter->required_capabilities())) {
                 invoke_reporter(*reporter, urlpath, request);
             } else {
+                vespalib::net::tls::CapabilityStatistics::get().inc_status_capability_checks_failed();
                 // TODO should print peer address as well; not currently exposed
                 LOG(warning, "Peer with %s denied status page access to '%s' due to insufficient "
                              "credentials (had %s, needed %s)",
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
index 5e281152b2b..ad74e020a82 100644
--- a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp
@@ -27,9 +27,14 @@ TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* own
               "connections broken due to failures during frame encoding or decoding", this),
       failed_tls_config_reloads("failed-tls-config-reloads", {}, "Number of times "
               "background reloading of TLS config has failed", this),
+      rpc_capability_checks_failed("rpc-capability-checks-failed", {},
+              "Number of RPC operations that failed to due one or more missing capabilities", this),
+      status_capability_checks_failed("status-capability-checks-failed", {},
+              "Number of status page operations that failed to due one or more missing capabilities", this),
       last_client_stats_snapshot(),
       last_server_stats_snapshot(),
-      last_config_stats_snapshot()
+      last_config_stats_snapshot(),
+      last_capability_stats_snapshot()
 {}
 
 TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default;
@@ -60,9 +65,16 @@ void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() {
 
     failed_tls_config_reloads.set(config_delta.failed_config_reloads);
 
+    auto capability_current = vespalib::net::tls::CapabilityStatistics::get().snapshot();
+    auto capability_delta = capability_current.subtract(last_capability_stats_snapshot);
+
+    rpc_capability_checks_failed.set(capability_delta.rpc_capability_checks_failed);
+    status_capability_checks_failed.set(capability_delta.status_capability_checks_failed);
+
     last_server_stats_snapshot = server_current;
     last_client_stats_snapshot = client_current;
     last_config_stats_snapshot = config_current;
+    last_capability_stats_snapshot = capability_current;
 }
 
 }
diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
index 7bb51acd1fe..daf02b53b7a 100644
--- a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
+++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h
@@ -29,9 +29,13 @@ class TlsStatisticsMetricsWrapper : public metrics::MetricSet {
 
     metrics::LongCountMetric failed_tls_config_reloads;
 
+    metrics::LongCountMetric rpc_capability_checks_failed;
+    metrics::LongCountMetric status_capability_checks_failed;
+
     vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot;
     vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot;
     vespalib::net::tls::ConfigStatistics::Snapshot     last_config_stats_snapshot;
+    vespalib::net::tls::CapabilityStatistics::Snapshot last_capability_stats_snapshot;
 
 public:
     explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner);
-- 
cgit v1.2.3