Wire Prometheus metric export to state V1 APIs

Extends metric producer classes with the requested exposition format.
As a consequence, the State API server has been changed to allow
emitting other content types than just `application/json`.

Add custom Prometheus rendering for Slobrok, as it does its own
domain-specific metric tracking. However, since it has non-destructive
sampling properties, we can actually use proper `counter` types.

10 files changed, 168 insertions, 70 deletions

diff --git a/vespalib/src/tests/state_server/state_server_test.cpp b/vespalib/src/tests/state_server/state_server_test.cpp
index 2922a6d5069..6c248b54cc8 100644
--- a/vespalib/src/tests/state_server/state_server_test.cpp
+++ b/vespalib/src/tests/state_server/state_server_test.cpp
@@ -47,7 +47,8 @@ vespalib::string getPage(int port, const vespalib::string &path, const vespalib:
 
 vespalib::string getFull(int port, const vespalib::string &path) { return getPage(port, path, "-D -"); }
 
-vespalib::string get_json(const JsonGetHandler &handler,
+std::pair<vespalib::string, vespalib::string>
+get_body_and_content_type(const JsonGetHandler &handler,
                           const vespalib::string &host,
                           const vespalib::string &path,
                           const std::map<vespalib::string,vespalib::string> &params)
@@ -55,11 +56,19 @@ vespalib::string get_json(const JsonGetHandler &handler,
     net::ConnectionAuthContext dummy_ctx(net::tls::PeerCredentials(), net::tls::CapabilitySet::all());
     auto res = handler.get(host, path, params, dummy_ctx);
     if (res.ok()) {
-        return res.payload();
+        return {res.payload(), res.content_type()};
     }
     return {};
 }
 
+vespalib::string get_json(const JsonGetHandler &handler,
+                          const vespalib::string &host,
+                          const vespalib::string &path,
+                          const std::map<vespalib::string,vespalib::string> &params)
+{
+    return get_body_and_content_type(handler, host, path, params).first;
+}
+
 //-----------------------------------------------------------------------------
 
 struct DummyHandler : JsonGetHandler {
@@ -208,7 +217,7 @@ TEST_FFFF("require that the state server wires the appropriate url prefixes",
           SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(),
           StateServer(0, f1, f2, f3))
 {
-    f2.setTotalMetrics("{}"); // avoid empty result
+    f2.setTotalMetrics("{}", MetricsProducer::ExpositionFormat::JSON); // avoid empty result
     int port = f4.getListenPort();
     EXPECT_TRUE(getFull(port, short_root_path).find("HTTP/1.1 200 OK") == 0);
     EXPECT_TRUE(getFull(port, total_metrics_path).find("HTTP/1.1 200 OK") == 0);
@@ -282,7 +291,7 @@ TEST_FFFF("require that state api responds to the expected paths",
           SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(),
           StateApi(f1, f2, f3))
 {
-    f2.setTotalMetrics("{}"); // avoid empty result
+    f2.setTotalMetrics("{}", MetricsProducer::ExpositionFormat::JSON); // avoid empty result
     EXPECT_TRUE(!get_json(f4, host_tag, short_root_path, empty_params).empty());
     EXPECT_TRUE(!get_json(f4, host_tag, root_path, empty_params).empty());
     EXPECT_TRUE(!get_json(f4, host_tag, health_path, empty_params).empty());
@@ -340,9 +349,20 @@ TEST_FFFF("require that metrics resource works as expected",
     EXPECT_EQUAL("{\"status\":{\"code\":\"down\",\"message\":\"FAIL MSG\"}}",
                  get_json(f4, host_tag, metrics_path, empty_params));
     f1.setOk();
-    f2.setMetrics("{\"foo\":\"bar\"}");
-    EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":{\"foo\":\"bar\"}}",
-                 get_json(f4, host_tag, metrics_path, empty_params));
+    f2.setMetrics(R"({"foo":"bar"})", MetricsProducer::ExpositionFormat::JSON);
+    f2.setMetrics(R"(cool_stuff{hello="world"} 1 23456)", MetricsProducer::ExpositionFormat::Prometheus);
+
+    auto result = get_body_and_content_type(f4, host_tag, metrics_path, empty_params);
+    EXPECT_EQUAL(R"({"status":{"code":"up"},"metrics":{"foo":"bar"}})", result.first);
+    EXPECT_EQUAL("application/json", result.second);
+
+    result = get_body_and_content_type(f4, host_tag, metrics_path, {{"format", "json"}}); // Explicit JSON
+    EXPECT_EQUAL(R"({"status":{"code":"up"},"metrics":{"foo":"bar"}})", result.first);
+    EXPECT_EQUAL("application/json", result.second);
+
+    result = get_body_and_content_type(f4, host_tag, metrics_path, {{"format", "prometheus"}}); // Explicit Prometheus
+    EXPECT_EQUAL(R"(cool_stuff{hello="world"} 1 23456)", result.first);
+    EXPECT_EQUAL("text/plain; version=0.0.4", result.second);
 }
 
 TEST_FFFF("require that config resource works as expected",
@@ -367,9 +387,12 @@ TEST_FFFF("require that state api also can return total metric",
           SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(),
           StateApi(f1, f2, f3))
 {
-    f2.setTotalMetrics("{\"foo\":\"bar\"}");
-    EXPECT_EQUAL("{\"foo\":\"bar\"}",
+    f2.setTotalMetrics(R"({"foo":"bar"})", MetricsProducer::ExpositionFormat::JSON);
+    f2.setTotalMetrics(R"(cool_stuff{hello="world"} 1 23456)", MetricsProducer::ExpositionFormat::Prometheus);
+    EXPECT_EQUAL(R"({"foo":"bar"})",
                  get_json(f4, host_tag, total_metrics_path, empty_params));
+    EXPECT_EQUAL(R"(cool_stuff{hello="world"} 1 23456)",
+                 get_json(f4, host_tag, total_metrics_path, {{"format", "prometheus"}}));
 }
 
 TEST_FFFFF("require that custom handlers can be added to the state server",
@@ -384,12 +407,25 @@ TEST_FFFFF("require that custom handlers can be added to the state server",
 }
 
 struct EchoConsumer : MetricsProducer {
+    static constexpr const char* to_string(ExpositionFormat format) noexcept {
+        switch (format) {
+        case ExpositionFormat::JSON: return "JSON";
+        case ExpositionFormat::Prometheus: return "Prometheus";
+        }
+        abort();
+    }
+
+    static vespalib::string stringify_params(const vespalib::string &consumer, ExpositionFormat format) {
+        // Not semantically meaningful output if format == Prometheus, but doesn't really matter here.
+        return vespalib::make_string(R"(["%s", "%s"])", to_string(format), consumer.c_str());
+    }
+
     ~EchoConsumer() override;
-    vespalib::string getMetrics(const vespalib::string &consumer) override {
-        return "[\"" + consumer + "\"]";
+    vespalib::string getMetrics(const vespalib::string &consumer, ExpositionFormat format) override {
+        return stringify_params(consumer, format);
     }
-    vespalib::string getTotalMetrics(const vespalib::string &consumer) override {
-        return "[\"" + consumer + "\"]";
+    vespalib::string getTotalMetrics(const vespalib::string &consumer, ExpositionFormat format) override {
+        return stringify_params(consumer, format);
     }
 };
 
@@ -399,17 +435,17 @@ TEST_FFFF("require that empty v1 metrics consumer defaults to 'statereporter'",
           SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(),
           StateApi(f1, f2, f3))
 {
-    std::map<vespalib::string,vespalib::string> my_params;
-    EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"statereporter\"]}",
+    EXPECT_EQUAL(R"({"status":{"code":"up"},"metrics":["JSON", "statereporter"]})",
                  get_json(f4, host_tag, metrics_path, empty_params));
+    EXPECT_EQUAL(R"(["Prometheus", "statereporter"])",
+                 get_json(f4, host_tag, metrics_path, {{"format", "prometheus"}}));
 }
 
 TEST_FFFF("require that empty total metrics consumer defaults to the empty string",
           SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(),
           StateApi(f1, f2, f3))
 {
-    std::map<vespalib::string,vespalib::string> my_params;
-    EXPECT_EQUAL("[\"\"]", get_json(f4, host_tag, total_metrics_path, empty_params));
+    EXPECT_EQUAL(R"(["JSON", ""])", get_json(f4, host_tag, total_metrics_path, empty_params));
 }
 
 TEST_FFFF("require that metrics consumer is passed correctly",
@@ -418,8 +454,10 @@ TEST_FFFF("require that metrics consumer is passed correctly",
 {
     std::map<vespalib::string,vespalib::string> my_params;
     my_params["consumer"] = "ME";
-    EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"ME\"]}", get_json(f4, host_tag, metrics_path, my_params));
-    EXPECT_EQUAL("[\"ME\"]", get_json(f4, host_tag, total_metrics_path, my_params));
+    EXPECT_EQUAL(R"({"status":{"code":"up"},"metrics":["JSON", "ME"]})", get_json(f4, host_tag, metrics_path, my_params));
+    EXPECT_EQUAL(R"(["JSON", "ME"])", get_json(f4, host_tag, total_metrics_path, my_params));
+    my_params["format"] = "prometheus";
+    EXPECT_EQUAL(R"(["Prometheus", "ME"])", get_json(f4, host_tag, total_metrics_path, my_params));
 }
 
 void check_json(const vespalib::string &expect_json, const vespalib::string &actual_json) {
diff --git a/vespalib/src/vespa/vespalib/metrics/producer.cpp b/vespalib/src/vespa/vespalib/metrics/producer.cpp
index fe244607f43..ca6d773e129 100644
--- a/vespalib/src/vespa/vespalib/metrics/producer.cpp
+++ b/vespalib/src/vespa/vespalib/metrics/producer.cpp
@@ -4,15 +4,16 @@
 #include "metrics_manager.h"
 #include "json_formatter.h"
 
-namespace vespalib {
-namespace metrics {
+namespace vespalib::metrics {
 
 Producer::Producer(std::shared_ptr<MetricsManager> m)
-    : _manager(m)
+    : _manager(std::move(m))
 {}
 
+Producer::~Producer() = default;
+
 vespalib::string
-Producer::getMetrics(const vespalib::string &)
+Producer::getMetrics(const vespalib::string &, ExpositionFormat /*ignored*/)
 {
     Snapshot snap = _manager->snapshot();
     JsonFormatter fmt(snap);
@@ -20,14 +21,11 @@ Producer::getMetrics(const vespalib::string &)
 }
 
 vespalib::string
-Producer::getTotalMetrics(const vespalib::string &)
+Producer::getTotalMetrics(const vespalib::string &, ExpositionFormat /*ignored*/)
 {
     Snapshot snap = _manager->totalSnapshot();
     JsonFormatter fmt(snap);
     return fmt.asString();
 }
 
-
-
 } // namespace vespalib::metrics
-} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/metrics/producer.h b/vespalib/src/vespa/vespalib/metrics/producer.h
index b0b3e2bc701..95730258f86 100644
--- a/vespalib/src/vespa/vespalib/metrics/producer.h
+++ b/vespalib/src/vespa/vespalib/metrics/producer.h
@@ -15,9 +15,10 @@ class Producer : public vespalib::MetricsProducer {
 private:
     std::shared_ptr<MetricsManager> _manager;
 public:
-    Producer(std::shared_ptr<MetricsManager> m);
-    vespalib::string getMetrics(const vespalib::string &consumer) override;
-    vespalib::string getTotalMetrics(const vespalib::string &consumer) override;
+    explicit Producer(std::shared_ptr<MetricsManager> m);
+    ~Producer() override;
+    vespalib::string getMetrics(const vespalib::string &consumer, ExpositionFormat format) override;
+    vespalib::string getTotalMetrics(const vespalib::string &consumer, ExpositionFormat format) override;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/http/http_server.cpp b/vespalib/src/vespa/vespalib/net/http/http_server.cpp
index a307a4eca4f..15443276389 100644
--- a/vespalib/src/vespa/vespalib/net/http/http_server.cpp
+++ b/vespalib/src/vespa/vespalib/net/http/http_server.cpp
@@ -13,7 +13,7 @@ HttpServer::get(Portal::GetRequest req)
     if (response.failed()) {
         req.respond_with_error(response.status_code(), response.status_message());
     } else {
-        req.respond_with_content("application/json", response.payload());
+        req.respond_with_content(response.content_type(), response.payload());
     }
 }
 
diff --git a/vespalib/src/vespa/vespalib/net/http/json_get_handler.cpp b/vespalib/src/vespa/vespalib/net/http/json_get_handler.cpp
index c9d7859b5b4..7f04235f781 100644
--- a/vespalib/src/vespa/vespalib/net/http/json_get_handler.cpp
+++ b/vespalib/src/vespa/vespalib/net/http/json_get_handler.cpp
@@ -4,14 +4,18 @@
 
 namespace vespalib {
 
-JsonGetHandler::Response::Response(int status_code, vespalib::string status_or_payload)
+JsonGetHandler::Response::Response(int status_code,
+                                   vespalib::string status_or_payload,
+                                   vespalib::string content_type_override)
     : _status_code(status_code),
-      _status_or_payload(std::move(status_or_payload))
+      _status_or_payload(std::move(status_or_payload)),
+      _content_type_override(std::move(content_type_override))
 {}
 
 JsonGetHandler::Response::Response()
     : _status_code(500),
-      _status_or_payload("Internal Server Error")
+      _status_or_payload("Internal Server Error"),
+      _content_type_override()
 {}
 
 JsonGetHandler::Response::~Response() = default;
@@ -24,19 +28,25 @@ JsonGetHandler::Response& JsonGetHandler::Response::operator=(Response&&) noexce
 JsonGetHandler::Response
 JsonGetHandler::Response::make_ok_with_json(vespalib::string json)
 {
-    return {200, std::move(json)};
+    return {200, std::move(json), {}};
+}
+
+JsonGetHandler::Response
+JsonGetHandler::Response::make_ok_with_content_type(vespalib::string payload, vespalib::string content_type)
+{
+    return {200, std::move(payload), std::move(content_type)};
 }
 
 JsonGetHandler::Response
 JsonGetHandler::Response::make_failure(int status_code, vespalib::string status_message)
 {
-    return {status_code, std::move(status_message)};
+    return {status_code, std::move(status_message), {}};
 }
 
 JsonGetHandler::Response
 JsonGetHandler::Response::make_not_found()
 {
-    return {404, "Not Found"};
+    return {404, "Not Found", {}};
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/http/json_get_handler.h b/vespalib/src/vespa/vespalib/net/http/json_get_handler.h
index b7786ddd119..43793dbf1d8 100644
--- a/vespalib/src/vespa/vespalib/net/http/json_get_handler.h
+++ b/vespalib/src/vespa/vespalib/net/http/json_get_handler.h
@@ -13,8 +13,11 @@ struct JsonGetHandler {
     class Response {
         int              _status_code;
         vespalib::string _status_or_payload;
+        vespalib::string _content_type_override;
 
-        Response(int status_code, vespalib::string status_or_payload);
+        Response(int status_code,
+                 vespalib::string status_or_payload,
+                 vespalib::string content_type_override);
     public:
         Response(); // By default, 500 Internal Server Error
         ~Response();
@@ -40,8 +43,16 @@ struct JsonGetHandler {
                 return {};
             }
         }
+        [[nodiscard]] vespalib::stringref content_type() const noexcept {
+            if (_content_type_override.empty()) {
+                return "application/json";
+            } else {
+                return _content_type_override;
+            }
+        }
 
         [[nodiscard]] static Response make_ok_with_json(vespalib::string json);
+        [[nodiscard]] static Response make_ok_with_content_type(vespalib::string payload, vespalib::string content_type);
         [[nodiscard]] static Response make_failure(int status_code, vespalib::string status_message);
         [[nodiscard]] static Response make_not_found();
     };
diff --git a/vespalib/src/vespa/vespalib/net/http/metrics_producer.h b/vespalib/src/vespa/vespalib/net/http/metrics_producer.h
index 18e61ff05e3..0ffb1773456 100644
--- a/vespalib/src/vespa/vespalib/net/http/metrics_producer.h
+++ b/vespalib/src/vespa/vespalib/net/http/metrics_producer.h
@@ -7,8 +7,13 @@
 namespace vespalib {
 
 struct MetricsProducer {
-    virtual vespalib::string getMetrics(const vespalib::string &consumer) = 0;
-    virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) = 0;
+    enum class ExpositionFormat {
+        JSON,
+        Prometheus
+    };
+
+    virtual vespalib::string getMetrics(const vespalib::string &consumer, ExpositionFormat format) = 0;
+    virtual vespalib::string getTotalMetrics(const vespalib::string &consumer, ExpositionFormat format) = 0;
     virtual ~MetricsProducer() = default;
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp
index 5fbddd8d1b2..52836589ce3 100644
--- a/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp
+++ b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp
@@ -7,38 +7,38 @@ namespace vespalib {
 SimpleMetricsProducer::SimpleMetricsProducer()
     : _lock(),
       _metrics(),
-      _totalMetrics()
+      _total_metrics()
 {
 }
 
 SimpleMetricsProducer::~SimpleMetricsProducer() = default;
 
 void
-SimpleMetricsProducer::setMetrics(const vespalib::string &metrics)
+SimpleMetricsProducer::setMetrics(const vespalib::string &metrics, ExpositionFormat format)
 {
     std::lock_guard guard(_lock);
-    _metrics = metrics;
+    _metrics[format] = metrics;
 }
 
 vespalib::string
-SimpleMetricsProducer::getMetrics(const vespalib::string &)
+SimpleMetricsProducer::getMetrics(const vespalib::string &, ExpositionFormat format)
 {
     std::lock_guard guard(_lock);
-    return _metrics;
+    return _metrics[format]; // May implicitly create entry, but that's fine here.
 }
 
 void
-SimpleMetricsProducer::setTotalMetrics(const vespalib::string &metrics)
+SimpleMetricsProducer::setTotalMetrics(const vespalib::string &metrics, ExpositionFormat format)
 {
     std::lock_guard guard(_lock);
-    _totalMetrics = metrics;
+    _total_metrics[format] = metrics;
 }
 
 vespalib::string
-SimpleMetricsProducer::getTotalMetrics(const vespalib::string &)
+SimpleMetricsProducer::getTotalMetrics(const vespalib::string &, ExpositionFormat format)
 {
     std::lock_guard guard(_lock);
-    return _totalMetrics;
+    return _total_metrics[format];
 }
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h
index bebf357492c..670e8d494c2 100644
--- a/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h
+++ b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "metrics_producer.h"
+#include <map>
 #include <mutex>
 
 namespace vespalib {
@@ -11,16 +12,16 @@ class SimpleMetricsProducer : public MetricsProducer
 {
 private:
     std::mutex _lock;
-    vespalib::string _metrics;
-    vespalib::string _totalMetrics;
+    std::map<ExpositionFormat, vespalib::string> _metrics;
+    std::map<ExpositionFormat, vespalib::string> _total_metrics;
 
 public:
     SimpleMetricsProducer();
     ~SimpleMetricsProducer() override;
-    void setMetrics(const vespalib::string &metrics);
-    vespalib::string getMetrics(const vespalib::string &consumer) override;
-    void setTotalMetrics(const vespalib::string &metrics);
-    vespalib::string getTotalMetrics(const vespalib::string &consumer) override;
+    void setMetrics(const vespalib::string &metrics, ExpositionFormat format);
+    vespalib::string getMetrics(const vespalib::string &consumer, ExpositionFormat format) override;
+    void setTotalMetrics(const vespalib::string &metrics, ExpositionFormat format);
+    vespalib::string getTotalMetrics(const vespalib::string &consumer, ExpositionFormat format) override;
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/http/state_api.cpp b/vespalib/src/vespa/vespalib/net/http/state_api.cpp
index 1b233e4cdbc..31d0010d72d 100644
--- a/vespalib/src/vespa/vespalib/net/http/state_api.cpp
+++ b/vespalib/src/vespa/vespalib/net/http/state_api.cpp
@@ -58,14 +58,15 @@ void build_health_status(JSONStringer &json, const HealthProducer &healthProduce
     json.endObject();
 }
 
-vespalib::string get_consumer(const std::map<vespalib::string,vespalib::string> &params,
-                              vespalib::stringref default_consumer)
+vespalib::string get_param(const std::map<vespalib::string,vespalib::string> &params,
+                           vespalib::stringref param_name,
+                           vespalib::stringref default_value)
 {
-    auto consumer_lookup = params.find("consumer");
-    if (consumer_lookup == params.end()) {
-        return default_consumer;
+    auto maybe_value = params.find(param_name);
+    if (maybe_value == params.end()) {
+        return default_value;
     }
-    return consumer_lookup->second;
+    return maybe_value->second;
 }
 
 void render_link(JSONStringer &json, const vespalib::string &host, const vespalib::string &path) {
@@ -99,15 +100,15 @@ vespalib::string respond_health(const HealthProducer &healthProducer) {
     return json.toString();
 }
 
-vespalib::string respond_metrics(const vespalib::string &consumer,
-                                 const HealthProducer &healthProducer,
-                                 MetricsProducer &metricsProducer)
+vespalib::string respond_json_metrics(const vespalib::string &consumer,
+                                      const HealthProducer &healthProducer,
+                                      MetricsProducer &metricsProducer)
 {
     JSONStringer json;
     json.beginObject();
     build_health_status(json, healthProducer);
     { // metrics
-        vespalib::string metrics = metricsProducer.getMetrics(consumer);
+        vespalib::string metrics = metricsProducer.getMetrics(consumer, MetricsProducer::ExpositionFormat::JSON);
         if (!metrics.empty()) {
             json.appendKey("metrics");
             json.appendJSON(metrics);
@@ -117,6 +118,22 @@ vespalib::string respond_metrics(const vespalib::string &consumer,
     return json.toString();
 }
 
+JsonGetHandler::Response cap_check_and_respond_metrics(
+        const net::ConnectionAuthContext &auth_ctx,
+        const std::map<vespalib::string,vespalib::string> &params,
+        const vespalib::string& default_consumer,
+        std::function<JsonGetHandler::Response(const vespalib::string&, MetricsProducer::ExpositionFormat)> response_fn)
+{
+    if (!auth_ctx.capabilities().contains(Capability::content_metrics_api())) {
+        return JsonGetHandler::Response::make_failure(403, "Forbidden");
+    }
+    auto consumer   = get_param(params, "consumer", default_consumer);
+    auto format_str = get_param(params, "format", "json");
+    auto format     = (format_str == "prometheus" ? MetricsProducer::ExpositionFormat::Prometheus
+                                                  : MetricsProducer::ExpositionFormat::JSON);
+    return response_fn(consumer, format);
+}
+
 vespalib::string respond_config(ComponentConfigProducer &componentConfigProducer) {
     JSONStringer json;
     json.beginObject();
@@ -154,6 +171,10 @@ JsonGetHandler::Response cap_checked(const net::ConnectionAuthContext &auth_ctx,
     return cap_checked(auth_ctx, CapabilitySet::of({required_cap}), std::move(fn));
 }
 
+constexpr const char* prometheus_content_type() noexcept {
+    return "text/plain; version=0.0.4";
+}
+
 } // namespace vespalib::<unnamed>
 
 JsonGetHandler::Response
@@ -172,17 +193,30 @@ StateApi::get(const vespalib::string &host,
         });
     } else if (path == "/state/v1/metrics") {
         // Using a 'statereporter' consumer by default removes many uninteresting per-thread
-        // metrics but retains their aggregates.
-        return cap_checked(auth_ctx, Capability::content_metrics_api(), [&] {
-            return respond_metrics(get_consumer(params, "statereporter"), _healthProducer, _metricsProducer);
+        // metrics but retains their aggregates (side note: per-thread metrics are NOT included
+        // in Prometheus metrics regardless of the specified consumer).
+        return cap_check_and_respond_metrics(auth_ctx, params, "statereporter", [&](auto& consumer, auto format) {
+            if (format == MetricsProducer::ExpositionFormat::Prometheus) {
+                auto metrics_text = _metricsProducer.getMetrics(consumer, MetricsProducer::ExpositionFormat::Prometheus);
+                return JsonGetHandler::Response::make_ok_with_content_type(std::move(metrics_text), prometheus_content_type());
+            } else {
+                auto json = respond_json_metrics(consumer, _healthProducer, _metricsProducer);
+                return JsonGetHandler::Response::make_ok_with_json(std::move(json));
+            }
         });
     } else if (path == "/state/v1/config") {
         return cap_checked(auth_ctx, Capability::content_state_api(), [&] {
             return respond_config(_componentConfigProducer);
         });
     } else if (path == "/metrics/total") {
-        return cap_checked(auth_ctx, Capability::content_metrics_api(), [&] {
-            return _metricsProducer.getTotalMetrics(get_consumer(params, ""));
+        return cap_check_and_respond_metrics(auth_ctx, params, "", [&](auto& consumer, auto format) {
+            if (format == MetricsProducer::ExpositionFormat::Prometheus) {
+                auto metrics_text = _metricsProducer.getTotalMetrics(consumer, MetricsProducer::ExpositionFormat::Prometheus);
+                return JsonGetHandler::Response::make_ok_with_content_type(std::move(metrics_text), prometheus_content_type());
+            } else {
+                auto json = _metricsProducer.getTotalMetrics(consumer, vespalib::MetricsProducer::ExpositionFormat::JSON);
+                return JsonGetHandler::Response::make_ok_with_json(std::move(json));
+            }
         });
     } else {
         // Assume this is for the nested state v1 stuff; may delegate capability check to handler later if desired.