Support internal metric rendering in Prometheus text format in C++

Maps all internal metrics to one or more labelled time series.
Due to poor compatibility between the data model (and sampling
strategy) of the legacy metrics framework and that of Prometheus,
all time series are emitted as `untyped` metrics.

This is a stop-gap solution on the way to "properly" supporting
Prometheus exposition, and the output of this renderer should
therefore only be used for internal purposes.

6 files changed, 531 insertions, 31 deletions

diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp
index be63bed5bec..e5e39bd9dcb 100644
--- a/metrics/src/tests/metricmanagertest.cpp
+++ b/metrics/src/tests/metricmanagertest.cpp
@@ -5,19 +5,21 @@
 #include <vespa/metrics/metricmanager.h>
 #include <vespa/metrics/state_api_adapter.h>
 #include <vespa/metrics/textwriter.h>
+#include <vespa/metrics/prometheus_writer.h>
 #include <vespa/vespalib/data/slime/slime.h>
-#include <vespa/vespalib/gtest/gtest.h>
 #include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/util/time.h>
 #include <vespa/vespalib/data/simple_buffer.h>
 #include <vespa/vespalib/util/atomic.h>
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
 #include <mutex>
 #include <thread>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".test.metricmanager");
 
+using namespace ::testing;
 using namespace vespalib::atomic;
 using config::ConfigUri;
 
@@ -41,16 +43,16 @@ struct SubMetricSet : public MetricSet
     DoubleValueMetric val2;
     SumMetric<DoubleValueMetric> valsum;
 
-    SubMetricSet(const Metric::String & name, MetricSet* owner);
-    ~SubMetricSet();
+    explicit SubMetricSet(const Metric::String& name, MetricSet* owner = nullptr);
+    ~SubMetricSet() override;
 };
 
 
-SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner)
-    : MetricSet(name, {{"sub"}}, "sub desc", owner),
-      val1("val1", {{"tag4"},{"snaptest"}}, "val1 desc", this),
+SubMetricSet::SubMetricSet(const Metric::String& name, MetricSet* owner)
+    : MetricSet(name, {{"partofsum"}, {"sub"}}, "sub desc", owner),
+      val1("val1", {{"tag4"}, {"snaptest"}}, "val1 desc", this),
       val2("val2", {{"tag5"}}, "val2 desc", this),
-      valsum("valsum", {{"tag4"},{"snaptest"}}, "valsum desc", this)
+      valsum("valsum", {{"tag4"}, {"snaptest"}}, "valsum desc", this)
 {
     valsum.addMetricToSum(val1);
     valsum.addMetricToSum(val2);
@@ -65,7 +67,7 @@ struct MultiSubMetricSet
     SubMetricSet b;
     SumMetric<MetricSet> sum;
 
-    MultiSubMetricSet(MetricSet* owner);
+    explicit MultiSubMetricSet(MetricSet* owner);
     ~MultiSubMetricSet();
 };
 
@@ -156,13 +158,6 @@ getMatchedMetrics(const vespalib::string& config)
     mm.init(ConfigUri(config));
     MetricNameVisitor visitor;
 
-    /** Take a copy to verify clone works.
-    std::list<Metric::SP> ownerList;
-    MetricSet::UP copy(dynamic_cast<MetricSet*>(
-                mm.getMetrics().clone(ownerList)));
-    mm.visit(*copy, visitor, "consumer");
-    */
-
     MetricLockGuard g(mm.getMetricLock());
     mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer");
     const MetricManager::ConsumerSpec * consumerSpec = mm.getConsumerSpec(g, "consumer");
@@ -386,8 +381,8 @@ struct BriefValuePrinter : public MetricVisitor {
 
 bool waitForTimeProcessed(const MetricManager& mm, time_point::duration processtime, uint32_t timeout = 120)
 {
-    uint32_t lastchance = time(0) + timeout;
-    while (time(0) < lastchance) {
+    uint32_t lastchance = time(nullptr) + timeout;
+    while (time(nullptr) < lastchance) {
         if (mm.getLastProcessedTime() >= time_point(processtime)) return true;
         mm.timeChangedNotification();
         std::this_thread::sleep_for(10ms);
@@ -705,6 +700,16 @@ struct MetricSnapshotTestFixture
         }
         return ss.str();
     }
+
+    std::string render_last_snapshot_as_prometheus() const {
+        vespalib::asciistream os;
+        PrometheusWriter writer(os);
+        {
+            MetricLockGuard lockGuard(manager.getMetricLock());
+            manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300s, false), writer, "snapper");
+        }
+        return os.str();
+    }
 };
 
 class JsonMetricWrapper
@@ -762,7 +767,7 @@ struct DimensionTestMetricSet : MetricSet
     DoubleValueMetric val1;
     LongCountMetric val2;
 
-    DimensionTestMetricSet(MetricSet* owner = nullptr);
+    explicit DimensionTestMetricSet(MetricSet* owner = nullptr);
     ~DimensionTestMetricSet() override;
 };
 
@@ -801,7 +806,7 @@ struct NestedDimensionTestMetricSet : MetricSet
     DimensionTestMetricSet nestedSet;
 
     NestedDimensionTestMetricSet();
-    ~NestedDimensionTestMetricSet();
+    ~NestedDimensionTestMetricSet() override;
 };
 
 NestedDimensionTestMetricSet::NestedDimensionTestMetricSet()
@@ -839,7 +844,7 @@ struct DimensionOverridableTestMetricSet : MetricSet
 {
     DoubleValueMetric val;
 
-    DimensionOverridableTestMetricSet(const std::string& dimValue, MetricSet* owner = nullptr);
+    explicit DimensionOverridableTestMetricSet(const std::string& dimValue, MetricSet* owner = nullptr);
     ~DimensionOverridableTestMetricSet() override;
 };
 
@@ -855,7 +860,7 @@ struct SameNamesTestMetricSet : MetricSet
     DimensionOverridableTestMetricSet set2;
 
     SameNamesTestMetricSet();
-    ~SameNamesTestMetricSet();
+    ~SameNamesTestMetricSet() override;
 };
 
 SameNamesTestMetricSet::SameNamesTestMetricSet()
@@ -894,14 +899,14 @@ TEST_F(MetricManagerTest, test_text_output)
         MetricLockGuard lockGuard(mm.getMetricLock());
         mm.registerMetric(lockGuard, mySet.set);
     }
-        // Adding metrics to have some values in them
+    // Adding metrics to have some values in them
     mySet.val6.addValue(2);
     mySet.val9.val1.addValue(4);
     mySet.val10.count.inc();
     mySet.val10.a.val1.addValue(7);
     mySet.val10.a.val2.addValue(2);
     mySet.val10.b.val1.addValue(1);
-        // Initialize metric manager to get snapshots created.
+    // Initialize metric manager to get snapshots created.
     mm.init(ConfigUri("raw:"
                       "consumer[2]\n"
                       "consumer[0].name snapper\n"
@@ -953,6 +958,137 @@ TEST_F(MetricManagerTest, text_output_supports_dimensions)
     EXPECT_EQ(expected, actual);
 }
 
+TEST_F(MetricManagerTest, prometheus_output_groups_related_time_series) {
+    SameNamesTestMetricSet mset;
+    MetricSnapshotTestFixture fixture(*this, mset);
+
+    mset.set1.val.addValue(2);
+    mset.set1.val.addValue(3);
+    mset.set2.val.addValue(5);
+    mset.set2.val.addValue(7);
+
+    fixture.takeSnapshotsOnce();
+    std::string actual = fixture.render_last_snapshot_as_prometheus();
+    std::string expected(R"(# NOTE: THIS API IS NOT INTENDED FOR PUBLIC USE
+# TYPE outer_temp_val_count untyped
+outer_temp_val_count{foo="bar",fancy="stuff"} 2 1300000
+outer_temp_val_count{foo="baz",fancy="stuff"} 2 1300000
+# TYPE outer_temp_val_max untyped
+outer_temp_val_max{foo="bar",fancy="stuff"} 3 1300000
+outer_temp_val_max{foo="baz",fancy="stuff"} 7 1300000
+# TYPE outer_temp_val_min untyped
+outer_temp_val_min{foo="bar",fancy="stuff"} 2 1300000
+outer_temp_val_min{foo="baz",fancy="stuff"} 5 1300000
+# TYPE outer_temp_val_sum untyped
+outer_temp_val_sum{foo="bar",fancy="stuff"} 5 1300000
+outer_temp_val_sum{foo="baz",fancy="stuff"} 12 1300000
+)");
+    EXPECT_EQ(expected, actual);
+}
+
+struct MetricSetWrapper : MetricSet {
+    MultiSubMetricSet sub;
+
+    MetricSetWrapper();
+    ~MetricSetWrapper() override;
+};
+
+MetricSetWrapper::MetricSetWrapper()
+    : MetricSet("top_level", {}, "stuff and junk", nullptr),
+      sub(this)
+{
+}
+
+MetricSetWrapper::~MetricSetWrapper() = default;
+
+TEST_F(MetricManagerTest, prometheus_output_only_emits_sum_metric_aggregate_values) {
+    MetricSetWrapper mset;
+    MetricSnapshotTestFixture fixture(*this, mset);
+
+    mset.sub.a.val1.addValue(21);
+    mset.sub.a.val2.addValue(17);
+    mset.sub.b.val1.addValue(7);
+    mset.sub.b.val2.addValue(3);
+
+    fixture.takeSnapshotsOnce();
+    std::string actual = fixture.render_last_snapshot_as_prometheus();
+    std::string expected = R"(# NOTE: THIS API IS NOT INTENDED FOR PUBLIC USE
+# TYPE top_level_multisub_sum_val1_count untyped
+top_level_multisub_sum_val1_count 2 1300000
+# TYPE top_level_multisub_sum_val1_max untyped
+top_level_multisub_sum_val1_max 21 1300000
+# TYPE top_level_multisub_sum_val1_min untyped
+top_level_multisub_sum_val1_min 7 1300000
+# TYPE top_level_multisub_sum_val1_sum untyped
+top_level_multisub_sum_val1_sum 56 1300000
+# TYPE top_level_multisub_sum_val2_count untyped
+top_level_multisub_sum_val2_count 2 1300000
+# TYPE top_level_multisub_sum_val2_max untyped
+top_level_multisub_sum_val2_max 17 1300000
+# TYPE top_level_multisub_sum_val2_min untyped
+top_level_multisub_sum_val2_min 3 1300000
+# TYPE top_level_multisub_sum_val2_sum untyped
+top_level_multisub_sum_val2_sum 40 1300000
+# TYPE top_level_multisub_sum_valsum_count untyped
+top_level_multisub_sum_valsum_count 4 1300000
+# TYPE top_level_multisub_sum_valsum_max untyped
+top_level_multisub_sum_valsum_max 21 1300000
+# TYPE top_level_multisub_sum_valsum_min untyped
+top_level_multisub_sum_valsum_min 3 1300000
+# TYPE top_level_multisub_sum_valsum_sum untyped
+top_level_multisub_sum_valsum_sum 192 1300000
+)";
+    EXPECT_EQ(expected, actual);
+}
+
+TEST_F(MetricManagerTest, prometheus_output_can_emit_inf_values_verbatim) {
+    SameNamesTestMetricSet mset;
+    MetricSnapshotTestFixture fixture(*this, mset);
+
+    // We have explicit guards against setting Inf/NaN directly, so we have to fudge the numbers
+    // a bit to get +/- Inf by saturating additions towards infinity. TODO how to test NaN...? :o
+    mset.set1.val.addValue(std::numeric_limits<double>::max());
+    mset.set1.val.addValue(std::numeric_limits<double>::max());
+    mset.set2.val.addValue(std::numeric_limits<double>::lowest());
+    mset.set2.val.addValue(std::numeric_limits<double>::lowest());
+
+    fixture.takeSnapshotsOnce();
+    std::string actual = fixture.render_last_snapshot_as_prometheus();
+    EXPECT_THAT(actual, HasSubstr("outer_temp_val_sum{foo=\"bar\",fancy=\"stuff\"} +Inf 1300000\n"));
+    EXPECT_THAT(actual, HasSubstr("outer_temp_val_sum{foo=\"baz\",fancy=\"stuff\"} -Inf 1300000\n"));
+}
+
+struct SneakyNamesMetricSet : public MetricSet {
+    DoubleValueMetric val1;
+    DoubleValueMetric val2;
+
+    SneakyNamesMetricSet();
+    ~SneakyNamesMetricSet() override;
+};
+
+
+SneakyNamesMetricSet::SneakyNamesMetricSet()
+    : MetricSet("sneaky/path", {}, "sub desc", nullptr),
+      val1("a.name", {{"foo.bar", "blah\nbaz\"zoid\\"}}, "", this),
+      val2("another-name", {}, "", this)
+{
+}
+
+SneakyNamesMetricSet::~SneakyNamesMetricSet() = default;
+
+TEST_F(MetricManagerTest, prometheus_output_normalizes_and_escapes_names_and_labels) {
+    SneakyNamesMetricSet mset;
+    MetricSnapshotTestFixture fixture(*this, mset);
+
+    mset.val1.addValue(123);
+    mset.val2.addValue(42);
+
+    fixture.takeSnapshotsOnce();
+    std::string actual = fixture.render_last_snapshot_as_prometheus();
+    EXPECT_THAT(actual, HasSubstr(R"(sneaky_path_a_name_count{foo_bar="blah\nbaz\"zoid\\"} 1 1300000)"));
+    EXPECT_THAT(actual, HasSubstr("sneaky_path_another_name_count 1 1300000"));
+}
+
 namespace {
     struct MyUpdateHook : public UpdateHook {
         std::ostringstream& _output;
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt
index 00e80ddec26..06a5febaea7 100644
--- a/metrics/src/vespa/metrics/CMakeLists.txt
+++ b/metrics/src/vespa/metrics/CMakeLists.txt
@@ -12,6 +12,7 @@ vespa_add_library(metrics
     metrictimer.cpp
     metricvalueset.cpp
     name_repo.cpp
+    prometheus_writer.cpp
     state_api_adapter.cpp
     summetric.cpp
     textwriter.cpp
diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp
index db27ca63839..652a5e6bd5f 100644
--- a/metrics/src/vespa/metrics/metric.cpp
+++ b/metrics/src/vespa/metrics/metric.cpp
@@ -54,7 +54,7 @@ Tag::Tag(vespalib::stringref k, vespalib::stringref v)
 
 Tag::Tag(const Tag &) noexcept = default;
 Tag & Tag::operator = (const Tag &) = default;
-Tag::~Tag() {}
+Tag::~Tag() = default;
 
 Metric::Metric(const String& name,
                Tags dimensions,
@@ -139,11 +139,11 @@ Metric::createMangledNameWithDimensions() const
 void
 Metric::verifyConstructionParameters()
 {
-    if (getName().size() == 0) {
-        throw vespalib::IllegalArgumentException(
-                "Metric cannot have empty name", VESPA_STRLOC);
+    if (getName().empty()) {
+        throw vespalib::IllegalArgumentException("Metric cannot have empty name", VESPA_STRLOC);
     }
     const auto &name = getName();
+    // FIXME this is broken (should use std::regex_match instead, but we have metrics that will fail this test...!)
     if (!std::regex_search(name.c_str(), name.c_str() + name.size(), name_pattern_regex)) {
         throw vespalib::IllegalArgumentException(
                 "Illegal metric name '" + getName() + "'. Names must match pattern "
diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h
index 36f363a8fc5..afa2851ef13 100644
--- a/metrics/src/vespa/metrics/metric.h
+++ b/metrics/src/vespa/metrics/metric.h
@@ -17,7 +17,7 @@ class MemoryConsumption;
 
 /** Implement class to visit metrics. */
 struct MetricVisitor {
-    virtual ~MetricVisitor() {}
+    virtual ~MetricVisitor() = default;
 
     /**
      * Visit a snapshot. Return true to visit content of the snapshot
@@ -32,8 +32,7 @@ struct MetricVisitor {
      *                      fly such as in sum metrics.
      * @return True if you want to visit the content of this metric set.
      */
-    virtual bool visitMetricSet(const MetricSet&, bool autoGenerated)
-        { (void) autoGenerated; return true; }
+    virtual bool visitMetricSet(const MetricSet&, [[maybe_unused]] bool autoGenerated) { return true; }
 
     /**
      * Callback visitors can use if they need to know the tree traversal of
diff --git a/metrics/src/vespa/metrics/prometheus_writer.cpp b/metrics/src/vespa/metrics/prometheus_writer.cpp
new file mode 100644
index 00000000000..27c509638b2
--- /dev/null
+++ b/metrics/src/vespa/metrics/prometheus_writer.cpp
@@ -0,0 +1,287 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "countmetric.h"
+#include "metricset.h"
+#include "metricsnapshot.h"
+#include "prometheus_writer.h"
+#include "valuemetric.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/hash_set.hpp>
+#include <vespa/vespalib/util/small_vector.h>
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+
+VESPALIB_HASH_SET_INSTANTIATE(vespalib::stringref);
+
+using vespalib::ArrayRef;
+using vespalib::ConstArrayRef;
+using vespalib::stringref;
+using vespalib::asciistream;
+
+namespace metrics {
+
+namespace {
+
+[[nodiscard]] bool any_metric_in_path_has_nonempty_tag(const Metric& m) noexcept {
+    const Metric* current = &m;
+    do {
+        if (std::ranges::any_of(current->getTags(), [](auto& t) noexcept { return t.hasValue(); })) {
+            return true;
+        }
+        current = current->getOwner();
+    } while (current != nullptr);
+    return false;
+}
+
+[[nodiscard]] constexpr bool valid_prometheus_char(char ch) noexcept {
+    // Prometheus also allows ':', but we don't.
+    return ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_');
+}
+
+[[nodiscard]] bool valid_prometheus_name(stringref name) noexcept {
+    return std::ranges::all_of(name, [](char ch) noexcept { return valid_prometheus_char(ch); });
+}
+
+[[nodiscard]] constexpr bool label_char_needs_escaping(char ch) noexcept {
+    return (ch == '\\' || ch == '\n' || ch == '"');
+}
+
+[[nodiscard]] bool label_value_needs_escaping(stringref value) noexcept {
+    return std::ranges::any_of(value, [](char ch) noexcept { return label_char_needs_escaping(ch); });
+}
+
+[[nodiscard]] vespalib::string prometheus_escaped_name(stringref str) {
+    asciistream os;
+    for (char ch : str) {
+        if (valid_prometheus_char(ch)) [[likely]] {
+            os << ch;
+        } else {
+            os << '_';
+        }
+    }
+    return os.str();
+}
+
+[[nodiscard]] bool arrays_eq(ConstArrayRef<stringref> lhs, ConstArrayRef<stringref> rhs) noexcept {
+    return std::ranges::equal(lhs, rhs);
+}
+
+[[nodiscard]] bool arrays_lt(ConstArrayRef<stringref> lhs, ConstArrayRef<stringref> rhs) noexcept {
+    return std::ranges::lexicographical_compare(lhs, rhs);
+}
+
+}
+
+PrometheusWriter::PrometheusWriter(asciistream& out)
+    : MetricVisitor(),
+      _arena(),
+      _timestamp_str(),
+      _samples(),
+      _unique_str_refs(),
+      _path(),
+      _out(out)
+{}
+
+PrometheusWriter::~PrometheusWriter() = default;
+
+bool PrometheusWriter::TimeSeriesSample::operator<(const TimeSeriesSample& rhs) const noexcept {
+    // Standard multidimensional strict-weak ordering, with an indirection via
+    // ConstArrayRefs for the first and last dimension.
+    if (!arrays_eq(metric_path, rhs.metric_path)) {
+        return arrays_lt(metric_path, rhs.metric_path);
+    }
+    if (aggr != rhs.aggr) {
+        return aggr < rhs.aggr;
+    }
+    return arrays_lt(labels, rhs.labels);
+}
+
+stringref PrometheusWriter::arena_stable_string_ref(stringref str) {
+    auto maybe_iter = _unique_str_refs.find(str);
+    if (maybe_iter != _unique_str_refs.end()) {
+        return *maybe_iter;
+    }
+    auto buf = _arena.create_uninitialized_array<char>(str.size());
+    memcpy(buf.data(), str.data(), buf.size());
+    stringref ref(buf.data(), buf.size());
+    _unique_str_refs.insert(ref);
+    return ref;
+}
+
+stringref PrometheusWriter::stable_name_string_ref(stringref raw_name) {
+    if (valid_prometheus_name(raw_name)) [[likely]] {
+        return arena_stable_string_ref(raw_name);
+    } else {
+        return arena_stable_string_ref(prometheus_escaped_name(raw_name));
+    }
+}
+
+ConstArrayRef<stringref> PrometheusWriter::metric_to_path_ref(stringref leaf_metric_name) {
+    vespalib::SmallVector<stringref, 16> path_refs;
+    // _path strings are already in canonical (sanitized) form and arena-allocated
+    for (const auto& p :_path) {
+        path_refs.emplace_back(p);
+    }
+    path_refs.emplace_back(stable_name_string_ref(leaf_metric_name));
+    return _arena.copy_array<stringref>({path_refs.data(), path_refs.size()});
+}
+
+vespalib::string PrometheusWriter::escaped_label_value(stringref value) {
+    asciistream out;
+    for (char ch : value) {
+        if (ch == '\\') {
+            out << "\\\\";
+        } else if (ch == '"') {
+            out << "\\\"";
+        } else if (ch == '\n') {
+            out << "\\n";
+        } else [[likely]] {
+            out << ch; // assumed to be part of a valid UTF-8 sequence
+        }
+    }
+    return out.str();
+}
+
+stringref PrometheusWriter::stable_label_value_string_ref(stringref raw_label_value) {
+    if (!label_value_needs_escaping(raw_label_value)) [[likely]] {
+        return arena_stable_string_ref(raw_label_value);
+    } else {
+        return arena_stable_string_ref(escaped_label_value(raw_label_value));
+    }
+}
+
+void PrometheusWriter::build_labels_upto_root(vespalib::SmallVector<stringref, 16>& out, const Metric& m) {
+    const Metric* current = &m;
+    do {
+        for (const auto& tag : current->getTags()) {
+            if (!tag.hasValue()) {
+                continue; // Don't emit value-less tags, as these are not proper labels
+            }
+            out.emplace_back(stable_name_string_ref(tag.key()));
+            out.emplace_back(stable_label_value_string_ref(tag.value()));
+        }
+        current = current->getOwner();
+    } while (current != nullptr);
+}
+
+ConstArrayRef<stringref> PrometheusWriter::as_prometheus_labels(const Metric& m) {
+    if (!any_metric_in_path_has_nonempty_tag(m)) {
+        return {};
+    }
+    vespalib::SmallVector<stringref, 16> kv_refs;
+    build_labels_upto_root(kv_refs, m);
+    return _arena.copy_array<stringref>(kv_refs);
+}
+
+bool PrometheusWriter::visitSnapshot(const MetricSnapshot& ms) {
+    // Pre-cache timestamp in string form to avoid same conversion for every time series
+    _timestamp_str = std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(
+            ms.getToTime().time_since_epoch()).count());
+    return true;
+}
+
+void PrometheusWriter::doneVisitingSnapshot(const MetricSnapshot&) {
+    // No-op
+}
+
+bool PrometheusWriter::visitMetricSet(const MetricSet& set, bool) {
+    // Don't include metric sets that will be aggregated up into a separate sum metric.
+    // We don't care about individual threads etc., just their aggregate values.
+    if (set.hasTag("partofsum")) {
+        return false;
+    }
+    if (set.getOwner()) {
+        _path.emplace_back(stable_name_string_ref(set.getName()));
+    } // else: don't add the topmost set
+    return true;
+}
+
+void PrometheusWriter::doneVisitingMetricSet(const MetricSet& set) {
+    if (set.getOwner()) {
+        assert(!_path.empty());
+        _path.pop_back();
+    }
+}
+
+bool PrometheusWriter::visitCountMetric(const AbstractCountMetric& m, bool) {
+    auto full_path = metric_to_path_ref(m.getName());
+    auto labels    = as_prometheus_labels(m);
+    _samples.emplace_back(TimeSeriesSample{full_path, "count", labels, {m.getLongValue("count")}});
+    return true;
+}
+
+bool PrometheusWriter::visitValueMetric(const AbstractValueMetric& m, bool) {
+    auto full_path = metric_to_path_ref(m.getName());
+    auto labels    = as_prometheus_labels(m);
+    _samples.emplace_back(TimeSeriesSample{full_path, "count", labels, {m.getLongValue("count")}});
+    _samples.emplace_back(TimeSeriesSample{full_path, "sum",   labels, {m.getDoubleValue("total")}});
+    _samples.emplace_back(TimeSeriesSample{full_path, "min",   labels, {m.getDoubleValue("min")}});
+    _samples.emplace_back(TimeSeriesSample{full_path, "max",   labels, {m.getDoubleValue("max")}});
+    return true;
+}
+
+void PrometheusWriter::render_path_as_metric_name_prefix(asciistream& out, ConstArrayRef<stringref> path) {
+    for (const auto& p : path) {
+        out << p << '_';
+    }
+}
+
+void PrometheusWriter::render_label_pairs(asciistream& out, ConstArrayRef<stringref> labels) {
+    if (!labels.empty()) {
+        assert((labels.size() % 2) == 0);
+        out << '{';
+        for (size_t i = 0; i < labels.size(); i += 2) {
+            if (i > 0) {
+                out << ',';
+            }
+            // We expect both label key and value to be pre-normalized/sanitized.
+            out << labels[i] << "=\"" << labels[i + 1] << '"';
+        }
+        out << '}';
+    }
+}
+
+void PrometheusWriter::render_sample_value(asciistream& out, I64OrDouble value) {
+    if (std::holds_alternative<double>(value)) {
+        const double v = std::get<double>(value);
+        const bool inf = std::isinf(v);
+        const bool nan = std::isnan(v);
+        // Prometheus allows "-Inf", "+Inf" and "NaN" as special values for negative infinity,
+        // positive infinity and "not a number", respectively.
+        if (!inf && !nan) [[likely]] {
+            out << asciistream::Precision(16) << vespalib::automatic << v;
+        } else if (inf) {
+            out << (v < 0.0 ? "-Inf" : "+Inf");
+        } else {
+            out << "NaN";
+        }
+    } else {
+        const int64_t v = std::get<int64_t>(value);
+        out << v;
+    }
+}
+
+void PrometheusWriter::doneVisiting() {
+    _out << "# NOTE: THIS API IS NOT INTENDED FOR PUBLIC USE\n";
+    // Sort and implicitly group all related metrics together, ordered by name -> aggregation -> dimensions
+    std::sort(_samples.begin(), _samples.end());
+    ConstArrayRef<stringref> last_metric;
+    stringref last_aggr;
+    for (const auto& s : _samples) {
+        if ((s.aggr != last_aggr) || !arrays_eq(s.metric_path, last_metric)) {
+            _out << "# TYPE ";
+            render_path_as_metric_name_prefix(_out, s.metric_path);
+            _out << s.aggr << " untyped\n";
+            last_metric = s.metric_path;
+            last_aggr   = s.aggr;
+        }
+        render_path_as_metric_name_prefix(_out, s.metric_path);
+        _out << s.aggr;
+        render_label_pairs(_out, s.labels);
+        _out << ' ';
+        render_sample_value(_out, s.value);
+        _out << ' ' << _timestamp_str << '\n';
+    }
+}
+
+}
diff --git a/metrics/src/vespa/metrics/prometheus_writer.h b/metrics/src/vespa/metrics/prometheus_writer.h
new file mode 100644
index 00000000000..410ea3b2b8e
--- /dev/null
+++ b/metrics/src/vespa/metrics/prometheus_writer.h
@@ -0,0 +1,77 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "metric.h"
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/util/small_vector.h>
+#include <vespa/vespalib/util/stash.h>
+#include <vespa/vespalib/util/time.h>
+#include <variant>
+#include <vector>
+
+namespace vespalib { class asciistream; }
+
+namespace metrics {
+
+/**
+ * Metric snapshot writer which emits text output conforming to the Prometheus
+ * 0.0.4 exposition format.
+ *
+ * Metrics are emitted by effectively "flattening" all paths in the metric tree
+ * and grouping by path and aggregation type (sum, count etc).
+ *
+ * - For CountMetrics, emits `_count` (i64) time series only.
+ * - For ValueMetrics, emits `_count` (i64), `_sum`, `_min` and `_max` (double)
+ *   time series (`_last` and `_average` are _not_ included).
+ *
+ * Due to poor compatibility between Prometheus and our internal data model, all
+ * time series are emitted as "untyped".
+ */
+class PrometheusWriter : public MetricVisitor {
+    using I64OrDouble = std::variant<int64_t, double>;
+    struct TimeSeriesSample {
+        // All referenced strings shall be either arena-allocated or static
+        vespalib::ConstArrayRef<vespalib::stringref> metric_path;
+        vespalib::stringref                          aggr;
+        // Labels are laid out in key/value pairs, that is size() is always % 2 == 0
+        vespalib::ConstArrayRef<vespalib::stringref> labels;
+        I64OrDouble                                  value;
+
+        bool operator<(const TimeSeriesSample& rhs) const noexcept;
+    };
+
+    vespalib::Stash                         _arena;
+    std::string                             _timestamp_str;
+    std::vector<TimeSeriesSample>           _samples;
+    vespalib::hash_set<vespalib::stringref> _unique_str_refs;
+    std::vector<vespalib::stringref>        _path;
+    vespalib::asciistream&                  _out;
+public:
+    explicit PrometheusWriter(vespalib::asciistream& out);
+    ~PrometheusWriter() override;
+
+private:
+    [[nodiscard]] vespalib::stringref arena_stable_string_ref(vespalib::stringref str);
+    [[nodiscard]] vespalib::ConstArrayRef<vespalib::stringref> as_prometheus_labels(const Metric& m);
+    [[nodiscard]] vespalib::ConstArrayRef<vespalib::stringref> metric_to_path_ref(vespalib::stringref leaf_metric_name);
+    [[nodiscard]] vespalib::stringref stable_name_string_ref(vespalib::stringref raw_name);
+    [[nodiscard]] vespalib::stringref stable_label_value_string_ref(vespalib::stringref raw_label_value);
+    void build_labels_upto_root(vespalib::SmallVector<vespalib::stringref, 16>& out, const Metric& m);
+
+    [[nodiscard]] static vespalib::string escaped_label_value(vespalib::stringref value);
+    // Renders name with a tailing '_' character, as the caller is expected to append an aggregate.
+    static void render_path_as_metric_name_prefix(vespalib::asciistream& out, vespalib::ConstArrayRef<vespalib::stringref> path);
+    static void render_label_pairs(vespalib::asciistream& out, vespalib::ConstArrayRef<vespalib::stringref> labels);
+    static void render_sample_value(vespalib::asciistream& out, I64OrDouble value);
+
+    // MetricVisitor impl
+    bool visitSnapshot(const MetricSnapshot&) override;
+    void doneVisitingSnapshot(const MetricSnapshot&) override;
+    bool visitMetricSet(const MetricSet&, bool autoGenerated) override;
+    void doneVisitingMetricSet(const MetricSet&) override;
+    bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override;
+    bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override;
+    void doneVisiting() override;
+};
+
+}