diff options
author | Arne Juul <arnej@yahoo-inc.com> | 2017-11-13 13:24:01 +0000 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2017-12-03 20:05:01 +0000 |
commit | 00404e5b3966e20dac4a012ad8329b49bac281e9 (patch) | |
tree | c1382c677ebf3cd1366161c28f5a93794a947478 /staging_vespalib/src | |
parent | 7635f081e3ef57797de662b77206e64236960e68 (diff) |
first take on new simple metrics library.
This is still a Work In Progress, the most basic
public APIs should be stable now but most of the
implementation is still untested and undocumented.
Diffstat (limited to 'staging_vespalib/src')
45 files changed, 1908 insertions, 27 deletions
diff --git a/staging_vespalib/src/testlist.txt b/staging_vespalib/src/testlist.txt deleted file mode 100644 index f29e51218ec..00000000000 --- a/staging_vespalib/src/testlist.txt +++ /dev/null @@ -1,27 +0,0 @@ -tests/array -tests/benchmark -tests/bits -tests/clock -tests/crc -tests/databuffer -tests/directio -tests/dotproduct -tests/encoding/base64 -tests/fileheader -tests/floatingpointtype -tests/growablebytebuffer -tests/json -tests/librarypool -tests/memorydatastore -tests/objectdump -tests/objects -tests/objectselection -tests/programoptions -tests/polymorphicarray -tests/rusage -tests/shutdownguard -tests/state_server -tests/stllike -tests/timer -tests/util/process_memory_stats -tests/xmlserializable diff --git a/staging_vespalib/src/tests/metrics/CMakeLists.txt b/staging_vespalib/src/tests/metrics/CMakeLists.txt new file mode 100644 index 00000000000..5319511e769 --- /dev/null +++ b/staging_vespalib/src/tests/metrics/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_metrics_test_app TEST + SOURCES + simple_metrics_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_metrics_test_app COMMAND staging_vespalib_metrics_test_app) diff --git a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp new file mode 100644 index 00000000000..0adef8d353f --- /dev/null +++ b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/metrics/simple_metrics.h> +#include <vespa/vespalib/metrics/simple_metrics_manager.h> +#include <vespa/vespalib/metrics/no_realloc_bunch.h> +#include <vespa/vespalib/metrics/json_formatter.h> +#include <stdio.h> +#include <unistd.h> + +using namespace vespalib; +using namespace vespalib::metrics; + +TEST("require that simple metrics gauge merge works") +{ + MetricIdentifier id(42); + GaugeAggregator a(id), b(id), c(id); + b.observedCount = 3; + b.sumValue = 24.0; + b.minValue = 7.0; + b.maxValue = 9.0; + b.lastValue = 8.0; + + EXPECT_EQUAL(a.observedCount, 0u); + EXPECT_EQUAL(a.sumValue, 0.0); + EXPECT_EQUAL(a.minValue, 0.0); + EXPECT_EQUAL(a.maxValue, 0.0); + EXPECT_EQUAL(a.lastValue, 0.0); + a.merge(b); + EXPECT_EQUAL(a.observedCount, 3u); + EXPECT_EQUAL(a.sumValue, 24.0); + EXPECT_EQUAL(a.minValue, 7.0); + EXPECT_EQUAL(a.maxValue, 9.0); + EXPECT_EQUAL(a.lastValue, 8.0); + a.merge(b); + EXPECT_EQUAL(a.observedCount, 6u); + EXPECT_EQUAL(a.sumValue, 48.0); + EXPECT_EQUAL(a.minValue, 7.0); + EXPECT_EQUAL(a.maxValue, 9.0); + EXPECT_EQUAL(a.lastValue, 8.0); + + c.observedCount = 2; + c.sumValue = 11.0; + c.minValue = 1.0; + c.maxValue = 10.0; + c.lastValue = 1.0; + + a.merge(c); + EXPECT_EQUAL(a.observedCount, 8u); + EXPECT_EQUAL(a.sumValue, 59.0); + EXPECT_EQUAL(a.minValue, 1.0); + EXPECT_EQUAL(a.maxValue, 10.0); + EXPECT_EQUAL(a.lastValue, 1.0); +} + +struct Foo { + int a; + char *p; + explicit Foo(int v) : a(v), p(nullptr) {} + bool operator==(const Foo &other) const { + return a == other.a; + } +}; + +TEST("require that no_realloc_bunch works") +{ + vespalib::NoReallocBunch<Foo> bunch; + bunch.add(Foo(1)); + bunch.add(Foo(2)); + bunch.add(Foo(3)); + bunch.add(Foo(5)); + bunch.add(Foo(8)); + bunch.add(Foo(13)); + bunch.add(Foo(21)); + bunch.add(Foo(34)); + bunch.add(Foo(55)); + bunch.add(Foo(89)); + + EXPECT_EQUAL(bunch.size(), 10u); + + int sum = 0; + + bunch.apply([&sum](const Foo& value) { sum += value.a; }); + EXPECT_EQUAL(231, sum); + + const Foo& val = bunch.lookup(8); + EXPECT_TRUE(Foo(55) == val); + + for (int i = 0; i < 20000; ++i) { + bunch.add(Foo(i)); + } + EXPECT_TRUE(Foo(19999) == bunch.lookup(20009)); +} + +TEST("use simple_metrics_collector") +{ + using namespace vespalib::metrics; + SimpleManagerConfig cf; + cf.sliding_window_seconds = 5; + auto manager = SimpleMetricsManager::create(cf); + Counter myCounter = manager->counter("foo"); + myCounter.add(); + myCounter.add(16); + + Gauge myGauge = manager->gauge("bar"); + myGauge.sample(42.0); + myGauge.sample(41.0); + myGauge.sample(43.0); + myGauge.sample(42.0); + + Point one = manager->pointBuilder() + .bind("chain", "default") + .bind("documenttype", "music") + .bind("thread", "0").build(); + Point two = manager->pointBuilder() + .bind("chain", "vespa") + .bind("documenttype", "blogpost") + .bind("thread", "1"); + EXPECT_EQUAL(one.id(), 1u); + EXPECT_EQUAL(two.id(), 2u); + + Point anotherOne = manager->pointBuilder() + .bind("chain", "default") + .bind("documenttype", "music") + .bind("thread", "0"); + EXPECT_EQUAL(anotherOne.id(), 1u); + + Point three = manager->pointBuilder(two).bind("thread", "2"); + EXPECT_EQUAL(three.id(), 3u); + + myCounter.add(3, one); + myCounter.add(one); + myGauge.sample(14.0, two); + myGauge.sample(11.0, three); + + // sleep(2); + + Snapshot snap = manager->snapshot(); + fprintf(stdout, "snap begin: %15f\n", snap.startTime()); + fprintf(stdout, "snap end: %15f\n", snap.endTime()); + + // for (const auto& entry : snap.points()) { + // fprintf(stdout, "snap point: %zd dimension(s)\n", entry.dimensions.size()); + // for (const auto& dim : entry.dimensions) { + // fprintf(stdout, " label: [%s] = '%s'\n", + // dim.dimensionName().c_str(), dim.labelValue().c_str()); + // } + // } + for (const auto& entry : snap.counters()) { + fprintf(stdout, "snap counter: '%s'\n", entry.name().c_str()); + for (const auto& dim : entry.point().dimensions) { + fprintf(stdout, " label: [%s] = '%s'\n", + dim.dimensionName().c_str(), dim.labelValue().c_str()); + } + fprintf(stdout, " count: %zd\n", entry.count()); + } + for (const auto& entry : snap.gauges()) { + fprintf(stdout, "snap gauge: '%s'\n", entry.name().c_str()); + for (const auto& dim : entry.point().dimensions) { + fprintf(stdout, " label: [%s] = '%s'\n", + dim.dimensionName().c_str(), dim.labelValue().c_str()); + } + fprintf(stdout, " observed: %zd\n", entry.observedCount()); + fprintf(stdout, " avg: %f\n", entry.averageValue()); + fprintf(stdout, " min: %f\n", entry.minValue()); + fprintf(stdout, " max: %f\n", entry.maxValue()); + fprintf(stdout, " last: %f\n", entry.lastValue()); + } + + JsonFormatter fmt(snap); + fprintf(stdout, "JSON format:\n>>>\n%s\n<<<\n", fmt.asString().c_str()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt index 24d576eb775..76c69284839 100644 --- a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(staging_vespalib $<TARGET_OBJECTS:staging_vespalib_vespalib_encoding> $<TARGET_OBJECTS:staging_vespalib_vespalib_util> $<TARGET_OBJECTS:staging_vespalib_vespalib_data> + $<TARGET_OBJECTS:staging_vespalib_vespalib_metrics> $<TARGET_OBJECTS:staging_vespalib_vespalib_objects> $<TARGET_OBJECTS:staging_vespalib_vespalib_stllike> $<TARGET_OBJECTS:staging_vespalib_vespalib_net> diff --git a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt new file mode 100644 index 00000000000..657565fc974 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_metrics OBJECT + SOURCES + clock.cpp + counter.cpp + dimension.cpp + dummy_metrics_manager.cpp + gauge.cpp + json_formatter.cpp + label.cpp + mergers.cpp + metric_identifier.cpp + metrics_manager.cpp + metric_types.cpp + name_collection.cpp + no_realloc_bunch.cpp + point_builder.cpp + point.cpp + point_map.cpp + producer.cpp + simple_metrics.cpp + simple_metrics_manager.cpp + snapshots.cpp + + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp b/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp new file mode 100644 index 00000000000..088dd721cbf --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "clock.h" + +namespace vespalib { + +std::chrono::microseconds since_epoch(InternalTimeStamp stamp) +{ + using namespace std::chrono; + using MyInt = microseconds::rep; + + auto before = system_clock::now(); + auto now = steady_clock::now(); + auto after = system_clock::now(); + + MyInt beforems = (time_point_cast<microseconds>(before)).time_since_epoch().count(); + MyInt nowms = (time_point_cast<microseconds>(now)).time_since_epoch().count(); + MyInt afterms = (time_point_cast<microseconds>(after)).time_since_epoch().count(); + + MyInt difference = beforems - nowms; + MyInt adjust = (afterms - beforems) / 2; + + MyInt stampms = stamp.time_since_epoch().count(); + + return microseconds(stampms + difference + adjust); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.h b/staging_vespalib/src/vespa/vespalib/metrics/clock.h new file mode 100644 index 00000000000..9bdb8241139 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/clock.h @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <chrono> + +namespace vespalib { + +using InternalClock = std::chrono::steady_clock; +using InternalTimeStamp = std::chrono::time_point<std::chrono::steady_clock, + std::chrono::microseconds>; +using WallClock = std::chrono::system_clock; +using WallTimeStamp = std::chrono::time_point<std::chrono::system_clock, + std::chrono::microseconds>; + +inline InternalTimeStamp now_stamp() +{ + using namespace std::chrono; + return time_point_cast<microseconds>(steady_clock::now()); +} + +std::chrono::microseconds since_epoch(InternalTimeStamp stamp); + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp b/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp new file mode 100644 index 00000000000..2eb3719801b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "counter.h" +#include "metrics_manager.h" + +namespace vespalib { +namespace metrics { + +void +Counter::add() const +{ + add(1); +} + +void +Counter::add(Point p) const +{ + add(1, p); +} + +void +Counter::add(size_t count) const +{ + if (_manager) { + _manager->add(CounterIncrement(ident(), count)); + } +} + +void +Counter::add(size_t count, Point point) const +{ + if (_manager) { + MetricIdentifier id(_idx.name_idx, point.id()); + _manager->add(CounterIncrement(id, count)); + } +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.h b/staging_vespalib/src/vespa/vespalib/metrics/counter.h new file mode 100644 index 00000000000..56b5d277e90 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/counter.h @@ -0,0 +1,45 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include "metric_identifier.h" +#include "point.h" + +namespace vespalib { +namespace metrics { + +class MetricsManager; +class CounterAggregator; + +struct CounterIncrement { + MetricIdentifier idx; + size_t value; + CounterIncrement() = delete; + CounterIncrement(MetricIdentifier id, size_t v) : idx(id), value(v) {} +}; + +class Counter { + std::shared_ptr<MetricsManager> _manager; + MetricIdentifier _idx; + MetricIdentifier ident() const { return _idx; } +public: + Counter() : _manager(), _idx() {} + Counter(const Counter&) = delete; + Counter(Counter &&other) = default; + Counter& operator= (const Counter &) = delete; + Counter& operator= (Counter &&other) = default; + Counter(std::shared_ptr<MetricsManager> m, MetricIdentifier id) + : _manager(std::move(m)), _idx(id) + {} + + void add() const; + void add(size_t count) const; + void add(Point p) const; + void add(size_t count, Point p) const; + + typedef CounterAggregator aggregator_type; + typedef CounterIncrement sample_type; +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp new file mode 100644 index 00000000000..05bdd42f34a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp @@ -0,0 +1,2 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "dimension.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h b/staging_vespalib/src/vespa/vespalib/metrics/dimension.h new file mode 100644 index 00000000000..3d6e2cc0eea --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/dimension.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { +namespace metrics { + +using DimensionName = vespalib::string; + +class Dimension { + const size_t _dimension_idx; +public: + size_t id() const { return _dimension_idx; } + Dimension(size_t id) : _dimension_idx(id) {} + bool operator< (const Dimension &other) const { return id() < other.id(); } +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp new file mode 100644 index 00000000000..50b26cf60fb --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "dummy_metrics_manager.h" + +namespace vespalib { +namespace metrics { + +DummyMetricsManager::~DummyMetricsManager() {} + +Snapshot +DummyMetricsManager::snapshot() +{ + InternalTimeStamp endTime = now_stamp(); + std::chrono::microseconds s = since_epoch(_startTime); + std::chrono::microseconds e = since_epoch(endTime); + const double micro = 0.000001; + Snapshot snap(s.count() * micro, e.count() * micro); + return snap; +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h new file mode 100644 index 00000000000..1eeda79e117 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h @@ -0,0 +1,62 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <thread> +#include <vespa/vespalib/stllike/string.h> +#include "name_collection.h" +#include "mergers.h" +#include "snapshots.h" +#include "metrics_manager.h" +#include "clock.h" + +namespace vespalib { +namespace metrics { + +/** + * Dummy manager that discards everything, use + * for unit tests where you don't care about + * metrics. + **/ +class DummyMetricsManager : public MetricsManager +{ +private: + InternalTimeStamp _startTime; + DummyMetricsManager() : _startTime(now_stamp()) {} +public: + ~DummyMetricsManager(); + + static std::shared_ptr<MetricsManager> create() { + return std::shared_ptr<MetricsManager>(new DummyMetricsManager()); + } + + Counter counter(const vespalib::string &) override { + return Counter(shared_from_this(), MetricIdentifier(0)); + } + Gauge gauge(const vespalib::string &) override { + return Gauge(shared_from_this(), MetricIdentifier(0)); + } + + Dimension dimension(const vespalib::string &) override { + return Dimension(0); + } + Label label(const vespalib::string &) override { + return Label(0); + } + PointBuilder pointBuilder(Point) override { + return PointBuilder(shared_from_this()); + } + Point pointFrom(PointMapBacking &&) override { + return Point(0); + } + + Snapshot snapshot() override; + + // for use from Counter only + void add(CounterIncrement) override {} + // for use from Gauge only + void sample(GaugeMeasurement) override {} +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp b/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp new file mode 100644 index 00000000000..b2d9f6b946a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "gauge.h" +#include "metrics_manager.h" + +namespace vespalib { +namespace metrics { + +void +Gauge::sample(double value) const +{ + if (_manager) { + _manager->sample(GaugeMeasurement(ident(), value)); + } +} + +void +Gauge::sample(double value, Point point) const +{ + if (_manager) { + MetricIdentifier id(_idx.name_idx, point.id()); + _manager->sample(GaugeMeasurement(id, value)); + } +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h b/staging_vespalib/src/vespa/vespalib/metrics/gauge.h new file mode 100644 index 00000000000..c5a1bfb25d8 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/gauge.h @@ -0,0 +1,39 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include "metric_identifier.h" +#include "point.h" + +namespace vespalib { +namespace metrics { + +class MetricsManager; +class GaugeAggregator; + +struct GaugeMeasurement { + MetricIdentifier idx; + double value; + GaugeMeasurement() = delete; + GaugeMeasurement(MetricIdentifier id, double v) : idx(id), value(v) {} +}; + +class Gauge { +private: + std::shared_ptr<MetricsManager> _manager; + MetricIdentifier _idx; + MetricIdentifier ident() const { return _idx; } +public: + Gauge(std::shared_ptr<MetricsManager> m, MetricIdentifier id) + : _manager(std::move(m)), _idx(id) + {} + + void sample(double value) const; + void sample(double value, Point p) const; + + typedef GaugeAggregator aggregator_type; + typedef GaugeMeasurement sample_type; +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp new file mode 100644 index 00000000000..58ef76ce8b0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp @@ -0,0 +1,75 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "json_formatter.h" + +namespace vespalib { +namespace metrics { + +JsonFormatter::JsonFormatter(const Snapshot &snapshot) + : _data(), + _top(_data.setObject()), + _snapLen(snapshot.endTime() - snapshot.startTime()) +{ + if (_snapLen < 0.1) { + _snapLen = 0.1; + } + // cosmetics: ordering inside objects + _data.insert("name"); + _data.insert("dimensions"); + vespalib::slime::Cursor& meta = _top.setObject("snapshot"); + meta.setLong("from", (long)snapshot.startTime()); + meta.setLong("to", (long)snapshot.endTime()); + handle(snapshot, _top.setArray("values")); +} + +void +JsonFormatter::handle(const Snapshot &snapshot, vespalib::slime::Cursor &target) +{ + for (const CounterSnapshot &entry : snapshot.counters()) { + handle(entry, target.addObject()); + } + for (const GaugeSnapshot &entry : snapshot.gauges()) { + handle(entry, target.addObject()); + } +} + +void +JsonFormatter::handle(const CounterSnapshot &snapshot, vespalib::slime::Cursor &target) +{ + target.setString("name", snapshot.name()); + // target.setString("description", ?); + handle(snapshot.point(), target); + Cursor& inner = target.setObject("values"); + inner.setLong("count", snapshot.count()); + inner.setDouble("rate", snapshot.count() / _snapLen); +} + +void +JsonFormatter::handle(const GaugeSnapshot &snapshot, vespalib::slime::Cursor &target) +{ + target.setString("name", snapshot.name()); + // target.setString("description", ?); + handle(snapshot.point(), target); + Cursor& inner = target.setObject("values"); + inner.setDouble("average", snapshot.averageValue()); + inner.setDouble("min", snapshot.minValue()); + inner.setDouble("max", snapshot.maxValue()); + inner.setDouble("last", snapshot.lastValue()); + inner.setLong("count", snapshot.observedCount()); + inner.setDouble("rate", snapshot.observedCount() / _snapLen); +} + +void +JsonFormatter::handle(const PointSnapshot &snapshot, vespalib::slime::Cursor &target) +{ + if (snapshot.dimensions.size() == 0) { + return; + } + Cursor& inner = target.setObject("dimensions"); + for (const AxisMeasure &entry : snapshot.dimensions) { + inner.setString(entry.dimensionName(), entry.labelValue()); + } +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h new file mode 100644 index 00000000000..c1dc5302c1a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "snapshots.h" +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> + +namespace vespalib { +namespace metrics { + +class JsonFormatter +{ +private: + using Cursor = vespalib::slime::Cursor; + vespalib::Slime _data; + Cursor& _top; + double _snapLen; + + void handle(const Snapshot &snapshot, Cursor &target); + void handle(const PointSnapshot &snapshot, Cursor &target); + void handle(const CounterSnapshot &snapshot, Cursor &target); + void handle(const GaugeSnapshot &snapshot, Cursor &target); +public: + JsonFormatter(const Snapshot &snapshot); + + vespalib::string asString() const { + return _data.toString(); + } +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.cpp b/staging_vespalib/src/vespa/vespalib/metrics/label.cpp new file mode 100644 index 00000000000..218db1ca2ce --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/label.cpp @@ -0,0 +1,2 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "label.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.h b/staging_vespalib/src/vespa/vespalib/metrics/label.h new file mode 100644 index 00000000000..09c5b20e45b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/label.h @@ -0,0 +1,19 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { +namespace metrics { + +using LabelValue = vespalib::string; + +class Label { + const size_t _coord_idx; +public: + size_t id() const { return _coord_idx; } + Label(size_t id) : _coord_idx(id) {} +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/mergers.cpp b/staging_vespalib/src/vespa/vespalib/metrics/mergers.cpp new file mode 100644 index 00000000000..1a4731065b5 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/mergers.cpp @@ -0,0 +1,174 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "mergers.h" +#include <assert.h> +#include <map> + +namespace vespalib { +namespace metrics { + +CounterAggregator::CounterAggregator(MetricIdentifier id) + : idx(id), count(0) +{} + +void +CounterAggregator::merge(const CounterIncrement &increment) +{ + assert(idx == increment.idx); + count += increment.value; +} + +void +CounterAggregator::merge(const CounterAggregator &other) +{ + assert(idx == other.idx); + count += other.count; +} + + +GaugeAggregator::GaugeAggregator(MetricIdentifier id) + : idx(id), + observedCount(0), + sumValue(0.0), + minValue(0.0), + maxValue(0.0), + lastValue(0.0) +{} + +void +GaugeAggregator::merge(const GaugeMeasurement &other) +{ + assert(idx == other.idx); + if (observedCount == 0) { + sumValue = other.value; + minValue = other.value; + maxValue = other.value; + } else { + sumValue += other.value; + minValue = std::min(minValue, other.value); + maxValue = std::max(maxValue, other.value); + } + lastValue = other.value; + ++observedCount; +} + +void +GaugeAggregator::merge(const GaugeAggregator &other) +{ + assert(idx == other.idx); + if (observedCount == 0) { + minValue = other.minValue; + maxValue = other.maxValue; + } else { + minValue = std::min(minValue, other.minValue); + maxValue = std::max(maxValue, other.maxValue); + } + sumValue += other.sumValue; + lastValue = other.lastValue; + observedCount += other.observedCount; +} + +namespace { + +template<typename T> +void +mergeWithMap(const NoReallocBunch<typename T::sample_type> &other, + std::vector<typename T::aggregator_type> &result) +{ + using Aggregator = typename T::aggregator_type; + using Sample = typename T::sample_type; + using Map = std::map<MetricIdentifier, Aggregator>; + using MapValue = typename Map::value_type; + + assert(result.size() == 0); + Map map; + other.apply([&map] (const Sample &sample) { + MetricIdentifier id = sample.idx; + auto iter = map.find(id); + if (iter != map.end()) { + iter->second.merge(sample); + } else { + auto check_iter = map.insert(MapValue(id, Aggregator(id))); + assert(check_iter.second); + check_iter.first->second.merge(sample); + } + }); + for (const MapValue &entry : map) { + result.push_back(entry.second); + } +} + +template<typename T> +std::vector<T> +mergeVectors(const std::vector<T> &a, + const std::vector<T> &b) +{ + std::vector<T> result; + auto a_iter = a.begin(); + auto b_iter = b.begin(); + while (a_iter != a.end() && + b_iter != b.end()) + { + if (a_iter->idx < b_iter->idx) { + result.push_back(*a_iter); + ++a_iter; + } else if (b_iter->idx < a_iter->idx) { + result.push_back(*b_iter); + ++b_iter; + } else { + T both = *a_iter; + both.merge(*b_iter); + result.push_back(both); + ++a_iter; + ++b_iter; + } + } + while (a_iter != a.end()) { + result.push_back(*a_iter); + ++a_iter; + } + while (b_iter != b.end()) { + result.push_back(*b_iter); + ++b_iter; + } + return result; +} + +} // namespace <unnamed> + +void Bucket::merge(const CurrentSamples &other) +{ + mergeWithMap<Counter>(other.counterIncrements, counters); + mergeWithMap<Gauge>(other.gaugeMeasurements, gauges); +} + +void Bucket::merge(const Bucket &other) +{ + assert(startTime <= other.startTime); + assert(endTime <= other.endTime); + endTime = other.endTime; + + std::vector<CounterAggregator> nextCounters = mergeVectors(counters, other.counters); + counters = std::move(nextCounters); + + std::vector<GaugeAggregator> nextGauges = mergeVectors(gauges, other.gauges); + gauges = std::move(nextGauges); +} + +void swap(CurrentSamples& a, CurrentSamples& b) +{ + using std::swap; + swap(a.counterIncrements, b.counterIncrements); + swap(a.gaugeMeasurements, b.gaugeMeasurements); +} + +void swap(Bucket& a, Bucket& b) +{ + using std::swap; + swap(a.startTime, b.startTime); + swap(a.endTime, b.endTime); + swap(a.counters, b.counters); + swap(a.gauges, b.gauges); +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/mergers.h b/staging_vespalib/src/vespa/vespalib/metrics/mergers.h new file mode 100644 index 00000000000..e08efff1599 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/mergers.h @@ -0,0 +1,81 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <mutex> +#include "no_realloc_bunch.h" +#include "metric_identifier.h" +#include "counter.h" +#include "gauge.h" +#include "clock.h" + +namespace vespalib { +namespace metrics { + +// internal +struct CounterAggregator { + MetricIdentifier idx; + size_t count; + + CounterAggregator(MetricIdentifier id); + + void merge(const CounterIncrement &other); + void merge(const CounterAggregator &other); +}; + +// internal +struct GaugeAggregator { + MetricIdentifier idx; + size_t observedCount; + double sumValue; + double minValue; + double maxValue; + double lastValue; + + GaugeAggregator(MetricIdentifier id); + + void merge(const GaugeMeasurement &other); + void merge(const GaugeAggregator &other); +}; + +// internal +struct CurrentSamples { + std::mutex lock; + NoReallocBunch<CounterIncrement> counterIncrements; + NoReallocBunch<GaugeMeasurement> gaugeMeasurements; + + ~CurrentSamples() {} + + void add(CounterIncrement inc) { + std::lock_guard<std::mutex> guard(lock); + counterIncrements.add(inc); + } + void sample(GaugeMeasurement value) { + std::lock_guard<std::mutex> guard(lock); + gaugeMeasurements.add(value); + } +}; + +// internal +struct Bucket { + InternalTimeStamp startTime; + InternalTimeStamp endTime; + std::vector<CounterAggregator> counters; + std::vector<GaugeAggregator> gauges; + + void merge(const CurrentSamples &other); + void merge(const Bucket &other); + + Bucket(InternalTimeStamp started, InternalTimeStamp ended) + : startTime(started), + endTime(ended), + counters(), + gauges() + {} + ~Bucket() {} +}; + +void swap(CurrentSamples& a, CurrentSamples& b); +void swap(Bucket& a, Bucket& b); + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.cpp new file mode 100644 index 00000000000..950adc3462d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.cpp @@ -0,0 +1,2 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "metric_identifier.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h new file mode 100644 index 00000000000..4feb8f1a22d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <cstddef> +#include <functional> + +namespace vespalib { +namespace metrics { + +struct MetricIdentifier { + const size_t name_idx; + const size_t point_idx; + + MetricIdentifier() : name_idx(-1), point_idx(0) {} + + explicit MetricIdentifier(size_t id) + : name_idx(id), point_idx(0) {} + + MetricIdentifier(size_t id, size_t pt) + : name_idx(id), point_idx(pt) {} + + bool operator< (const MetricIdentifier &other) const { + if (name_idx < other.name_idx) return true; + if (name_idx == other.name_idx) { + return (point_idx < other.point_idx); + } + return false; + } + bool operator== (const MetricIdentifier &other) const { + return (name_idx == other.name_idx && + point_idx == other.point_idx); + } +}; + +} // namespace vespalib::metrics +} // namespace vespalib + +namespace std +{ + template<> struct hash<vespalib::metrics::MetricIdentifier> + { + typedef vespalib::metrics::MetricIdentifier argument_type; + typedef std::size_t result_type; + result_type operator()(argument_type const& ident) const noexcept + { + return (ident.point_idx << 20) + ident.name_idx; + } + }; +} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp new file mode 100644 index 00000000000..2f34390f7c9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "metric_types.h" + +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.metrics.metric_types"); + +namespace vespalib { +namespace metrics { + +const char* MetricTypes::_typeNames[] = { + "NONE", + "Counter", + "Gauge", + "Histogram", + "IntegerHistogram" +}; + +void +MetricTypes::check(size_t id, const vespalib::string &name, MetricType ty) +{ + std::lock_guard<std::mutex> guard(_lock); + if (id < _seen.size()) { + MetricType old = _seen[id]; + if (old == ty) { + return; + } + if (old == NONE) { + _seen[id] = ty; + return; + } + LOG(warning, "metric '%s' with different types %s and %s, this will be confusing", + name.c_str(), _typeNames[ty], _typeNames[old]); + } + while (_seen.size() < id) { + _seen.push_back(NONE); + } + _seen.push_back(ty); +} + +} // namespace vespalib::metrics +} // namespace vespalib + + + diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h new file mode 100644 index 00000000000..e2f30d18b4c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <mutex> +#include <vector> +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { +namespace metrics { + +class MetricTypes { + static const char *_typeNames[]; +public: + enum MetricType { + NONE, + COUNTER, + GAUGE, + HISTOGRAM, + INT_HISTOGRAM + }; + + void check(size_t id, const vespalib::string& name, MetricType ty); + + MetricTypes() = default; + ~MetricTypes() {} +private: + std::mutex _lock; + std::vector<MetricType> _seen; +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp new file mode 100644 index 00000000000..be6612bd869 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "metrics_manager.h" + +namespace vespalib { +namespace metrics { + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h new file mode 100644 index 00000000000..e05d42810cb --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <thread> +#include <vespa/vespalib/stllike/string.h> +#include "name_collection.h" +#include "counter.h" +#include "gauge.h" +#include "mergers.h" +#include "snapshots.h" +#include "point.h" +#include "point_builder.h" +#include "dimension.h" +#include "label.h" + +namespace vespalib { +namespace metrics { + + +class MetricsManager + : public std::enable_shared_from_this<MetricsManager> +{ +public: + virtual ~MetricsManager() {} + + virtual Counter counter(const vespalib::string &name) = 0; // get or create + virtual Gauge gauge (const vespalib::string &name) = 0; // get or create + + virtual Dimension dimension(const vespalib::string &name) = 0; // get or create + virtual Label label(const vespalib::string &value) = 0; // get or create + PointBuilder pointBuilder() { + return PointBuilder(shared_from_this()); + } + virtual PointBuilder pointBuilder(Point from) = 0; + + virtual Point pointFrom(PointMapBacking &&map) = 0; + + virtual Snapshot snapshot() = 0; + + // for use from Counter only + virtual void add(CounterIncrement inc) = 0; + + // for use from Gauge only + virtual void sample(GaugeMeasurement value) = 0; +}; + + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp new file mode 100644 index 00000000000..61dec6ef048 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "name_collection.h" +#include <assert.h> + +namespace vespalib { +namespace metrics { + +using Guard = std::lock_guard<std::mutex>; + +const vespalib::string & +NameCollection::lookup(int idx) const +{ + size_t id = idx; + Guard guard(_lock); + assert(id < _names_by_id.size()); + return _names_by_id[id]->first; +} + +size_t +NameCollection::resolve(const vespalib::string& name) +{ + Guard guard(_lock); + Map::const_iterator iter = _names.find(name); + if (iter != _names.end()) { + return iter->second; + } else { + size_t id = _names_by_id.size(); + auto iter_check = _names.insert(Map::value_type(name, id)); + assert(iter_check.second); + _names_by_id.push_back(iter_check.first); + return id; + } +} + +size_t +NameCollection::size() const +{ + Guard guard(_lock); + return _names_by_id.size(); +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h new file mode 100644 index 00000000000..536075b2987 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <mutex> +#include <map> +#include <vector> +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { +namespace metrics { + +class NameCollection { +private: + using Map = std::map<vespalib::string, size_t>; + mutable std::mutex _lock; + Map _names; + std::vector<Map::const_iterator> _names_by_id; +public: + const vespalib::string &lookup(int idx) const; + size_t resolve(const vespalib::string& name); + size_t size() const; + + NameCollection() = default; + ~NameCollection() {} +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.cpp b/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.cpp new file mode 100644 index 00000000000..4e641ff1c31 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.cpp @@ -0,0 +1,4 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "no_realloc_bunch.h" + diff --git a/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.h b/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.h new file mode 100644 index 00000000000..e5bd8f7af62 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/no_realloc_bunch.h @@ -0,0 +1,102 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vector> +#include <assert.h> + +namespace vespalib { + +template <typename T> +class NoReallocBunch +{ + using MyClass = NoReallocBunch<T>; + template<typename U> + friend void swap(NoReallocBunch<U> &a, NoReallocBunch<U> &b); +public: + typedef std::unique_ptr<MyClass> UP; + + NoReallocBunch(); + ~NoReallocBunch() {} + + void add(T t) { + size_t sz = _mine.size(); + if (sz == _mine.capacity()) { + UP next(new MyClass(_size, std::move(_more), std::move(_mine)));; + _mine.clear(); + _mine.reserve(sz << 1); + _more = std::move(next); + } + _mine.push_back(t); + ++_size; + } + + template<typename FUNC> + void apply(FUNC &&func) const { + std::vector<const MyClass *> vv; + dffill(vv); + for (const MyClass *p : vv) { + for (const T& elem : p->_mine) { + func(elem); + } + } + } + + size_t size() const { return _size; } + + const T& lookup(size_t idx) const { + assert(idx < _size); + std::vector<const MyClass *> vv; + dffill(vv); + for (const MyClass *p : vv) { + size_t sz = p->_mine.size(); + if (idx < sz) { + return p->_mine[idx]; + } + idx -= sz; + } + // assert: + return *((T*)nullptr); + } + +private: + void dffill(std::vector<const MyClass *> &vv) const { + if (_more) { _more->dffill(vv); } + vv.push_back(this); + } + + NoReallocBunch(size_t sz, UP &&more, std::vector<T> &&mine); + + size_t _size; + UP _more; + std::vector<T> _mine; +}; + +template<typename T> +NoReallocBunch<T>::NoReallocBunch() + : _size(0), + _more(), + _mine() +{ + _mine.reserve(3); +} + +template<typename T> +NoReallocBunch<T>::NoReallocBunch(size_t sz, UP &&more, std::vector<T> &&mine) + : _size(sz), + _more(std::move(more)), + _mine(std::move(mine)) +{} + +template <typename T> +void swap(NoReallocBunch<T> &a, + NoReallocBunch<T> &b) +{ + using std::swap; + swap(a._size, b._size); + swap(a._mine, b._mine); + swap(a._more, b._more); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point.cpp new file mode 100644 index 00000000000..a5f99a4ba6d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point.cpp @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "point.h" + +namespace vespalib { +namespace metrics { + +Point Point::empty(0); + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.h b/staging_vespalib/src/vespa/vespalib/metrics/point.h new file mode 100644 index 00000000000..5843f6d3c51 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <cstddef> + +namespace vespalib { +namespace metrics { + +class Point { +private: + const size_t _point_idx; +public: + size_t id() const { return _point_idx; } + + static Point empty; + + explicit Point(size_t id) : _point_idx(id) {} +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp new file mode 100644 index 00000000000..c2846ef592e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "point.h" +#include "metrics_manager.h" + +namespace vespalib { +namespace metrics { + +PointBuilder::PointBuilder(std::shared_ptr<MetricsManager> m) + : _owner(std::move(m)), _map() +{} + +PointBuilder::PointBuilder(std::shared_ptr<MetricsManager> m, + const PointMapBacking ©From) + : _owner(std::move(m)), _map(copyFrom) +{} + +PointBuilder && +PointBuilder::bind(Dimension dimension, Label label) && +{ + _map.erase(dimension); + _map.insert(PointMapBacking::value_type(dimension, label)); + return std::move(*this); +} + +PointBuilder && +PointBuilder::bind(Dimension dimension, LabelValue label) && +{ + Label c = _owner->label(label); + return std::move(*this).bind(dimension, c); +} + +PointBuilder && +PointBuilder::bind(DimensionName dimension, LabelValue label) && +{ + Dimension a = _owner->dimension(dimension); + Label c = _owner->label(label); + return std::move(*this).bind(a, c); +} + +Point +PointBuilder::build() +{ + return _owner->pointFrom(PointMapBacking(_map)); +} + +PointBuilder::operator Point() && +{ + return _owner->pointFrom(std::move(_map)); +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h new file mode 100644 index 00000000000..ca0d21b9b8a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/vespalib/stllike/string.h> + +#include "point.h" +#include "point_map.h" + +namespace vespalib { +namespace metrics { + +class MetricsManager; + +class PointBuilder { +private: + std::shared_ptr<MetricsManager> _owner; + PointMapBacking _map; + +public: + PointBuilder(std::shared_ptr<MetricsManager> m); + PointBuilder(std::shared_ptr<MetricsManager> m, const PointMapBacking &from); + ~PointBuilder() {} + + PointBuilder &&bind(Dimension dimension, Label label) &&; + PointBuilder &&bind(Dimension dimension, LabelValue label) &&; + PointBuilder &&bind(DimensionName dimension, LabelValue label) &&; + + Point build(); + operator Point () &&; +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp new file mode 100644 index 00000000000..5fb29cee005 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp @@ -0,0 +1,45 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "point.h" +#include "metrics_manager.h" + +namespace vespalib { +namespace metrics { + +PointMap::PointMap(PointMapBacking &&from) + : _map(std::move(from)), + _hash(0) +{ + for (const PointMapBacking::value_type &entry : _map) { + _hash = (_hash << 7) + (_hash >> 31) + entry.first.id(); + _hash = (_hash << 7) + (_hash >> 31) + entry.second.id(); + } +} + +bool +PointMap::operator< (const PointMap &other) const +{ + // cheap comparison first + if (_hash < other._hash) return true; + if (_hash > other._hash) return false; + auto m = _map.begin(); + auto o = other._map.begin(); + while (m != _map.end()) { + size_t my_f = m->first.id(); + size_t ot_f = o->first.id(); + if (my_f < ot_f) return true; + if (my_f > ot_f) return false; + + size_t my_s = m->second.id(); + size_t ot_s = o->second.id(); + if (my_s < ot_s) return true; + if (my_s > ot_s) return false; + + ++m; + ++o; + } + // equal + return false; +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h b/staging_vespalib/src/vespa/vespalib/metrics/point_map.h new file mode 100644 index 00000000000..fd4498e1b20 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_map.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <map> +#include "dimension.h" +#include "label.h" + +namespace vespalib { +namespace metrics { + +using PointMapBacking = std::map<Dimension, Label>; + +class PointMap { +private: + const PointMapBacking _map; + size_t _hash; +public: + PointMap() : _map(), _hash(0) {} + PointMap(PointMapBacking &&from); + bool operator< (const PointMap &other) const; + + const PointMapBacking &backing() const { return _map; } +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp b/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp new file mode 100644 index 00000000000..a848628a4c0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "producer.h" +#include "metrics_manager.h" +#include "json_formatter.h" + +namespace vespalib { +namespace metrics { + +Producer::Producer(std::shared_ptr<MetricsManager> m) + : _manager(m) +{} + +vespalib::string +Producer::getMetrics(const vespalib::string &) +{ + Snapshot snap = _manager->snapshot(); + JsonFormatter fmt(snap); + return fmt.asString(); +} + +vespalib::string +Producer::getTotalMetrics(const vespalib::string &) +{ + // not implemented + return ""; +} + + + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.h b/staging_vespalib/src/vespa/vespalib/metrics/producer.h new file mode 100644 index 00000000000..c1a9cae86ed --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/producer.h @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/vespalib/net/metrics_producer.h> + +namespace vespalib { +namespace metrics { + +class MetricsManager; + +class Producer : public vespalib::MetricsProducer { +private: + std::shared_ptr<MetricsManager> _manager; +public: + Producer(std::shared_ptr<MetricsManager> m); + vespalib::string getMetrics(const vespalib::string &consumer) override; + vespalib::string getTotalMetrics(const vespalib::string &consumer) override; +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp new file mode 100644 index 00000000000..d9bd3f00892 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "simple_metrics.h" + +namespace vespalib { +namespace metrics { + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h new file mode 100644 index 00000000000..826a43f6713 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h @@ -0,0 +1,7 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <chrono> +#include <memory> +#include <vespa/vespalib/stllike/string.h> diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp new file mode 100644 index 00000000000..9c55a4e2aff --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp @@ -0,0 +1,198 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "simple_metrics_manager.h" + +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.metrics.simple_metrics_manager"); + +namespace vespalib { +namespace metrics { + +using Guard = std::lock_guard<std::mutex>; + +SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config) + : _metricNames(), + _dimensionNames(), + _labelValues(), + _pointMaps(), + _currentBucket(), + _startTime(now_stamp()), + _curTime(_startTime), + _buckets(), + _firstBucket(0), + _maxBuckets(config.sliding_window_seconds), + _stopFlag(false), + _collectorThread(doCollectLoop, this) +{ + if (_maxBuckets < 1) _maxBuckets = 1; + Point empty = pointFrom(PointMapBacking()); + assert(empty.id() == 0); +} + +SimpleMetricsManager::~SimpleMetricsManager() +{ + _stopFlag = true; + _collectorThread.join(); +} + + +std::shared_ptr<MetricsManager> +SimpleMetricsManager::create(const SimpleManagerConfig &config) +{ + return std::shared_ptr<MetricsManager>( + new SimpleMetricsManager(config)); +} + +Counter +SimpleMetricsManager::counter(const vespalib::string &name) +{ + size_t id = _metricNames.resolve(name); + _metricTypes.check(id, name, MetricTypes::COUNTER); + LOG(debug, "metric name %s -> %zd", name.c_str(), id); + return Counter(shared_from_this(), MetricIdentifier(id)); +} + +Gauge +SimpleMetricsManager::gauge(const vespalib::string &name) +{ + size_t id = _metricNames.resolve(name); + _metricTypes.check(id, name, MetricTypes::GAUGE); + LOG(debug, "metric name %s -> %zd", name.c_str(), id); + return Gauge(shared_from_this(), MetricIdentifier(id)); +} + +Bucket +SimpleMetricsManager::mergeBuckets() +{ + Guard bucketsGuard(_bucketsLock); + if (_buckets.size() > 0) { + InternalTimeStamp startTime = _buckets[_firstBucket].startTime; + Bucket merger(startTime, startTime); + for (size_t i = 0; i < _buckets.size(); i++) { + size_t off = (_firstBucket + i) % _buckets.size(); + merger.merge(_buckets[off]); + } + return merger; + } + // no data + return Bucket(_startTime, _curTime); +} + +Snapshot +SimpleMetricsManager::snapshot() +{ + Bucket merged = mergeBuckets(); + std::vector<PointSnapshot> points; + + std::chrono::microseconds s = since_epoch(merged.startTime); + std::chrono::microseconds e = since_epoch(merged.endTime); + const double micro = 0.000001; + Snapshot snap(s.count() * micro, e.count() * micro); + { + Guard guard(_pointMaps.lock); + for (auto entry : _pointMaps.vec) { + const PointMapBacking &map = entry->first.backing(); + PointSnapshot point; + for (const PointMapBacking::value_type &kv : map) { + point.dimensions.emplace_back(nameFor(kv.first), valueFor(kv.second)); + } + snap.add(point); + } + } + for (const CounterAggregator& entry : merged.counters) { + size_t ni = entry.idx.name_idx; + size_t pi = entry.idx.point_idx; + const vespalib::string &name = _metricNames.lookup(ni); + CounterSnapshot val(name, snap.points()[pi], entry); + snap.add(val); + } + for (const GaugeAggregator& entry : merged.gauges) { + size_t ni = entry.idx.name_idx; + size_t pi = entry.idx.point_idx; + const vespalib::string &name = _metricNames.lookup(ni); + GaugeSnapshot val(name, snap.points()[pi], entry); + snap.add(val); + } + return snap; +} + +void +SimpleMetricsManager::doCollectLoop(SimpleMetricsManager *me) +{ + const std::chrono::milliseconds jiffy{20}; + const std::chrono::seconds oneSec{1}; + while (!me->_stopFlag) { + std::this_thread::sleep_for(jiffy); + InternalTimeStamp now = now_stamp(); + InternalTimeStamp::duration elapsed = now - me->_curTime; + if (elapsed >= oneSec) { + me->collectCurrentBucket(); + } + } +} + +void +SimpleMetricsManager::collectCurrentBucket() +{ + InternalTimeStamp prev = _curTime; + InternalTimeStamp curr = now_stamp(); + + CurrentSamples samples; + { + Guard guard(_currentBucket.lock); + swap(samples, _currentBucket); + } + + Bucket merger(prev, curr); + Guard guard(_bucketsLock); + if (_buckets.size() < _maxBuckets) { + _buckets.push_back(merger); + _buckets.back().merge(samples); + } else { + merger.merge(samples); + swap(_buckets[_firstBucket], merger); + _firstBucket = (_firstBucket + 1) % _buckets.size(); + } + _curTime = curr; +} + +Dimension +SimpleMetricsManager::dimension(const vespalib::string &name) +{ + size_t id = _dimensionNames.resolve(name); + LOG(debug, "dimension name %s -> %zd", name.c_str(), id); + return Dimension(id); +} + +Label +SimpleMetricsManager::label(const vespalib::string &value) +{ + size_t id = _labelValues.resolve(value); + LOG(debug, "label value %s -> %zd", value.c_str(), id); + return Label(id); +} + +PointBuilder +SimpleMetricsManager::pointBuilder(Point from) +{ + Guard guard(_pointMaps.lock); + const PointMap &map = _pointMaps.vec[from.id()]->first; + return PointBuilder(shared_from_this(), map.backing()); +} + +Point +SimpleMetricsManager::pointFrom(PointMapBacking &&map) +{ + Guard guard(_pointMaps.lock); + size_t nextId = _pointMaps.vec.size(); + auto iter_check = _pointMaps.map.emplace(std::move(map), nextId); + if (iter_check.second) { + LOG(debug, "new point map -> %zd / %zd", nextId, iter_check.first->second); + _pointMaps.vec.push_back(iter_check.first); + } else { + LOG(debug, "found point map -> %zd", iter_check.first->second); + } + return Point(iter_check.first->second); +} + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h new file mode 100644 index 00000000000..a768ef7f29e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h @@ -0,0 +1,88 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <thread> +#include <vespa/vespalib/stllike/string.h> +#include "name_collection.h" +#include "mergers.h" +#include "snapshots.h" +#include "metrics_manager.h" +#include "metric_types.h" +#include "clock.h" + +namespace vespalib { +namespace metrics { + +struct SimpleManagerConfig { + int sliding_window_seconds; + // possibly more config later + SimpleManagerConfig() : sliding_window_seconds(60) {} +}; + + +/** + * Simple manager class that puts everything into a + * single global repo with std::mutex locks used around + * most operations. Only implements sliding window + * and a fixed (1 Hz) collecting interval. + * Consider renaming to "LockingManager" or "SlidingWindowManager". + **/ +class SimpleMetricsManager : public MetricsManager +{ +private: + NameCollection _metricNames; + MetricTypes _metricTypes; + NameCollection _dimensionNames; + NameCollection _labelValues; + using PointMapMap = std::map<PointMap, size_t>; + struct { + std::mutex lock; + PointMapMap map; + std::vector<PointMapMap::const_iterator> vec; + } _pointMaps; + + const vespalib::string& nameFor(Dimension dimension) { return _dimensionNames.lookup(dimension.id()); } + const vespalib::string& valueFor(Label label) { return _labelValues.lookup(label.id()); } + + CurrentSamples _currentBucket; + + InternalTimeStamp _startTime; + InternalTimeStamp _curTime; + + std::mutex _bucketsLock; + std::vector<Bucket> _buckets; + size_t _firstBucket; + size_t _maxBuckets; + + bool _stopFlag; + std::thread _collectorThread; + static void doCollectLoop(SimpleMetricsManager *me); + void collectCurrentBucket(); // called once per second from another thread + Bucket mergeBuckets(); + + SimpleMetricsManager(const SimpleManagerConfig &config); +public: + ~SimpleMetricsManager(); + static std::shared_ptr<MetricsManager> create(const SimpleManagerConfig &config); + + Counter counter(const vespalib::string &name) override; + Gauge gauge(const vespalib::string &name) override; + Dimension dimension(const vespalib::string &name) override; + Label label(const vespalib::string &value) override; + PointBuilder pointBuilder(Point from) override; + Point pointFrom(PointMapBacking &&map) override; + Snapshot snapshot() override; + + // for use from Counter only + void add(CounterIncrement inc) override { + _currentBucket.add(inc); + } + // for use from Gauge only + void sample(GaugeMeasurement value) override { + _currentBucket.sample(value); + } +}; + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp new file mode 100644 index 00000000000..46de7beaac3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "snapshots.h" + +namespace vespalib { +namespace metrics { + +} // namespace vespalib::metrics +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h new file mode 100644 index 00000000000..87be72d2d52 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h @@ -0,0 +1,100 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "mergers.h" + +namespace vespalib { +namespace metrics { + +class AxisMeasure { +private: + const vespalib::string &_dimensionName; + const vespalib::string &_labelValue; +public: + const vespalib::string &dimensionName() const { return _dimensionName; } + const vespalib::string &labelValue() const { return _labelValue; } + AxisMeasure(const vespalib::string &a, + const vespalib::string &v) + : _dimensionName(a), _labelValue(v) + {} +}; + +struct PointSnapshot { + std::vector<AxisMeasure> dimensions; +}; + +class CounterSnapshot { +private: + const vespalib::string &_name; + const PointSnapshot &_point; + const size_t _count; +public: + CounterSnapshot(const vespalib::string &n, const PointSnapshot &p, const CounterAggregator &c) + : _name(n), _point(p), _count(c.count) + {} + const vespalib::string &name() const { return _name; } + const PointSnapshot &point() const { return _point; } + size_t count() const { return _count; } +}; + +class GaugeSnapshot { +private: + const vespalib::string &_name; + const PointSnapshot &_point; + const size_t _observedCount; + const double _averageValue; + const double _minValue; + const double _maxValue; + const double _lastValue; +public: + GaugeSnapshot(const vespalib::string &n, const PointSnapshot &p, const GaugeAggregator &c) + : _name(n), + _point(p), + _observedCount(c.observedCount), + _averageValue(c.sumValue / c.observedCount), + _minValue(c.minValue), + _maxValue(c.maxValue), + _lastValue(c.lastValue) + {} + const vespalib::string &name() const { return _name; } + const PointSnapshot &point() const { return _point; } + size_t observedCount() const { return _observedCount; } + double averageValue() const { return _averageValue; } + double minValue() const { return _minValue; } + double maxValue() const { return _maxValue; } + double lastValue() const { return _lastValue; } +}; + +class Snapshot { +private: + double _start; + double _end; + std::vector<CounterSnapshot> _counters; + std::vector<GaugeSnapshot> _gauges; + std::vector<PointSnapshot> _points; +public: + double startTime() const { return _start; }; // seconds since 1970 + double endTime() const { return _end; }; // seconds since 1970 + + const std::vector<CounterSnapshot> &counters() const { + return _counters; + } + const std::vector<GaugeSnapshot> &gauges() const { + return _gauges; + } + const std::vector<PointSnapshot> &points() const { + return _points; + } + + // builders: + Snapshot(double s, double e) + : _start(s), _end(e), _counters(), _gauges() + {} + void add(const PointSnapshot &entry) { _points.push_back(entry); } + void add(const CounterSnapshot &entry) { _counters.push_back(entry); } + void add(const GaugeSnapshot &entry) { _gauges.push_back(entry); } +}; + +} // namespace vespalib::metrics +} // namespace vespalib |