diff options
author | Arne Juul <arnej@yahoo-inc.com> | 2017-11-30 11:02:27 +0000 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2017-12-04 08:25:13 +0000 |
commit | 3a4bf578ddf141622ac40d5bd7346f7f1cfb2970 (patch) | |
tree | 5fbe1dcf62f7f2522e90af27d7744804fe71049a /staging_vespalib | |
parent | 746556112b390e7f428b5c8c51353fcffd914f64 (diff) |
update metrics library after reviews
* use slimmer handle subclasses
* new tick abstraction
* gc unused time supplier
* rename current bucket concept to current samples
* rename TimeStamp type
* add padding making old metrics visible
* improve test, now with less debug printing.
Diffstat (limited to 'staging_vespalib')
32 files changed, 477 insertions, 341 deletions
diff --git a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp index d0a7ca4c4a9..6af03177ed8 100644 --- a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp +++ b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/metrics/simple_metrics.h> #include <vespa/vespalib/metrics/simple_metrics_manager.h> #include <vespa/vespalib/metrics/stable_store.h> #include <vespa/vespalib/metrics/json_formatter.h> +#include <vespa/vespalib/metrics/mock_tick.h> #include <stdio.h> #include <unistd.h> @@ -67,13 +70,60 @@ TEST("require that simple metrics gauge merge works") EXPECT_EQUAL(a.lastValue, 1.0); } +bool compare_json(const vespalib::string &a, const vespalib::string &b) +{ + using vespalib::Memory; + using vespalib::slime::JsonFormat; + + Slime slimeA, slimeB; + if (! JsonFormat::decode(a, slimeA)) { +fprintf(stderr, "bad json a:\n>>>%s\n<<<\n", a.c_str()); + return false; + } + if (! JsonFormat::decode(b, slimeB)) { +fprintf(stderr, "bad json b\n"); + return false; + } + return slimeA == slimeB; +} + +void check_json(const vespalib::string &actual) +{ + vespalib::string expect = "{" + " \"snapshot\": { \"from\": 1, \"to\": 4 }," + " \"values\": [ { \"name\": \"foo\"," + " \"values\": { \"count\": 17, \"rate\": 4.85714 }" + " }, {" + " \"name\": \"foo\"," + " \"dimensions\": { \"chain\": \"default\", \"documenttype\": \"music\", \"thread\": \"0\" }," + " \"values\": { \"count\": 4, \"rate\": 1.14286 }" + " }, {" + " \"name\": \"bar\"," + " \"values\": { \"count\": 4, \"rate\": 1.14286, \"average\": 42, \"min\": 41, \"max\": 43, \"last\": 42 }" + " }, {" + " \"name\": \"bar\"," + " \"dimensions\": { \"chain\": \"vespa\", \"documenttype\": \"blogpost\", \"thread\": \"1\" }," + " \"values\": { \"count\": 1, \"rate\": 0.285714, \"average\": 14, \"min\": 14, \"max\": 14, \"last\": 14 }" + " }, {" + " \"name\": \"bar\"," + " \"dimensions\": { \"chain\": \"vespa\", \"documenttype\": \"blogpost\", \"thread\": \"2\" }," + " \"values\": { \"count\": 1, \"rate\": 0.285714, \"average\": 11, \"min\": 11, \"max\": 11, \"last\": 11 }" + " } ]" + "}"; + EXPECT_TRUE(compare_json(expect, actual)); +} + TEST("use simple_metrics_collector") { using namespace vespalib::metrics; SimpleManagerConfig cf; cf.sliding_window_seconds = 5; - auto manager = SimpleMetricsManager::create(cf); + std::shared_ptr<MockTick> ticker = std::make_shared<MockTick>(); + ticker->provide(TimeStamp(1.0)); + auto manager = SimpleMetricsManager::createForTest(cf, std::make_unique<TickProxy>(ticker)); + EXPECT_EQUAL(1.0, ticker->waitUntilBlocked().count()); + Counter myCounter = manager->counter("foo"); myCounter.add(); myCounter.add(16); @@ -84,6 +134,25 @@ TEST("use simple_metrics_collector") myGauge.sample(43.0); myGauge.sample(42.0); + EXPECT_EQUAL(1.0, ticker->waitUntilBlocked().count()); + ticker->provide(TimeStamp(2.0)); + EXPECT_EQUAL(2.0, ticker->waitUntilBlocked().count()); + + Snapshot snap1 = manager->snapshot(); + EXPECT_EQUAL(1.0, snap1.startTime()); + EXPECT_EQUAL(2.0, snap1.endTime()); + + EXPECT_EQUAL(1u, snap1.counters().size()); + EXPECT_EQUAL("foo", snap1.counters()[0].name()); + EXPECT_EQUAL(17u, snap1.counters()[0].count()); + + EXPECT_EQUAL(1u, snap1.gauges().size()); + EXPECT_EQUAL("bar", snap1.gauges()[0].name()); + EXPECT_EQUAL(4u, snap1.gauges()[0].observedCount()); + EXPECT_EQUAL(41.0, snap1.gauges()[0].minValue()); + EXPECT_EQUAL(43.0, snap1.gauges()[0].maxValue()); + EXPECT_EQUAL(42.0, snap1.gauges()[0].lastValue()); + Point one = manager->pointBuilder() .bind("chain", "default") .bind("documenttype", "music") @@ -109,44 +178,45 @@ TEST("use simple_metrics_collector") myGauge.sample(14.0, two); myGauge.sample(11.0, three); - for (int i = 0; i < 61; ++i) { - ((SimpleMetricsManager &)*manager).tick(); - } + EXPECT_EQUAL(2.0, ticker->waitUntilBlocked().count()); + ticker->provide(TimeStamp(4.5)); + EXPECT_EQUAL(4.5, ticker->waitUntilBlocked().count()); - Snapshot snap = manager->totalSnapshot(); - fprintf(stdout, "snap begin: %15f\n", snap.startTime()); - fprintf(stdout, "snap end: %15f\n", snap.endTime()); - - // for (const auto& entry : snap.points()) { - // fprintf(stdout, "snap point: %zd dimension(s)\n", entry.dimensions.size()); - // for (const auto& dim : entry.dimensions) { - // fprintf(stdout, " label: [%s] = '%s'\n", - // dim.dimensionName().c_str(), dim.labelValue().c_str()); - // } - // } - for (const auto& entry : snap.counters()) { - fprintf(stdout, "snap counter: '%s'\n", entry.name().c_str()); - for (const auto& dim : entry.point().dimensions) { - fprintf(stdout, " label: [%s] = '%s'\n", - dim.dimensionName().c_str(), dim.labelValue().c_str()); - } - fprintf(stdout, " count: %zd\n", entry.count()); - } - for (const auto& entry : snap.gauges()) { - fprintf(stdout, "snap gauge: '%s'\n", entry.name().c_str()); - for (const auto& dim : entry.point().dimensions) { - fprintf(stdout, " label: [%s] = '%s'\n", - dim.dimensionName().c_str(), dim.labelValue().c_str()); - } - fprintf(stdout, " observed: %zd\n", entry.observedCount()); - fprintf(stdout, " avg: %f\n", entry.averageValue()); - fprintf(stdout, " min: %f\n", entry.minValue()); - fprintf(stdout, " max: %f\n", entry.maxValue()); - fprintf(stdout, " last: %f\n", entry.lastValue()); - } + Snapshot snap2 = manager->snapshot(); + EXPECT_EQUAL(1.0, snap2.startTime()); + EXPECT_EQUAL(4.5, snap2.endTime()); + EXPECT_EQUAL(2u, snap2.counters().size()); + EXPECT_EQUAL(3u, snap2.gauges().size()); - JsonFormatter fmt(snap); - fprintf(stdout, "JSON format:\n>>>\n%s\n<<<\n", fmt.asString().c_str()); + JsonFormatter fmt2(snap2); + check_json(fmt2.asString()); + + // flush sliding window + for (int i = 5; i <= 10; ++i) { + ticker->provide(TimeStamp(i)); + ticker->waitUntilBlocked(); + } + Snapshot snap3 = manager->snapshot(); + EXPECT_EQUAL(5.0, snap3.startTime()); + EXPECT_EQUAL(10.0, snap3.endTime()); + EXPECT_EQUAL(2u, snap3.counters().size()); + EXPECT_EQUAL(0u, snap3.counters()[0].count()); + EXPECT_EQUAL(0u, snap3.counters()[1].count()); + EXPECT_EQUAL(3u, snap3.gauges().size()); + EXPECT_EQUAL(0u, snap3.gauges()[0].observedCount()); + EXPECT_EQUAL(0u, snap3.gauges()[1].observedCount()); + EXPECT_EQUAL(0u, snap3.gauges()[2].observedCount()); + + Snapshot snap4 = manager->totalSnapshot(); + EXPECT_EQUAL(1.0, snap4.startTime()); + EXPECT_EQUAL(10.0, snap4.endTime()); + EXPECT_EQUAL(2u, snap4.counters().size()); + EXPECT_NOT_EQUAL(0u, snap4.counters()[0].count()); + EXPECT_NOT_EQUAL(0u, snap4.counters()[1].count()); + EXPECT_EQUAL(3u, snap4.gauges().size()); + EXPECT_NOT_EQUAL(0u, snap4.gauges()[0].observedCount()); + EXPECT_NOT_EQUAL(0u, snap4.gauges()[1].observedCount()); + EXPECT_NOT_EQUAL(0u, snap4.gauges()[2].observedCount()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt index d641a768d57..6fec0900b06 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(staging_vespalib_vespalib_metrics OBJECT current_samples.cpp dimension.cpp dummy_metrics_manager.cpp - dummy_time_supplier.cpp gauge_aggregator.cpp gauge.cpp handle.cpp @@ -18,6 +17,7 @@ vespa_add_library(staging_vespalib_vespalib_metrics OBJECT metric_name.cpp metrics_manager.cpp metric_types.cpp + mock_tick.cpp name_collection.cpp point_builder.cpp point.cpp @@ -26,12 +26,9 @@ vespa_add_library(staging_vespalib_vespalib_metrics OBJECT producer.cpp simple_metrics.cpp simple_metrics_manager.cpp + simple_tick.cpp snapshots.cpp stable_store.cpp - test_time_supplier.cpp - ticker_thread.cpp - time_supplier.cpp - wallclock_time_supplier.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp b/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp index 9062066fc84..52e95e1ac3f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp @@ -67,6 +67,39 @@ mergeVectors(const std::vector<T> &a, return result; } +template<typename T> +std::vector<T> +findMissing(const std::vector<T> &already, + const std::vector<T> &complete) +{ + std::vector<T> result; + auto a_iter = already.begin(); + auto c_iter = complete.begin(); + while (a_iter != already.end() && + c_iter != complete.end()) + { + if (a_iter->idx < c_iter->idx) { + // missing from "complete", should not happen + ++a_iter; + } else if (c_iter->idx < a_iter->idx) { + // missing this + result.push_back(*c_iter); + ++c_iter; + } else { + // already have this + ++a_iter; + ++c_iter; + } + } + while (c_iter != complete.end()) { + // missing this + result.push_back(*c_iter); + ++c_iter; + } + return result; +} + + } // namespace <unnamed> void Bucket::merge(const CurrentSamples &samples) @@ -89,5 +122,22 @@ void Bucket::merge(const Bucket &other) gauges = std::move(nextGauges); } +void Bucket::padMetrics(const Bucket &source) +{ + std::vector<CounterAggregator> missingC = findMissing(counters, source.counters); + for (CounterAggregator aggr : missingC) { + aggr.count = 0; + counters.push_back(aggr); + } + std::vector<GaugeAggregator> missingG = findMissing(gauges, source.gauges); + for (GaugeAggregator aggr : missingG) { + aggr.observedCount = 0; + aggr.sumValue = 0; + aggr.minValue = 0; + aggr.maxValue = 0; + gauges.push_back(aggr); + } +} + } // namespace vespalib::metrics } // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h b/staging_vespalib/src/vespa/vespalib/metrics/bucket.h index a0a35e9240d..5a88e435502 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/bucket.h @@ -17,15 +17,16 @@ namespace metrics { // internal struct Bucket { size_t genCnt; - InternalTimeStamp startTime; - InternalTimeStamp endTime; + TimeStamp startTime; + TimeStamp endTime; std::vector<CounterAggregator> counters; std::vector<GaugeAggregator> gauges; void merge(const CurrentSamples &other); void merge(const Bucket &other); + void padMetrics(const Bucket &source); - Bucket(size_t generation, InternalTimeStamp started, InternalTimeStamp ended) + Bucket(size_t generation, TimeStamp started, TimeStamp ended) : genCnt(generation), startTime(started), endTime(ended), diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp b/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp index 9d38d48b281..82144da895d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp @@ -1,14 +1,2 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "clock.h" - -namespace vespalib { -namespace metrics { - -std::chrono::microseconds since_epoch(InternalTimeStamp stamp) -{ - using namespace std::chrono; - return duration_cast<microseconds>(stamp.time_since_epoch()); -} - -} // namespace metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.h b/staging_vespalib/src/vespa/vespalib/metrics/clock.h index d29c79bb927..ae67b542fcd 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/clock.h @@ -1,12 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "wallclock_time_supplier.h" +#include <chrono> +#include <memory> -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { -using InternalTimeStamp = WallclockTimeSupplier::TimeStamp; +using TimeStamp = std::chrono::duration<double, std::ratio<1,1>>; -} // namespace metrics -} // namespace vespalib +/** + * Simple interface abstracting both timing and time measurement for + * threads wanting to do stuff at regular intervals and also knowing + * at what time stuff was done. The 'next' function blocks until the + * next tick is due and returns the current number of seconds since + * epoch. The parameter passed to the 'next' function should be its + * previous return value, except the first time it is called, then 0 + * should be used. A convenience function called 'first' is added for + * this purpose. + **/ +struct Tick { + using UP = std::unique_ptr<Tick>; + virtual TimeStamp next(TimeStamp prev) = 0; + TimeStamp first() { return next(TimeStamp(0.0)); } + virtual void kill() = 0; + virtual ~Tick() {} +}; + +struct TickProxy : Tick { + std::shared_ptr<Tick> tick; + TickProxy(std::shared_ptr<Tick> tick_in) : tick(std::move(tick_in)) {} + TimeStamp next(TimeStamp prev) override { return tick->next(prev); } + void kill() override { tick->kill(); } +}; + +} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp b/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp index 5102ebb121b..a02cc1e7a7e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp @@ -7,7 +7,7 @@ namespace vespalib { namespace metrics { CounterAggregator::CounterAggregator(const Counter::Increment &increment) - : idx(increment.idx), count(increment.value) + : idx(increment.idx), count(increment.value) {} void diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h b/staging_vespalib/src/vespa/vespalib/metrics/dimension.h index 680370ba84c..ee16bc6f98a 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/dimension.h @@ -4,18 +4,15 @@ #include <vespa/vespalib/stllike/string.h> #include "handle.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { using DimensionName = vespalib::string; +struct DimensionTag {}; + /** * Opaque handle representing an uniquely named dimension. **/ -class Dimension : public Handle<Dimension> { -public: - explicit Dimension(size_t id) : Handle<Dimension>(id) {} -}; +using Dimension = Handle<DimensionTag>; } // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.cpp deleted file mode 100644 index 53032d2b0c3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dummy_time_supplier.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.h b/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.h deleted file mode 100644 index 9e74f1006ce..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_time_supplier.h +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace vespalib { -namespace metrics { - -struct DummyTimeSupplier { - typedef int TimeStamp; - TimeStamp now_stamp() const { return 0; } - double stamp_to_s(TimeStamp) const { return 0.0; } -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp b/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp index d7bafb54d0f..84ce8aef28d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp @@ -7,12 +7,12 @@ namespace vespalib { namespace metrics { GaugeAggregator::GaugeAggregator(const Gauge::Measurement &sample) - : idx(sample.idx), - observedCount(1), - sumValue(sample.value), - minValue(sample.value), - maxValue(sample.value), - lastValue(sample.value) + : idx(sample.idx), + observedCount(1), + sumValue(sample.value), + minValue(sample.value), + maxValue(sample.value), + lastValue(sample.value) {} void diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.h b/staging_vespalib/src/vespa/vespalib/metrics/handle.h index 416b50f0bcd..94a9ee33830 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/handle.h @@ -15,9 +15,8 @@ template <typename T> class Handle { private: const size_t _id; -protected: - explicit Handle(size_t id) : _id(id) {} public: + explicit Handle(size_t id) : _id(id) {} size_t id() const { return _id; } }; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.h b/staging_vespalib/src/vespa/vespalib/metrics/label.h index 81e96728c27..e2356415fd6 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/label.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/label.h @@ -4,18 +4,15 @@ #include <vespa/vespalib/stllike/string.h> #include "handle.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { using LabelValue = vespalib::string; +struct LabelTag {}; + /** * Opaque handle representing an uniquely named label. **/ -class Label : public Handle<Label> { -public: - explicit Label(size_t id) : Handle<Label>(id) {} -}; +using Label = Handle<LabelTag>; } // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h index 7565a5ed7ea..020b44dc4b7 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h @@ -15,7 +15,7 @@ struct MetricIdentifier { MetricIdentifier() = delete; MetricIdentifier(MetricName name, Point point) - : _name(name), _point(point) {} + : _name(name), _point(point) {} bool operator< (const MetricIdentifier &other) const { if (_name != other._name) { diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_name.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_name.h index baf35b05e45..60dfa0aa00f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_name.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_name.h @@ -3,16 +3,13 @@ #include "handle.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { + +struct MetricNameTag {}; /** * Opaque handle representing an uniquely named metric. **/ -class MetricName : public Handle<MetricName> { -public: - explicit MetricName(size_t id) : Handle<MetricName>(id) {} -}; +using MetricName = Handle<MetricNameTag>; } // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.cpp b/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.cpp new file mode 100644 index 00000000000..e78b7c4644f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.cpp @@ -0,0 +1,71 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "mock_tick.h" + +namespace vespalib::metrics { + +const std::chrono::seconds oneSec{1}; + +TimeStamp +MockTick::next(TimeStamp prev) +{ + std::unique_lock<std::mutex> locker(_lock); + _prevValue = prev; + while (_runFlag) { + if (_provided) { + _blocked.store(false); + _provided.store(false); + return _nextValue; + } + _blocked.store(true); + _blockedCond.notify_all(); + auto r = _providedCond.wait_for(locker, oneSec); + (void)r; + } + return TimeStamp(0); +} + +void +MockTick::kill() +{ + std::unique_lock<std::mutex> locker(_lock); + _runFlag.store(false); + _blockedCond.notify_all(); + _providedCond.notify_all(); +} + +void +MockTick::provide(TimeStamp value) +{ + std::unique_lock<std::mutex> locker(_lock); + _nextValue = value; + _blocked.store(false); + _provided.store(true); + _providedCond.notify_all(); +} + +TimeStamp +MockTick::waitUntilBlocked() +{ + std::unique_lock<std::mutex> locker(_lock); + while (_runFlag) { + if (_blocked) { + return _prevValue; + } + auto r = _blockedCond.wait_for(locker, oneSec); + (void)r; + } + return TimeStamp(0); +} + +MockTick::MockTick() + : _lock(), + _runFlag(true), + _provided(false), + _blocked(false), + _providedCond(), + _blockedCond(), + _nextValue(0.0), + _prevValue(0.0) +{} + +} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.h b/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.h new file mode 100644 index 00000000000..d9338cfea23 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/mock_tick.h @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "clock.h" + +#include <atomic> +#include <condition_variable> +#include <mutex> + +namespace vespalib::metrics { + +// share the MockTick between the tested and the tester. +class MockTick : public Tick { +private: + std::mutex _lock; + std::atomic<bool> _runFlag; + std::atomic<bool> _provided; + std::atomic<bool> _blocked; + std::condition_variable _providedCond; + std::condition_variable _blockedCond; + TimeStamp _nextValue; + TimeStamp _prevValue; +public: + MockTick(); + TimeStamp next(TimeStamp prev) override; + void kill() override; + + void provide(TimeStamp value); + TimeStamp waitUntilBlocked(); +}; + +} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.h b/staging_vespalib/src/vespa/vespalib/metrics/point.h index 3ef66f731f6..e50283bd94c 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/point.h @@ -3,8 +3,7 @@ #include "handle.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { /** * Opaque handle representing an unique N-dimensional point @@ -16,4 +15,3 @@ public: }; } // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp index 6818050233f..51908ae235e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp @@ -30,16 +30,16 @@ PointMap::operator< (const PointMap &other) const m != _map.end(); ++m, ++o) { - const Dimension& d1 = m->first; - const Dimension& d2 = o->first; - if (d1 != d2) { - return d1 < d2; - } - const Label &l1 = m->second; - const Label &l2 = o->second; - if (l1 != l2) { - return l1 < l2; - } + const Dimension& d1 = m->first; + const Dimension& d2 = o->first; + if (d1 != d2) { + return d1 < d2; + } + const Label &l1 = m->second; + const Label &l2 = o->second; + if (l1 != l2) { + return l1 < l2; + } } // equal return false; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp index ec2b8bbc66f..90c9ef0eab6 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simple_metrics_manager.h" +#include "simple_tick.h" #include <vespa/log/log.h> LOG_SETUP(".vespalib.metrics.simple_metrics_manager"); @@ -9,22 +10,23 @@ namespace metrics { using Guard = std::lock_guard<std::mutex>; -WallclockTimeSupplier timeSupplier; - -SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config) +SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config, + Tick::UP tick_supplier) : _metricNames(), _dimensionNames(), _labelValues(), _pointMaps(), - _currentBucket(), - _startTime(timeSupplier.now_stamp()), + _currentSamples(), + _tickSupplier(std::move(tick_supplier)), + _startTime(_tickSupplier->first()), _curTime(_startTime), _collectCnt(0), _buckets(), _firstBucket(0), _maxBuckets(config.sliding_window_seconds), _totalsBucket(0, _startTime, _startTime), - _ticker(this) + _runFlag(true), + _thread(&SimpleMetricsManager::tickerLoop, this) { if (_maxBuckets < 1) _maxBuckets = 1; Point empty = pointFrom(PointMap::BackingMap()); @@ -33,15 +35,23 @@ SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config) SimpleMetricsManager::~SimpleMetricsManager() { - _ticker.stop(); + _tickSupplier->kill(); + stopThread(); } - std::shared_ptr<MetricsManager> SimpleMetricsManager::create(const SimpleManagerConfig &config) { return std::shared_ptr<MetricsManager>( - new SimpleMetricsManager(config)); + new SimpleMetricsManager(config, std::make_unique<SimpleTick>())); +} + +std::shared_ptr<MetricsManager> +SimpleMetricsManager::createForTest(const SimpleManagerConfig &config, + Tick::UP tick_supplier) +{ + return std::shared_ptr<MetricsManager>( + new SimpleMetricsManager(config, std::move(tick_supplier))); } Counter @@ -49,7 +59,7 @@ SimpleMetricsManager::counter(const vespalib::string &name) { size_t id = _metricNames.resolve(name); _metricTypes.check(id, name, MetricTypes::MetricType::COUNTER); - LOG(debug, "counter with metric name %s -> %zd", name.c_str(), id); + LOG(debug, "counter with metric name %s -> %zu", name.c_str(), id); return Counter(shared_from_this(), MetricName(id)); } @@ -58,7 +68,7 @@ SimpleMetricsManager::gauge(const vespalib::string &name) { size_t id = _metricNames.resolve(name); _metricTypes.check(id, name, MetricTypes::MetricType::GAUGE); - LOG(debug, "gauge with metric name %s -> %zd", name.c_str(), id); + LOG(debug, "gauge with metric name %s -> %zu", name.c_str(), id); return Gauge(shared_from_this(), MetricName(id)); } @@ -67,12 +77,13 @@ SimpleMetricsManager::mergeBuckets() { Guard bucketsGuard(_bucketsLock); if (_buckets.size() > 0) { - InternalTimeStamp startTime = _buckets[_firstBucket].startTime; + TimeStamp startTime = _buckets[_firstBucket].startTime; Bucket merger(0, startTime, startTime); for (size_t i = 0; i < _buckets.size(); i++) { size_t off = (_firstBucket + i) % _buckets.size(); merger.merge(_buckets[off]); } + merger.padMetrics(_totalsBucket); return merger; } // no data @@ -91,17 +102,17 @@ SimpleMetricsManager::snapshotFrom(const Bucket &bucket) { std::vector<PointSnapshot> points; - double s = timeSupplier.stamp_to_s(bucket.startTime); - double e = timeSupplier.stamp_to_s(bucket.endTime); + double s = bucket.startTime.count(); + double e = bucket.endTime.count(); Snapshot snap(s, e); { for (size_t i = 0; i < _pointMaps.size(); ++i) { - const PointMap::BackingMap &map = _pointMaps.lookup(i).backingMap(); - PointSnapshot point; - for (const PointMap::BackingMap::value_type &kv : map) { - point.dimensions.emplace_back(nameFor(kv.first), valueFor(kv.second)); - } - snap.add(point); + const PointMap::BackingMap &map = _pointMaps.lookup(i).backingMap(); + PointSnapshot point; + for (const PointMap::BackingMap::value_type &kv : map) { + point.dimensions.emplace_back(nameFor(kv.first), valueFor(kv.second)); + } + snap.add(point); } } for (const CounterAggregator& entry : bucket.counters) { @@ -136,11 +147,11 @@ SimpleMetricsManager::totalSnapshot() } void -SimpleMetricsManager::collectCurrentBucket(InternalTimeStamp prev, - InternalTimeStamp curr) +SimpleMetricsManager::collectCurrentSamples(TimeStamp prev, + TimeStamp curr) { CurrentSamples samples; - _currentBucket.extract(samples); + _currentSamples.extract(samples); Bucket newBucket(++_collectCnt, prev, curr); newBucket.merge(samples); @@ -158,7 +169,7 @@ Dimension SimpleMetricsManager::dimension(const vespalib::string &name) { size_t id = _dimensionNames.resolve(name); - LOG(debug, "dimension name %s -> %zd", name.c_str(), id); + LOG(debug, "dimension name %s -> %zu", name.c_str(), id); return Dimension(id); } @@ -166,7 +177,7 @@ Label SimpleMetricsManager::label(const vespalib::string &value) { size_t id = _labelValues.resolve(value); - LOG(debug, "label value %s -> %zd", value.c_str(), id); + LOG(debug, "label value %s -> %zu", value.c_str(), id); return Label(id); } @@ -184,14 +195,29 @@ SimpleMetricsManager::pointFrom(PointMap::BackingMap map) return Point(id); } + +void +SimpleMetricsManager::tickerLoop() +{ + while (_runFlag) { + TimeStamp now = _tickSupplier->next(_curTime); + tick(now); + } +} + +void +SimpleMetricsManager::stopThread() +{ + _runFlag.store(false); + _thread.join(); +} + void -SimpleMetricsManager::tick() +SimpleMetricsManager::tick(TimeStamp now) { - Guard guard(_tickLock); - InternalTimeStamp prev = _curTime; - InternalTimeStamp curr = timeSupplier.now_stamp(); - collectCurrentBucket(prev, curr); - _curTime = curr; + TimeStamp prev = _curTime; + collectCurrentSamples(prev, now); + _curTime = now; } } // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h index 0d855e63660..aa2efba49be 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <atomic> +#include <mutex> #include <memory> #include <thread> #include <vespa/vespalib/stllike/string.h> @@ -12,7 +14,6 @@ #include "clock.h" #include "point_map_collection.h" #include "bucket.h" -#include "ticker_thread.h" namespace vespalib { namespace metrics { @@ -43,10 +44,11 @@ private: const vespalib::string& nameFor(Dimension dimension) { return _dimensionNames.lookup(dimension.id()); } const vespalib::string& valueFor(Label label) { return _labelValues.lookup(label.id()); } - CurrentSamples _currentBucket; + CurrentSamples _currentSamples; - InternalTimeStamp _startTime; - InternalTimeStamp _curTime; + Tick::UP _tickSupplier; + TimeStamp _startTime; + TimeStamp _curTime; std::mutex _bucketsLock; size_t _collectCnt; @@ -55,18 +57,24 @@ private: size_t _maxBuckets; Bucket _totalsBucket; - std::mutex _tickLock; - TickerThread _ticker; - void collectCurrentBucket(InternalTimeStamp prev, InternalTimeStamp curr); + std::atomic<bool> _runFlag; + std::thread _thread; + void tickerLoop(); + void stopThread(); + void tick(TimeStamp now); // called once per second from another thread + + void collectCurrentSamples(TimeStamp prev, TimeStamp curr); Bucket mergeBuckets(); Bucket totalsBucket(); Snapshot snapshotFrom(const Bucket &bucket); - SimpleMetricsManager(const SimpleManagerConfig &config); + SimpleMetricsManager(const SimpleManagerConfig &config, + Tick::UP tick_supplier); public: ~SimpleMetricsManager(); static std::shared_ptr<MetricsManager> create(const SimpleManagerConfig &config); - + static std::shared_ptr<MetricsManager> createForTest(const SimpleManagerConfig &config, + Tick::UP tick_supplier); Counter counter(const vespalib::string &name) override; Gauge gauge(const vespalib::string &name) override; Dimension dimension(const vespalib::string &name) override; @@ -78,14 +86,12 @@ public: // for use from Counter only void add(Counter::Increment inc) override { - _currentBucket.add(inc); + _currentSamples.add(inc); } // for use from Gauge only void sample(Gauge::Measurement value) override { - _currentBucket.sample(value); + _currentSamples.sample(value); } - - void tick(); // called once per second from another thread }; } // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp new file mode 100644 index 00000000000..c71655ff7d0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "simple_tick.h" + +namespace vespalib::metrics { + +namespace { + +const TimeStamp oneSec{1.0}; + +TimeStamp now() +{ + using Clock = std::chrono::system_clock; + Clock::time_point now = Clock::now(); + return now.time_since_epoch(); +} + +} // namespace <unnamed> + +SimpleTick::SimpleTick() + : _lock(), _runFlag(true), _cond() +{} + +TimeStamp +SimpleTick::next(TimeStamp prev) +{ + std::unique_lock<std::mutex> locker(_lock); + while (_runFlag) { + TimeStamp curr = now(); + if (curr - prev >= oneSec) { + return curr; + } + if (curr <= prev) { + _cond.wait_for(locker, oneSec); + } else { + _cond.wait_for(locker, oneSec - (curr - prev)); + } + } + return now(); +} + +void +SimpleTick::kill() +{ + std::unique_lock<std::mutex> locker(_lock); + _runFlag.store(false); + _cond.notify_all(); +} + +} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h new file mode 100644 index 00000000000..653944ea257 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "clock.h" + +#include <atomic> +#include <condition_variable> +#include <mutex> + +namespace vespalib::metrics { + +class SimpleTick : public Tick { +private: + std::mutex _lock; + std::atomic<bool> _runFlag; + std::condition_variable _cond; +public: + SimpleTick(); + TimeStamp next(TimeStamp prev) override; + void kill() override; +}; + +} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h index 406df44d894..0e9915cfb6d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h @@ -56,7 +56,7 @@ public: : _name(n), _point(p), _observedCount(c.observedCount), - _averageValue(c.sumValue / c.observedCount), + _averageValue(c.sumValue / (c.observedCount > 0 ? c.observedCount : 1)), _minValue(c.minValue), _maxValue(c.maxValue), _lastValue(c.lastValue) diff --git a/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.cpp b/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.cpp deleted file mode 100644 index 9916b7f8884..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "test_time_supplier.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.h b/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.h deleted file mode 100644 index 33142b3202d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/test_time_supplier.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <cstddef> - -namespace vespalib { -namespace metrics { - -class TestTimeSupplier { -private: - mutable size_t _cnt; -public: - typedef size_t TimeStamp; - TimeStamp now_stamp() const { return ++_cnt; } - double stamp_to_s(TimeStamp stamp) const { return (double)stamp; } - TestTimeSupplier() : _cnt(0) {} -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp index 3e05b9e45e4..e69de29bb2d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ticker_thread.h" -#include "simple_metrics_manager.h" -#include <chrono> - -#include <vespa/log/log.h> -LOG_SETUP(".vespalib.metrics.ticker_thread"); - -namespace vespalib { -namespace metrics { - -void -TickerThread::tickerLoop() -{ - const std::chrono::seconds oneSec{1}; - std::unique_lock<std::mutex> locker(_lock); - while (_runFlag) { - auto r = _cond.wait_for(locker, oneSec); - if (r == std::cv_status::timeout) { - _owner->tick(); - } - } -} - -void -TickerThread::stop() -{ - if (_runFlag) { - std::unique_lock<std::mutex> locker(_lock); - _runFlag.store(false); - _cond.notify_all(); - locker.unlock(); - _thread.join(); - } -} - - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h index 5ecb0714d2c..e69de29bb2d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <atomic> -#include <condition_variable> -#include <mutex> -#include <thread> - -namespace vespalib { -namespace metrics { - -class SimpleMetricsManager; - -class TickerThread { -private: - SimpleMetricsManager *_owner; - std::mutex _lock; - std::atomic<bool> _runFlag; - std::condition_variable _cond; - std::thread _thread; - - void tickerLoop(); -public: - TickerThread(SimpleMetricsManager * owner) - : _owner(owner), - _runFlag(true), - _thread(&TickerThread::tickerLoop, this) - {} - ~TickerThread() { stop(); } - - void stop(); -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.cpp b/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.cpp deleted file mode 100644 index 9d708e7f9be..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "time_supplier.h" -#include <stdlib.h> - -namespace vespalib::metrics { - -// these should never be used for anything: - -TimeSupplier::TimeStamp -TimeSupplier::now_stamp() const -{ abort(); } - -double -TimeSupplier::stamp_to_s(TimeStamp) const -{ abort(); } - -TimeSupplier::TimeSupplier() -{ abort(); } - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.h b/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.h deleted file mode 100644 index 578a97f1a90..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/time_supplier.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace vespalib::metrics { - -/** - * This is the API you need to implement in order to be used as a TimeSupplier. - **/ -struct TimeSupplier { - /** - * provide a type that can be used for time stamping, like a time_point: - **/ - typedef int TimeStamp; - - /** - * provide a method that can be called to get a time stamp for "now": - **/ - TimeStamp now_stamp() const; - - /** - * provide a method that can convert the time stamp (obtained from - * above method) into seconds since 1970, as a double: - **/ - double stamp_to_s(TimeStamp) const; - - /** - * constructor will usually be trivial. - **/ - TimeSupplier(); -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.cpp b/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.cpp deleted file mode 100644 index bb2393d0867..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wallclock_time_supplier.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.h b/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.h deleted file mode 100644 index 8fa30fc799c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/wallclock_time_supplier.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <chrono> - -namespace vespalib { -namespace metrics { - -class WallclockTimeSupplier { -private: - using Clock = std::chrono::system_clock; - using seconds = std::chrono::duration<double>; -public: - typedef Clock::time_point TimeStamp; - TimeStamp now_stamp() const { return Clock::now(); } - double stamp_to_s(TimeStamp stamp) const { - seconds s = stamp.time_since_epoch(); - return s.count(); - } -}; - -} // namespace vespalib::metrics -} // namespace vespalib |