Move state_server, metrivs and some all executors from staging_vespalib too vespalib.

5 files changed, 399 insertions, 0 deletions

diff --git a/vespalib/src/tests/metrics/CMakeLists.txt b/vespalib/src/tests/metrics/CMakeLists.txt
new file mode 100644
index 00000000000..6019a6c2d4c
--- /dev/null
+++ b/vespalib/src/tests/metrics/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_metrics_test_app TEST
+    SOURCES
+    simple_metrics_test.cpp
+    mock_tick.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_metrics_test_app COMMAND vespalib_metrics_test_app)
+
+vespa_add_executable(vespalib_stablestore_test_app TEST
+    SOURCES
+    stable_store_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_stablestore_test_app COMMAND vespalib_stablestore_test_app)
diff --git a/vespalib/src/tests/metrics/mock_tick.cpp b/vespalib/src/tests/metrics/mock_tick.cpp
new file mode 100644
index 00000000000..c16ef25cfe6
--- /dev/null
+++ b/vespalib/src/tests/metrics/mock_tick.cpp
@@ -0,0 +1,6 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "mock_tick.h"
+
+namespace vespalib::metrics {
+
+} // namespace vespalib::metrics
diff --git a/vespalib/src/tests/metrics/mock_tick.h b/vespalib/src/tests/metrics/mock_tick.h
new file mode 100644
index 00000000000..4d9f6758537
--- /dev/null
+++ b/vespalib/src/tests/metrics/mock_tick.h
@@ -0,0 +1,92 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <vespa/vespalib/metrics/clock.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
+namespace vespalib::metrics {
+
+// used to test clients of the Tick interface
+// values shared between threads are bounded queues with max size 1
+class MockTick : public Tick {
+private:
+    using Guard = std::unique_lock<std::mutex>;
+    struct Value {
+        Value() noexcept : value(0.0), valid(false) {}
+        TimeStamp value;
+        bool    valid;
+    };
+
+    TimeStamp               _first_value;
+    std::mutex              _lock;
+    std::condition_variable _cond;
+    bool                    _alive;
+    Value                   _prev;
+    Value                   _next;
+
+    void push(Value &dst, TimeStamp value) {
+        Guard guard(_lock);
+        while (_alive && dst.valid) {
+            _cond.wait(guard);
+        }
+        dst.value = value;
+        dst.valid = true;
+        _cond.notify_one();
+    }
+
+    TimeStamp pop(Value &src) {
+        Guard guard(_lock);
+        while (_alive && !src.valid) {
+            _cond.wait(guard);
+        }
+        src.valid = false;
+        _cond.notify_one();
+        return src.value;
+    }
+
+    TimeStamp peek(const Value &src) {
+        Guard guard(_lock);
+        while (_alive && !src.valid) {
+            _cond.wait(guard);
+        }
+        return src.value;
+    }
+
+public:
+    explicit MockTick(TimeStamp first_value) noexcept
+        : _first_value(first_value), _lock(), _cond(), _alive(true), _prev(), _next() {}
+    TimeStamp first() override { return _first_value; }
+    TimeStamp next(TimeStamp prev) override {
+        push(_prev, prev);
+        return pop(_next);
+    }
+    TimeStamp give(TimeStamp next_value) {
+        TimeStamp prev_value = pop(_prev);
+        push(_next, next_value);
+        EXPECT_EQUAL(peek(_prev).count(), next_value.count());
+        return prev_value;
+    }
+    bool alive() const override { return _alive; }
+    void kill() override {
+        Guard guard(_lock);
+        _alive = false;
+        _cond.notify_all();
+    }
+};
+
+// share the MockTick between the tested and the tester.
+class TickProxy : public Tick {
+private:
+    std::shared_ptr<Tick> _tick;
+public:
+    explicit TickProxy(std::shared_ptr<Tick> tick) noexcept : _tick(std::move(tick)) {}
+    TimeStamp first() override { return _tick->first(); }
+    TimeStamp next(TimeStamp prev) override { return _tick->next(prev); }
+    bool alive() const override { return _tick->alive(); }
+    void kill() override { _tick->kill(); }
+};
+
+} // namespace vespalib::metrics
diff --git a/vespalib/src/tests/metrics/simple_metrics_test.cpp b/vespalib/src/tests/metrics/simple_metrics_test.cpp
new file mode 100644
index 00000000000..3006022a43d
--- /dev/null
+++ b/vespalib/src/tests/metrics/simple_metrics_test.cpp
@@ -0,0 +1,219 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/data/slime/json_format.h>
+#include <vespa/vespalib/metrics/simple_metrics.h>
+#include <vespa/vespalib/metrics/simple_metrics_manager.h>
+#include <vespa/vespalib/metrics/stable_store.h>
+#include <vespa/vespalib/metrics/json_formatter.h>
+#include "mock_tick.h"
+#include <stdio.h>
+#include <unistd.h>
+
+using namespace vespalib;
+using namespace vespalib::metrics;
+
+TEST("require that simple metrics gauge merge works")
+{
+    std::pair<MetricId, Point> id(MetricId(42), Point(17));
+    Gauge::Measurement a1(id, 0.0);
+    Gauge::Measurement b1(id, 7.0);
+    Gauge::Measurement b2(id, 9.0);
+    Gauge::Measurement b3(id, 8.0);
+    Gauge::Measurement c1(id, 10.0);
+    Gauge::Measurement c2(id, 1.0);
+
+    GaugeAggregator a(a1), b(b1), c(c1);
+    b.merge(b2);
+    b.merge(b3);
+    c.merge(c2);
+
+    EXPECT_EQUAL(a.observedCount, 1u);
+    EXPECT_EQUAL(a.sumValue, 0.0);
+    EXPECT_EQUAL(a.minValue, 0.0);
+    EXPECT_EQUAL(a.maxValue, 0.0);
+    EXPECT_EQUAL(a.lastValue, 0.0);
+
+    EXPECT_EQUAL(b.observedCount, 3u);
+    EXPECT_EQUAL(b.sumValue, 24.0);
+    EXPECT_EQUAL(b.minValue, 7.0);
+    EXPECT_EQUAL(b.maxValue, 9.0);
+    EXPECT_EQUAL(b.lastValue, 8.0);
+
+    EXPECT_EQUAL(c.observedCount, 2u);
+    EXPECT_EQUAL(c.sumValue, 11.0);
+    EXPECT_EQUAL(c.minValue, 1.0);
+    EXPECT_EQUAL(c.maxValue, 10.0);
+    EXPECT_EQUAL(c.lastValue, 1.0);
+
+    a.minValue = 8;
+
+    a.merge(b);
+    EXPECT_EQUAL(a.observedCount, 4u);
+    EXPECT_EQUAL(a.sumValue, 24.0);
+    EXPECT_EQUAL(a.minValue, 7.0);
+    EXPECT_EQUAL(a.maxValue, 9.0);
+    EXPECT_EQUAL(a.lastValue, 8.0);
+
+    a.merge(b);
+    EXPECT_EQUAL(a.observedCount, 7u);
+    EXPECT_EQUAL(a.sumValue, 48.0);
+    EXPECT_EQUAL(a.minValue, 7.0);
+    EXPECT_EQUAL(a.maxValue, 9.0);
+    EXPECT_EQUAL(a.lastValue, 8.0);
+
+    a.merge(c);
+    EXPECT_EQUAL(a.observedCount, 9u);
+    EXPECT_EQUAL(a.sumValue, 59.0);
+    EXPECT_EQUAL(a.minValue, 1.0);
+    EXPECT_EQUAL(a.maxValue, 10.0);
+    EXPECT_EQUAL(a.lastValue, 1.0);
+}
+
+bool compare_json(const vespalib::string &a, const vespalib::string &b)
+{
+    using vespalib::Memory;
+    using vespalib::slime::JsonFormat;
+
+    Slime slimeA, slimeB;
+    if (! JsonFormat::decode(a, slimeA)) {
+fprintf(stderr, "bad json a:\n>>>%s\n<<<\n", a.c_str());
+        return false;
+    }
+    if (! JsonFormat::decode(b, slimeB)) {
+fprintf(stderr, "bad json b\n");
+        return false;
+    }
+    if (!(slimeA == slimeB)) {
+fprintf(stderr, "compares unequal:\n[A]\n%s\n[B]\n%s\n", a.c_str(), b.c_str());
+    }
+    return slimeA == slimeB;
+}
+
+void check_json(const vespalib::string &actual)
+{
+    vespalib::string expect = "{"
+    "   snapshot: { from: 1, to: 4 },"
+    "   values: [ { name: 'foo',"
+    "       values: { count: 17, rate: 4.85714 }"
+    "   }, {"
+    "       name: 'foo',"
+    "       dimensions: { chain: 'default', documenttype: 'music', thread: '0' },"
+    "       values: { count: 4, rate: 1.14286 }"
+    "   }, {"
+    "       name: 'bar',"
+    "       values: { count: 4, rate: 1.14286, average: 42, sum: 168, min: 41, max: 43, last: 42 }"
+    "   }, {"
+    "       name: 'bar',"
+    "       dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '1' },"
+    "       values: { count: 1, rate: 0.285714, average: 14, sum: 14, min: 14, max: 14, last: 14 }"
+    "   }, {"
+    "       name: 'bar',"
+    "       dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '2' },"
+    "       values: { count: 1, rate: 0.285714, average: 11, sum: 11, min: 11, max: 11, last: 11 }"
+    "   } ]"
+    "}";
+    EXPECT_TRUE(compare_json(expect, actual));
+}
+
+
+TEST("use simple_metrics_collector")
+{
+    using namespace vespalib::metrics;
+    SimpleManagerConfig cf;
+    cf.sliding_window_seconds = 5;
+    std::shared_ptr<MockTick> ticker = std::make_shared<MockTick>(TimeStamp(1.0));
+    auto manager = SimpleMetricsManager::createForTest(cf, std::make_unique<TickProxy>(ticker));
+
+    Counter myCounter = manager->counter("foo", "no description");
+    myCounter.add();
+    myCounter.add(16);
+
+    Gauge myGauge = manager->gauge("bar", "dummy description");
+    myGauge.sample(42.0);
+    myGauge.sample(41.0);
+    myGauge.sample(43.0);
+    myGauge.sample(42.0);
+
+    EXPECT_EQUAL(1.0, ticker->give(TimeStamp(2.0)).count());
+
+    Snapshot snap1 = manager->snapshot();
+    EXPECT_EQUAL(1.0, snap1.startTime());
+    EXPECT_EQUAL(2.0, snap1.endTime());
+
+    EXPECT_EQUAL(1u, snap1.counters().size());
+    EXPECT_EQUAL("foo", snap1.counters()[0].name());
+    EXPECT_EQUAL(17u, snap1.counters()[0].count());
+
+    EXPECT_EQUAL(1u, snap1.gauges().size());
+    EXPECT_EQUAL("bar", snap1.gauges()[0].name());
+    EXPECT_EQUAL(4u, snap1.gauges()[0].observedCount());
+    EXPECT_EQUAL(41.0, snap1.gauges()[0].minValue());
+    EXPECT_EQUAL(43.0, snap1.gauges()[0].maxValue());
+    EXPECT_EQUAL(42.0, snap1.gauges()[0].lastValue());
+
+    Point one = manager->pointBuilder()
+            .bind("chain", "default")
+            .bind("documenttype", "music")
+            .bind("thread", "0").build();
+    PointBuilder b2 = manager->pointBuilder();
+    b2.bind("chain", "vespa")
+      .bind("documenttype", "blogpost");
+    b2.bind("thread", "1");
+    Point two = b2.build();
+    EXPECT_EQUAL(one.id(), 1u);
+    EXPECT_EQUAL(two.id(), 2u);
+
+    Point anotherOne = manager->pointBuilder()
+            .bind("chain", "default")
+            .bind("documenttype", "music")
+            .bind("thread", "0");
+    EXPECT_EQUAL(anotherOne.id(), 1u);
+
+    Point three = manager->pointBuilder(two).bind("thread", "2");
+    EXPECT_EQUAL(three.id(), 3u);
+
+    myCounter.add(3, one);
+    myCounter.add(one);
+    myGauge.sample(14.0, two);
+    myGauge.sample(11.0, three);
+
+    EXPECT_EQUAL(2.0, ticker->give(TimeStamp(4.5)).count());
+
+    Snapshot snap2 = manager->snapshot();
+    EXPECT_EQUAL(1.0, snap2.startTime());
+    EXPECT_EQUAL(4.5, snap2.endTime());
+    EXPECT_EQUAL(2u, snap2.counters().size());
+    EXPECT_EQUAL(3u, snap2.gauges().size());
+
+    JsonFormatter fmt2(snap2);
+    check_json(fmt2.asString());
+
+    // flush sliding window
+    for (int i = 5; i <= 10; ++i) {
+        ticker->give(TimeStamp(i));
+    }
+    Snapshot snap3 = manager->snapshot();
+    EXPECT_EQUAL(5.0, snap3.startTime());
+    EXPECT_EQUAL(10.0, snap3.endTime());
+    EXPECT_EQUAL(2u, snap3.counters().size());
+    EXPECT_EQUAL(0u, snap3.counters()[0].count());
+    EXPECT_EQUAL(0u, snap3.counters()[1].count());
+    EXPECT_EQUAL(3u, snap3.gauges().size());
+    EXPECT_EQUAL(0u, snap3.gauges()[0].observedCount());
+    EXPECT_EQUAL(0u, snap3.gauges()[1].observedCount());
+    EXPECT_EQUAL(0u, snap3.gauges()[2].observedCount());
+
+    Snapshot snap4 = manager->totalSnapshot();
+    EXPECT_EQUAL(1.0,    snap4.startTime());
+    EXPECT_EQUAL(10.0,   snap4.endTime());
+    EXPECT_EQUAL(2u,     snap4.counters().size());
+    EXPECT_NOT_EQUAL(0u, snap4.counters()[0].count());
+    EXPECT_NOT_EQUAL(0u, snap4.counters()[1].count());
+    EXPECT_EQUAL(3u,     snap4.gauges().size());
+    EXPECT_NOT_EQUAL(0u, snap4.gauges()[0].observedCount());
+    EXPECT_NOT_EQUAL(0u, snap4.gauges()[1].observedCount());
+    EXPECT_NOT_EQUAL(0u, snap4.gauges()[2].observedCount());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/metrics/stable_store_test.cpp b/vespalib/src/tests/metrics/stable_store_test.cpp
new file mode 100644
index 00000000000..cead112069f
--- /dev/null
+++ b/vespalib/src/tests/metrics/stable_store_test.cpp
@@ -0,0 +1,65 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/metrics/simple_metrics.h>
+#include <vespa/vespalib/metrics/simple_metrics_manager.h>
+#include <vespa/vespalib/metrics/stable_store.h>
+#include <vespa/vespalib/metrics/json_formatter.h>
+#include <stdio.h>
+#include <unistd.h>
+
+using namespace vespalib;
+using namespace vespalib::metrics;
+
+struct Foo {
+    int a;
+    char *p;
+    explicit Foo(int v) : a(v), p(nullptr) {}
+    bool operator==(const Foo &other) const {
+        return a == other.a;
+    }
+};
+
+TEST("require that stable_store works")
+{
+    vespalib::StableStore<Foo> bunch;
+    bunch.add(Foo(1));
+    bunch.add(Foo(2));
+    bunch.add(Foo(3));
+    bunch.add(Foo(5));
+    bunch.add(Foo(8));
+    bunch.add(Foo(13));
+    bunch.add(Foo(21));
+    bunch.add(Foo(34));
+    bunch.add(Foo(55));
+    bunch.add(Foo(89));
+
+    EXPECT_EQUAL(bunch.size(), 10u);
+
+    int sum = 0;
+
+    bunch.for_each([&sum](const Foo& value) { sum += value.a; });
+    EXPECT_EQUAL(231, sum);
+
+    std::vector<const Foo *> pointers;
+    bunch.for_each([&pointers](const Foo& value)
+                   { pointers.push_back(&value); });
+    EXPECT_EQUAL(1, pointers[0]->a);
+    EXPECT_EQUAL(2, pointers[1]->a);
+    EXPECT_EQUAL(55, pointers[8]->a);
+    EXPECT_EQUAL(89, pointers[9]->a);
+
+    for (int i = 0; i < 20000; ++i) {
+        bunch.add(Foo(i));
+    }
+    bunch.for_each([&sum](const Foo& value) { sum -= value.a; });
+    EXPECT_EQUAL(-199990000, sum);
+
+    std::vector<const Foo *> after;
+    bunch.for_each([&after](const Foo& value)
+                   { if (after.size() < 10) after.push_back(&value); });
+
+    EXPECT_EQUAL(pointers[0], after[0]);
+    EXPECT_EQUAL(pointers[9], after[9]);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }