Publish

10 files changed, 353 insertions, 0 deletions

diff --git a/vespalib/src/tests/util/generationhandler/.gitignore b/vespalib/src/tests/util/generationhandler/.gitignore
new file mode 100644
index 00000000000..c98a32727d9
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+generationhandler_test
+vespalib_generationhandler_test_app
diff --git a/vespalib/src/tests/util/generationhandler/CMakeLists.txt b/vespalib/src/tests/util/generationhandler/CMakeLists.txt
new file mode 100644
index 00000000000..98c0a59d006
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_generationhandler_test_app
+    SOURCES
+    generationhandler_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_generationhandler_test_app COMMAND vespalib_generationhandler_test_app)
diff --git a/vespalib/src/tests/util/generationhandler/DESC b/vespalib/src/tests/util/generationhandler/DESC
new file mode 100644
index 00000000000..5b4befa2c15
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler/DESC
@@ -0,0 +1 @@
+generationhandler test. Take a look at generationhandler_test.cpp for details.
diff --git a/vespalib/src/tests/util/generationhandler/FILES b/vespalib/src/tests/util/generationhandler/FILES
new file mode 100644
index 00000000000..07c356893a5
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler/FILES
@@ -0,0 +1 @@
+generationhandler_test.cpp
diff --git a/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp
new file mode 100644
index 00000000000..c3d7b874c36
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("generationhandler_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/generationhandler.h>
+#include <deque>
+
+namespace vespalib {
+
+typedef GenerationHandler::Guard GenGuard;
+
+class Test : public vespalib::TestApp {
+private:
+    void requireThatGenerationCanBeIncreased();
+    void requireThatReadersCanTakeGuards();
+    void requireThatGuardsCanBeCopied();
+    void requireThatTheFirstUsedGenerationIsCorrect();
+    void requireThatGenerationCanGrowLarge();
+public:
+    int Main();
+};
+
+void
+Test::requireThatGenerationCanBeIncreased()
+{
+    GenerationHandler gh;
+    EXPECT_EQUAL(0u, gh.getCurrentGeneration());
+    EXPECT_EQUAL(0u, gh.getFirstUsedGeneration());
+    gh.incGeneration();
+    EXPECT_EQUAL(1u, gh.getCurrentGeneration());
+    EXPECT_EQUAL(1u, gh.getFirstUsedGeneration());
+}
+
+void
+Test::requireThatReadersCanTakeGuards()
+{
+    GenerationHandler gh;
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount(0));
+    {
+        GenGuard g1 = gh.takeGuard();
+        EXPECT_EQUAL(1u, gh.getGenerationRefCount(0));
+        {
+            GenGuard g2 = gh.takeGuard();
+            EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+            gh.incGeneration();
+            {
+                GenGuard g3 = gh.takeGuard();
+                EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+                EXPECT_EQUAL(1u, gh.getGenerationRefCount(1));
+                EXPECT_EQUAL(3u, gh.getGenerationRefCount());
+            }
+            EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+            EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+            gh.incGeneration();
+            {
+                GenGuard g3 = gh.takeGuard();
+                EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+                EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+                EXPECT_EQUAL(1u, gh.getGenerationRefCount(2));
+            }
+            EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+            EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+            EXPECT_EQUAL(0u, gh.getGenerationRefCount(2));
+        }
+        EXPECT_EQUAL(1u, gh.getGenerationRefCount(0));
+        EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+        EXPECT_EQUAL(0u, gh.getGenerationRefCount(2));
+    }
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount(0));
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount(2));
+}
+
+void
+Test::requireThatGuardsCanBeCopied()
+{
+    GenerationHandler gh;
+    GenGuard g1 = gh.takeGuard();
+    EXPECT_EQUAL(1u, gh.getGenerationRefCount(0));
+    GenGuard g2(g1);
+    EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+    gh.incGeneration();
+    GenGuard g3 = gh.takeGuard();
+    EXPECT_EQUAL(2u, gh.getGenerationRefCount(0));
+    EXPECT_EQUAL(1u, gh.getGenerationRefCount(1));
+    g3 = g2;
+    EXPECT_EQUAL(3u, gh.getGenerationRefCount(0));
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount(1));
+}
+
+void
+Test::requireThatTheFirstUsedGenerationIsCorrect()
+{
+    GenerationHandler gh;
+    EXPECT_EQUAL(0u, gh.getFirstUsedGeneration());
+    gh.incGeneration();
+    EXPECT_EQUAL(1u, gh.getFirstUsedGeneration());
+    {
+        GenGuard g1 = gh.takeGuard();
+        gh.incGeneration();
+        EXPECT_EQUAL(true, gh.hasReaders());
+        EXPECT_EQUAL(1u, gh.getFirstUsedGeneration());
+    }
+    EXPECT_EQUAL(1u, gh.getFirstUsedGeneration());
+    gh.updateFirstUsedGeneration();	// Only writer should call this
+    EXPECT_EQUAL(false, gh.hasReaders());
+    EXPECT_EQUAL(2u, gh.getFirstUsedGeneration());
+    {
+        GenGuard g1 = gh.takeGuard();
+        gh.incGeneration();
+        gh.incGeneration();
+        EXPECT_EQUAL(true, gh.hasReaders());
+        EXPECT_EQUAL(2u, gh.getFirstUsedGeneration());
+        {
+            GenGuard g2 = gh.takeGuard();
+            EXPECT_EQUAL(2u, gh.getFirstUsedGeneration());
+        }
+    }
+    EXPECT_EQUAL(2u, gh.getFirstUsedGeneration());
+    gh.updateFirstUsedGeneration();	// Only writer should call this
+    EXPECT_EQUAL(false, gh.hasReaders());
+    EXPECT_EQUAL(4u, gh.getFirstUsedGeneration());
+}
+
+void
+Test::requireThatGenerationCanGrowLarge()
+{
+    GenerationHandler gh;
+    std::deque<GenGuard> guards;
+    for (size_t i = 0; i < 10000; ++i) {
+        EXPECT_EQUAL(i, gh.getCurrentGeneration());
+        guards.push_back(gh.takeGuard()); // take guard on current generation
+        if (i >= 128) {
+            EXPECT_EQUAL(i - 128, gh.getFirstUsedGeneration());
+            guards.pop_front();
+            EXPECT_EQUAL(128u, gh.getGenerationRefCount());
+        }
+        gh.incGeneration();
+    }
+}
+
+int
+Test::Main()
+{
+    TEST_INIT("generationhandler_test");
+
+    TEST_DO(requireThatGenerationCanBeIncreased());
+    TEST_DO(requireThatReadersCanTakeGuards());
+    TEST_DO(requireThatGuardsCanBeCopied());
+    TEST_DO(requireThatTheFirstUsedGenerationIsCorrect());
+    TEST_DO(requireThatGenerationCanGrowLarge());
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(vespalib::Test);
diff --git a/vespalib/src/tests/util/generationhandler_stress/.gitignore b/vespalib/src/tests/util/generationhandler_stress/.gitignore
new file mode 100644
index 00000000000..bd50526d7cd
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler_stress/.gitignore
@@ -0,0 +1 @@
+vespalib_generation_handler_stress_test_app
diff --git a/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt b/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt
new file mode 100644
index 00000000000..0842ccd35bc
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_generation_handler_stress_test_app
+    SOURCES
+    generation_handler_stress_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_generation_handler_stress_test_app COMMAND vespalib_generation_handler_stress_test_app BENCHMARK)
diff --git a/vespalib/src/tests/util/generationhandler_stress/DESC b/vespalib/src/tests/util/generationhandler_stress/DESC
new file mode 100644
index 00000000000..18f577aa095
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler_stress/DESC
@@ -0,0 +1 @@
+GenerationHandler stress test. Take a look at generation_handler_stress_test.cpp for details.
diff --git a/vespalib/src/tests/util/generationhandler_stress/FILES b/vespalib/src/tests/util/generationhandler_stress/FILES
new file mode 100644
index 00000000000..9c60622e3e9
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler_stress/FILES
@@ -0,0 +1 @@
+generation_handler_stress_test.cpp
diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp
new file mode 100644
index 00000000000..a11cff02d22
--- /dev/null
+++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("generation_handler_stress_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/vespalib/util/generationhandler.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+
+using vespalib::Executor;
+using vespalib::GenerationHandler;
+using vespalib::ThreadStackExecutor;
+
+
+struct WorkContext
+{
+    uint64_t _generation;
+
+    WorkContext()
+        : _generation(0)
+    {
+    }
+};
+
+struct Fixture {
+    GenerationHandler _generationHandler;
+    uint32_t _readThreads;
+    ThreadStackExecutor _writer; // 1 write thread
+    ThreadStackExecutor _readers; // multiple reader threads
+    std::atomic<long> _readSeed;
+    std::atomic<long> _doneWriteWork;
+    std::atomic<long> _doneReadWork;
+    std::atomic<int> _stopRead;
+    bool _reportWork;
+
+    Fixture(uint32_t readThreads = 1);
+
+    ~Fixture();
+
+    void readWork(const WorkContext &context);
+    void writeWork(uint32_t cnt, WorkContext &context);
+    uint32_t getReadThreads() const { return _readThreads; }
+    void stressTest(uint32_t writeCnt);
+
+private:
+    Fixture(const Fixture &index) = delete;
+    Fixture(Fixture &&index) = delete;
+    Fixture &operator=(const Fixture &index) = delete;
+    Fixture &operator=(Fixture &&index) = delete;
+};
+
+
+Fixture::Fixture(uint32_t readThreads)
+    : _generationHandler(),
+      _readThreads(readThreads),
+      _writer(1, 128 * 1024),
+      _readers(readThreads, 128 * 1024),
+      _doneWriteWork(0),
+      _doneReadWork(0),
+      _stopRead(0),
+      _reportWork(false)
+{
+}
+
+
+Fixture::~Fixture()
+{
+    _readers.sync();
+    _readers.shutdown();
+    _writer.sync();
+    _writer.shutdown();
+    if (_reportWork) {
+        LOG(info,
+            "readWork=%ld, writeWork=%ld",
+            _doneReadWork.load(), _doneWriteWork.load());
+    }
+}
+
+
+void
+Fixture::readWork(const WorkContext &context)
+{
+    uint32_t i;
+    uint32_t cnt = std::numeric_limits<uint32_t>::max();
+
+    for (i = 0; i < cnt && _stopRead.load() == 0; ++i) {
+        auto guard = _generationHandler.takeGuard();
+        auto generation = context._generation;
+        EXPECT_GREATER_EQUAL(generation, guard.getGeneration());
+    }
+    _doneReadWork += i;
+    LOG(info, "done %u read work", i);
+}
+
+
+void
+Fixture::writeWork(uint32_t cnt, WorkContext &context)
+{
+    for (uint32_t i = 0; i < cnt; ++i) {
+        context._generation = _generationHandler.getNextGeneration();
+        _generationHandler.incGeneration();
+    }
+    _doneWriteWork += cnt;
+    _stopRead = 1;
+    LOG(info, "done %u write work", cnt);
+}
+
+namespace
+{
+
+class ReadWorkTask : public vespalib::Executor::Task
+{
+    Fixture &_f;
+    std::shared_ptr<WorkContext> _context;
+public:
+    ReadWorkTask(Fixture &f, std::shared_ptr<WorkContext> context)
+        : _f(f),
+          _context(context)
+    {
+    }
+    virtual void run() override { _f.readWork(*_context); }
+};
+
+class WriteWorkTask : public vespalib::Executor::Task
+{
+    Fixture &_f;
+    uint32_t _cnt;
+    std::shared_ptr<WorkContext> _context;
+public:
+    WriteWorkTask(Fixture &f, uint32_t cnt,
+                  std::shared_ptr<WorkContext> context)
+        : _f(f),
+          _cnt(cnt),
+          _context(context)
+    {
+    }
+    virtual void run() override { _f.writeWork(_cnt, *_context); }
+};
+
+}
+
+
+void
+Fixture::stressTest(uint32_t writeCnt)
+{
+    _reportWork = true;
+    uint32_t readThreads = getReadThreads();
+    LOG(info,
+        "starting stress test, 1 write thread, %u read threads, %u writes",
+        readThreads, writeCnt);
+    auto context = std::make_shared<WorkContext>();
+    _writer.execute(std::make_unique<WriteWorkTask>(*this, writeCnt, context));
+    for (uint32_t i = 0; i < readThreads; ++i) {
+        _readers.execute(std::make_unique<ReadWorkTask>(*this, context));
+    }
+}
+
+
+TEST_F("stress test, 2 readers", Fixture(2))
+{
+    f.stressTest(1000000);
+}
+
+TEST_F("stress test, 4 readers", Fixture(4))
+{
+    f.stressTest(1000000);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }