Merge pull request #24309 from vespa-engine/geirst/datastore-free-list

Implement new free list handling for datastores with a simpler API.

8 files changed, 324 insertions, 2 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index df1e1006828..7aafb7c364e 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -58,6 +58,7 @@ vespa_define_module(
     src/tests/datastore/compact_buffer_candidates
     src/tests/datastore/datastore
     src/tests/datastore/fixed_size_hash_map
+    src/tests/datastore/free_list
     src/tests/datastore/sharded_hash_map
     src/tests/datastore/unique_store
     src/tests/datastore/unique_store_dictionary
diff --git a/vespalib/src/tests/datastore/free_list/CMakeLists.txt b/vespalib/src/tests/datastore/free_list/CMakeLists.txt
new file mode 100644
index 00000000000..97aff6d7ae7
--- /dev/null
+++ b/vespalib/src/tests/datastore/free_list/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_datastore_free_list_test_app TEST
+    SOURCES
+    free_list_test.cpp
+    DEPENDS
+    vespalib
+    GTest::GTest
+)
+vespa_add_test(NAME vespalib_datastore_free_list_test_app COMMAND vespalib_datastore_free_list_test_app)
diff --git a/vespalib/src/tests/datastore/free_list/free_list_test.cpp b/vespalib/src/tests/datastore/free_list/free_list_test.cpp
new file mode 100644
index 00000000000..d80020d3dc5
--- /dev/null
+++ b/vespalib/src/tests/datastore/free_list/free_list_test.cpp
@@ -0,0 +1,127 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/datastore/entryref.hpp>
+#include <vespa/vespalib/datastore/free_list.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vector>
+
+using namespace vespalib::datastore;
+
+using MyEntryRef = EntryRefT<8, 4>;
+constexpr uint32_t array_size = 6;
+
+struct FreeListTest : public testing::Test
+{
+    FreeList list;
+    std::atomic<ElemCount> dead_elems;
+    std::vector<BufferFreeList> bufs;
+    FreeListTest()
+        : list(),
+          bufs()
+    {
+        for (size_t i = 0; i < 3; ++i) {
+            bufs.emplace_back(dead_elems);
+            bufs.back().on_active(array_size);
+        }
+    }
+    void TearDown() override {
+        for (auto& buf : bufs) {
+            buf.disable();
+        }
+    }
+    void enable(uint32_t buffer_id) {
+        bufs[buffer_id].enable(list);
+    }
+    void enable_all() {
+        for (auto& buf : bufs) {
+            buf.enable(list);
+        }
+    }
+    void push_entry(MyEntryRef ref) {
+        bufs[ref.bufferId()].push_entry(ref);
+    }
+    MyEntryRef pop_entry() {
+        return {list.pop_entry()};
+    }
+};
+
+TEST_F(FreeListTest, entry_refs_are_reused_in_lifo_order)
+{
+    enable(0);
+    push_entry({10, 0});
+    push_entry({11, 0});
+    push_entry({12, 0});
+    EXPECT_EQ(MyEntryRef(12, 0), pop_entry());
+    EXPECT_EQ(MyEntryRef(11, 0), pop_entry());
+    EXPECT_EQ(MyEntryRef(10, 0), pop_entry());
+}
+
+TEST_F(FreeListTest, buffer_free_list_attaches_and_detaches_from_free_list)
+{
+    enable(0);
+    EXPECT_TRUE(list.empty());
+    push_entry({10, 0});
+    EXPECT_EQ(1, list.size());
+    push_entry({11, 0});
+    pop_entry();
+    EXPECT_EQ(1, list.size());
+    pop_entry();
+    EXPECT_TRUE(list.empty());
+}
+
+TEST_F(FreeListTest, disable_clears_all_entry_refs_and_detaches_from_free_list)
+{
+    enable(0);
+    push_entry({10, 0});
+    EXPECT_EQ(1, list.size());
+    EXPECT_FALSE(bufs[0].empty());
+    EXPECT_TRUE(bufs[0].enabled());
+
+    bufs[0].disable();
+    EXPECT_TRUE(list.empty());
+    EXPECT_TRUE(bufs[0].empty());
+    EXPECT_FALSE(bufs[0].enabled());
+}
+
+TEST_F(FreeListTest, buffer_free_lists_are_reused_in_lifo_order)
+{
+    enable_all();
+    EXPECT_TRUE(list.empty());
+    push_entry({10, 0});
+    EXPECT_EQ(1, list.size());
+    push_entry({11, 0});
+    push_entry({20, 1});
+    EXPECT_EQ(2, list.size());
+    push_entry({21, 1});
+    push_entry({30, 2});
+    EXPECT_EQ(3, list.size());
+    push_entry({31, 2});
+
+    EXPECT_EQ(MyEntryRef(31, 2), pop_entry());
+    EXPECT_EQ(MyEntryRef(30, 2), pop_entry());
+    EXPECT_EQ(2, list.size());
+    EXPECT_EQ(MyEntryRef(21, 1), pop_entry());
+    EXPECT_EQ(MyEntryRef(20, 1), pop_entry());
+    EXPECT_EQ(1, list.size());
+    EXPECT_EQ(MyEntryRef(11, 0), pop_entry());
+
+    push_entry({32, 2});
+    EXPECT_EQ(2, list.size());
+
+    EXPECT_EQ(MyEntryRef(32, 2), pop_entry());
+    EXPECT_EQ(1, list.size());
+    EXPECT_EQ(MyEntryRef(10, 0), pop_entry());
+    EXPECT_TRUE(list.empty());
+}
+
+TEST_F(FreeListTest, dead_elems_count_is_updated_when_popping_an_entry)
+{
+    enable(0);
+    push_entry({10, 0});
+    dead_elems.store(18, std::memory_order_relaxed);
+    pop_entry();
+    EXPECT_EQ(18 - array_size, dead_elems.load(std::memory_order_relaxed));
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
index f0f548c41fe..a2b561b6792 100644
--- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
@@ -4,16 +4,19 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT
     array_store.cpp
     array_store_config.cpp
     atomic_entry_ref.cpp
+    buffer_free_list.cpp
     buffer_type.cpp
     bufferstate.cpp
+    compact_buffer_candidates.cpp
     compacting_buffers.cpp
     compaction_strategy.cpp
-    compact_buffer_candidates.cpp
+    compaction_strategy.cpp
     datastore.cpp
     datastorebase.cpp
-    entryref.cpp
     entry_ref_filter.cpp
+    entryref.cpp
     fixed_size_hash_map.cpp
+    free_list.cpp
     large_array_buffer_type.cpp
     sharded_hash_map.cpp
     small_array_buffer_type.cpp
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp
new file mode 100644
index 00000000000..0a440e3e867
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp
@@ -0,0 +1,56 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "buffer_free_list.h"
+#include "free_list.h"
+#include <cassert>
+
+namespace vespalib::datastore {
+
+void
+BufferFreeList::attach()
+{
+    assert(_free_list != nullptr);
+    _free_list->attach(*this);
+}
+
+void
+BufferFreeList::detach()
+{
+    assert(_free_list != nullptr);
+    _free_list->detach(*this);
+}
+
+BufferFreeList::BufferFreeList(std::atomic<ElemCount>& dead_elems)
+    : _dead_elems(dead_elems),
+      _array_size(0),
+      _free_list(),
+      _free_refs()
+{
+}
+
+BufferFreeList::~BufferFreeList()
+{
+    assert(_free_list == nullptr);
+    assert(_free_refs.empty());
+}
+
+void
+BufferFreeList::enable(FreeList& free_list)
+{
+    assert(_free_list == nullptr);
+    assert(_free_refs.empty());
+    _free_list = &free_list;
+}
+
+void
+BufferFreeList::disable()
+{
+    if (!empty()) {
+        detach();
+        EntryRefArray().swap(_free_refs);
+    }
+    _free_list = nullptr;
+}
+
+}
+
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h
new file mode 100644
index 00000000000..9ad24f7f03e
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h
@@ -0,0 +1,56 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "entryref.h"
+#include "buffer_type.h"
+#include <vespa/vespalib/util/array.h>
+
+namespace vespalib::datastore {
+
+class FreeList;
+
+/**
+ * Class containing the free list for a single buffer.
+ *
+ * The free list is a stack of EntryRef's that can be reused.
+ */
+class BufferFreeList {
+private:
+    using EntryRefArray = vespalib::Array<EntryRef>;
+
+    std::atomic<ElemCount>& _dead_elems;
+    uint32_t _array_size;
+    FreeList* _free_list;
+    EntryRefArray _free_refs;
+
+    void attach();
+    void detach();
+
+public:
+    BufferFreeList(std::atomic<ElemCount>& dead_elems);
+    ~BufferFreeList();
+    void enable(FreeList& free_list);
+    void disable();
+
+    void on_active(uint32_t array_size) { _array_size = array_size; }
+    bool enabled() const { return _free_list != nullptr; }
+    bool empty() const { return _free_refs.empty(); }
+    void push_entry(EntryRef ref) {
+        if (empty()) {
+            attach();
+        }
+        _free_refs.push_back(ref);
+    }
+    EntryRef pop_entry() {
+        EntryRef ret = _free_refs.back();
+        _free_refs.pop_back();
+        if (empty()) {
+            detach();
+        }
+        _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed);
+        return ret;
+    }
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list.cpp b/vespalib/src/vespa/vespalib/datastore/free_list.cpp
new file mode 100644
index 00000000000..6c96e51241c
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/free_list.cpp
@@ -0,0 +1,36 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "free_list.h"
+#include <cassert>
+
+namespace vespalib::datastore {
+
+FreeList::FreeList()
+    : _free_lists()
+{
+}
+
+FreeList::~FreeList()
+{
+    assert(_free_lists.empty());
+}
+
+void
+FreeList::attach(BufferFreeList& buf_list)
+{
+    _free_lists.push_back(&buf_list);
+}
+
+void
+FreeList::detach(BufferFreeList& buf_list)
+{
+    if (!_free_lists.empty() && (_free_lists.back() == &buf_list)) {
+        _free_lists.pop_back();
+        return;
+    }
+    auto itr = std::find(_free_lists.begin(), _free_lists.end(), &buf_list);
+    assert(itr != _free_lists.end());
+    _free_lists.erase(itr);
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list.h b/vespalib/src/vespa/vespalib/datastore/free_list.h
new file mode 100644
index 00000000000..39e1713e47d
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/free_list.h
@@ -0,0 +1,34 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "buffer_free_list.h"
+#include "entryref.h"
+#include <vector>
+
+namespace vespalib::datastore {
+
+/**
+ * Class containing the free list for a single buffer type id.
+ *
+ * This consists of a stack of buffer free lists,
+ * where the newest attached is used when getting an EntryRef for reuse.
+ */
+class FreeList {
+private:
+    std::vector<BufferFreeList*> _free_lists;
+
+public:
+    FreeList();
+    ~FreeList();
+    void attach(BufferFreeList& buf_list);
+    void detach(BufferFreeList& buf_list);
+
+    bool empty() const { return _free_lists.empty(); }
+    size_t size() const { return _free_lists.size(); }
+    EntryRef pop_entry() {
+        return _free_lists.back()->pop_entry();
+    }
+};
+
+}