From 80c0e1b687bbe57f5fdbb501580a8cb05af47804 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 11 Feb 2022 10:21:33 +0100 Subject: Add memory allocator to array store. --- .../datastore/array_store/array_store_test.cpp | 20 ++++++++--- .../src/vespa/vespalib/datastore/CMakeLists.txt | 2 ++ .../src/vespa/vespalib/datastore/array_store.cpp | 6 ---- .../src/vespa/vespalib/datastore/array_store.h | 22 ++++-------- .../src/vespa/vespalib/datastore/array_store.hpp | 31 ++++------------- .../vespalib/datastore/large_array_buffer_type.cpp | 20 +++++++++++ .../vespalib/datastore/large_array_buffer_type.h | 39 ++++++++++++++++++++++ .../vespalib/datastore/large_array_buffer_type.hpp | 39 ++++++++++++++++++++++ .../vespalib/datastore/small_array_buffer_type.cpp | 14 ++++++++ .../vespalib/datastore/small_array_buffer_type.h | 37 ++++++++++++++++++++ .../vespalib/datastore/small_array_buffer_type.hpp | 26 +++++++++++++++ 11 files changed, 206 insertions(+), 50 deletions(-) create mode 100644 vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp create mode 100644 vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h create mode 100644 vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp create mode 100644 vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp create mode 100644 vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h create mode 100644 vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp (limited to 'vespalib') diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index c58e357a9a1..7c9f0770a17 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,9 @@ using vespalib::ArrayRef; using generation_t = vespalib::GenerationHandler::generation_t; using MemStats = vespalib::datastore::test::MemStats; using BufferStats = vespalib::datastore::test::BufferStats; +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; namespace { @@ -40,18 +44,21 @@ struct Fixture using value_type = EntryT; using ReferenceStore = vespalib::hash_map; + AllocStats stats; ArrayStoreType store; ReferenceStore refStore; generation_t generation; Fixture(uint32_t maxSmallArraySize, bool enable_free_lists = true) - : store(ArrayStoreConfig(maxSmallArraySize, + : store(std::make_unique(stats), + ArrayStoreConfig(maxSmallArraySize, ArrayStoreConfig::AllocSpec(16, RefT::offsetSize(), 8_Ki, ALLOC_GROW_FACTOR)).enable_free_lists(enable_free_lists)), refStore(), generation(1) {} Fixture(const ArrayStoreConfig &storeCfg) - : store(storeCfg), + : store(std::make_unique(stats), + storeCfg), refStore(), generation(1) {} @@ -162,10 +169,10 @@ TEST("require that we test with trivial and non-trivial types") TEST_F("control static sizes", NumberFixture(3)) { #ifdef _LIBCPP_VERSION - EXPECT_EQUAL(424u, sizeof(f.store)); + EXPECT_EQUAL(440u, sizeof(f.store)); EXPECT_EQUAL(296u, sizeof(NumberFixture::ArrayStoreType::DataStoreType)); #else - EXPECT_EQUAL(456u, sizeof(f.store)); + EXPECT_EQUAL(472u, sizeof(f.store)); EXPECT_EQUAL(328u, sizeof(NumberFixture::ArrayStoreType::DataStoreType)); #endif EXPECT_EQUAL(96u, sizeof(NumberFixture::ArrayStoreType::SmallArrayType)); @@ -447,4 +454,9 @@ TEST_F("require that offset in EntryRefT is within bounds when allocating memory f.assertStoreContent(); } +TEST_F("require that provided memory allocator is used", NumberFixture(3)) +{ + EXPECT_EQUAL(AllocStats(4, 0), f.stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index 5ff1eab61b8..a2f7a4d4ff4 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -12,7 +12,9 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT entryref.cpp entry_ref_filter.cpp fixed_size_hash_map.cpp + large_array_buffer_type.cpp sharded_hash_map.cpp + small_array_buffer_type.cpp unique_store.cpp unique_store_buffer_type.cpp unique_store_string_allocator.cpp diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.cpp b/vespalib/src/vespa/vespalib/datastore/array_store.cpp index 0e03b36d2f9..b03f21402a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.cpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.cpp @@ -5,10 +5,4 @@ namespace vespalib::datastore { -template class BufferType>; -template class BufferType>; -template class BufferType>; -template class BufferType>; -template class BufferType>; - } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index d9b62c310b5..c26a2a318dd 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -9,6 +9,8 @@ #include "entryref.h" #include "atomic_entry_ref.h" #include "i_compaction_context.h" +#include "large_array_buffer_type.h" +#include "small_array_buffer_type.h" #include namespace vespalib::datastore { @@ -32,27 +34,15 @@ public: using SmallArrayType = BufferType; using LargeArray = vespalib::Array; using AllocSpec = ArrayStoreConfig::AllocSpec; - private: - class LargeArrayType : public BufferType { - private: - using ParentType = BufferType; - using ParentType::_emptyEntry; - using CleanContext = typename ParentType::CleanContext; - public: - LargeArrayType(const AllocSpec &spec); - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - }; - - uint32_t _largeArrayTypeId; uint32_t _maxSmallArraySize; DataStoreType _store; - std::vector _smallArrayTypes; - LargeArrayType _largeArrayType; + std::vector> _smallArrayTypes; + LargeArrayBufferType _largeArrayType; using generation_t = vespalib::GenerationHandler::generation_t; - void initArrayTypes(const ArrayStoreConfig &cfg); + void initArrayTypes(std::shared_ptr memory_allocator, const ArrayStoreConfig &cfg); // 1-to-1 mapping between type ids and sizes for small arrays is enforced during initialization. uint32_t getTypeId(size_t arraySize) const { return arraySize; } size_t getArraySize(uint32_t typeId) const { return typeId; } @@ -68,7 +58,7 @@ private: } public: - ArrayStore(const ArrayStoreConfig &cfg); + ArrayStore(std::shared_ptr memory_allocator, const ArrayStoreConfig &cfg); ~ArrayStore(); EntryRef add(const ConstArrayRef &array); ConstArrayRef get(EntryRef ref) const { diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index bbbd52c354d..d3183ac93d0 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -6,40 +6,23 @@ #include "compaction_spec.h" #include "entry_ref_filter.h" #include "datastore.hpp" +#include "large_array_buffer_type.hpp" +#include "small_array_buffer_type.hpp" #include #include namespace vespalib::datastore { -template -ArrayStore::LargeArrayType::LargeArrayType(const AllocSpec &spec) - : BufferType(1, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor) -{ -} - -template -void -ArrayStore::LargeArrayType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) -{ - LargeArray *elem = static_cast(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { - cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); - *elem = _emptyEntry; - ++elem; - } -} - template void -ArrayStore::initArrayTypes(const ArrayStoreConfig &cfg) +ArrayStore::initArrayTypes(std::shared_ptr memory_allocator, const ArrayStoreConfig &cfg) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); _smallArrayTypes.reserve(_maxSmallArraySize); for (uint32_t arraySize = 1; arraySize <= _maxSmallArraySize; ++arraySize) { const AllocSpec &spec = cfg.specForSize(arraySize); - _smallArrayTypes.emplace_back(arraySize, spec.minArraysInBuffer, spec.maxArraysInBuffer, - spec.numArraysForNewBuffer, spec.allocGrowFactor); + _smallArrayTypes.emplace_back(arraySize, spec, memory_allocator); } for (auto & type : _smallArrayTypes) { uint32_t typeId = _store.addType(&type); @@ -48,14 +31,14 @@ ArrayStore::initArrayTypes(const ArrayStoreConfig &cfg) } template -ArrayStore::ArrayStore(const ArrayStoreConfig &cfg) +ArrayStore::ArrayStore(std::shared_ptr memory_allocator, const ArrayStoreConfig &cfg) : _largeArrayTypeId(0), _maxSmallArraySize(cfg.maxSmallArraySize()), _store(), _smallArrayTypes(), - _largeArrayType(cfg.specForSize(0)) + _largeArrayType(cfg.specForSize(0), memory_allocator) { - initArrayTypes(cfg); + initArrayTypes(std::move(memory_allocator), cfg); _store.init_primary_buffers(); if (cfg.enable_free_lists()) { _store.enableFreeLists(); diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp new file mode 100644 index 00000000000..f4ccb27abad --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "large_array_buffer_type.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class BufferType>; +template class BufferType>; +template class BufferType>; +template class BufferType>; +template class BufferType>; + +template class LargeArrayBufferType; +template class LargeArrayBufferType; +template class LargeArrayBufferType; +template class LargeArrayBufferType; +template class LargeArrayBufferType; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h new file mode 100644 index 00000000000..50d15d4a27c --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "array_store_config.h" +#include "buffer_type.h" +#include +#include + +namespace vespalib::alloc { class MemoryAllocator; } + +namespace vespalib::datastore { + +/* + * Class representing buffer type for large arrays in ArrayStore + */ +template +class LargeArrayBufferType : public BufferType> +{ + using AllocSpec = ArrayStoreConfig::AllocSpec; + using ArrayType = Array; + using ParentType = BufferType; + using ParentType::_emptyEntry; + using CleanContext = typename ParentType::CleanContext; + std::shared_ptr _memory_allocator; +public: + LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr memory_allocator) noexcept; + ~LargeArrayBufferType() override; + void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; +}; + +extern template class LargeArrayBufferType; +extern template class LargeArrayBufferType; +extern template class LargeArrayBufferType; +extern template class LargeArrayBufferType; +extern template class LargeArrayBufferType; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp new file mode 100644 index 00000000000..aeeef8166c6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "large_array_buffer_type.h" +#include + +namespace vespalib::datastore { + +template +LargeArrayBufferType::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr memory_allocator) noexcept + : BufferType>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + _memory_allocator(std::move(memory_allocator)) +{ +} + +template +LargeArrayBufferType::~LargeArrayBufferType() = default; + +template +void +LargeArrayBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +{ + ArrayType* elem = static_cast(buffer) + offset; + for (size_t i = 0; i < numElems; ++i) { + cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); + *elem = _emptyEntry; + ++elem; + } +} + +template +const vespalib::alloc::MemoryAllocator* +LargeArrayBufferType::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp new file mode 100644 index 00000000000..06a4c6007a9 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "small_array_buffer_type.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class SmallArrayBufferType; +template class SmallArrayBufferType; +template class SmallArrayBufferType; +template class SmallArrayBufferType; +template class SmallArrayBufferType; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h new file mode 100644 index 00000000000..4e4568c3d16 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "array_store_config.h" +#include "buffer_type.h" +#include + +namespace vespalib::alloc { class MemoryAllocator; } + +namespace vespalib::datastore { + +/* + * Class representing buffer type for small arrays in ArrayStore + */ +template +class SmallArrayBufferType : public BufferType +{ + using AllocSpec = ArrayStoreConfig::AllocSpec; + std::shared_ptr _memory_allocator; +public: + SmallArrayBufferType(const SmallArrayBufferType&) = delete; + SmallArrayBufferType& operator=(const SmallArrayBufferType&) = delete; + SmallArrayBufferType(SmallArrayBufferType&&) noexcept = default; + SmallArrayBufferType& operator=(SmallArrayBufferType&&) noexcept = default; + SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr memory_allocator) noexcept; + ~SmallArrayBufferType() override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; +}; + +extern template class SmallArrayBufferType; +extern template class SmallArrayBufferType; +extern template class SmallArrayBufferType; +extern template class SmallArrayBufferType; +extern template class SmallArrayBufferType; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp new file mode 100644 index 00000000000..414804417eb --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "small_array_buffer_type.h" + +namespace vespalib::datastore { + +template +SmallArrayBufferType::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr memory_allocator) noexcept + : BufferType(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + _memory_allocator(std::move(memory_allocator)) +{ +} + +template +SmallArrayBufferType::~SmallArrayBufferType() = default; + +template +const vespalib::alloc::MemoryAllocator* +SmallArrayBufferType::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +} -- cgit v1.2.3