diff options
author | Tor Egge <Tor.Egge@yahooinc.com> | 2023-06-21 12:11:15 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-21 12:11:15 +0200 |
commit | 9bcd67c4d50c0f5dc9df035a5c1c0ffbcc8f0c1c (patch) | |
tree | 0b79694d4e87fd81c7141b5e659e679afa47f913 /vespalib | |
parent | 06404d0a3dd88ff89d7b5f6356c3c461a42e32b1 (diff) | |
parent | be07722aec1cf08d4444e72283aefd5dffaedfb6 (diff) |
Merge pull request #27499 from vespa-engine/toregge/store-dynamic-array-size-at-start-of-entry
Store dynamic array size size at start of entry.
Diffstat (limited to 'vespalib')
8 files changed, 23 insertions, 20 deletions
diff --git a/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp index c2933dccf18..a703d9b18eb 100644 --- a/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp @@ -163,7 +163,7 @@ std::vector<int> DynamicArrayBufferTypeTest::get_vector(const void* buffer, uint32_t offset) { auto e = BufferType::get_entry(buffer, offset, _entry_size); - auto array_size = BufferType::get_dynamic_array_size(e, _entry_size); + auto array_size = BufferType::get_dynamic_array_size(e); EXPECT_GE(_buffer_type.getArraySize(), array_size); return get_vector(buffer, offset, array_size); } @@ -179,7 +179,7 @@ void DynamicArrayBufferTypeTest::write_entry1() { auto e1 = BufferType::get_entry(_buf.get(), 1, _entry_size); - BufferType::set_dynamic_array_size(e1, _entry_size, 2); + BufferType::set_dynamic_array_size(e1, 2); new (static_cast<void *>(e1)) WrapInt32(42); new (static_cast<void *>(e1 + 1)) WrapInt32(47); new (static_cast<void *>(e1 + 2)) WrapInt32(49); // Not cleaned by clean_hold diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp index f80ba607ce7..fe425b6a45c 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp @@ -88,7 +88,7 @@ Allocator<EntryT, RefT>::alloc_dynamic_array(ConstArrayRef array) for (size_t i = array.size(); i < max_array_size; ++i) { new (static_cast<void *>(buf + i)) EntryT(); } - BufferType::set_dynamic_array_size(buf, entry_size, array.size()); + BufferType::set_dynamic_array_size(buf, array.size()); state.stats().pushed_back(1); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index bff3a5cc5b2..f5c30c90c5b 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -101,7 +101,7 @@ private: template <typename BufferType> ConstArrayRef get_dynamic_array(const void* buffer, size_t offset, uint32_t entry_size) const { auto entry = BufferType::get_entry(buffer, offset, entry_size); - auto size = BufferType::get_dynamic_array_size(entry, entry_size); + auto size = BufferType::get_dynamic_array_size(entry); return ConstArrayRef(entry, size); } ConstArrayRef getLargeArray(RefT ref) const { @@ -122,7 +122,7 @@ public: const BufferAndMeta & bufferAndMeta = _store.getBufferMeta(internalRef.bufferId()); if (bufferAndMeta.getTypeId() != _largeArrayTypeId) [[likely]] { if constexpr (has_dynamic_buffer_type) { - if (_mapper.is_dynamic_buffer(bufferAndMeta.getTypeId())) [[unlikely]] { + if (_mapper.is_dynamic_buffer(bufferAndMeta.getTypeId())) { return get_dynamic_array<typename TypeMapper::DynamicBufferType>(bufferAndMeta.get_buffer_acquire(), internalRef.offset(), bufferAndMeta.get_entry_size()); } } diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h index 7d67d0cddf7..fbd3c2361d1 100644 --- a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h @@ -3,7 +3,9 @@ #pragma once #include "buffer_type.h" +#include "aligner.h" #include "array_store_config.h" +#include <algorithm> #include <memory> namespace vespalib::datastore { @@ -26,6 +28,10 @@ class DynamicArrayBufferType : public BufferTypeBase std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; public: using ElemType = ElemT; + + static constexpr size_t entry_min_align = std::max(alignof(uint32_t), alignof(ElemT)); + using EntryMinAligner = Aligner<entry_min_align>; + static constexpr size_t entry_bias = EntryMinAligner::align(sizeof(uint32_t)); protected: static const ElemType& empty_entry() noexcept; ElemType* get_entry(void *buffer, size_t offset) noexcept { return get_entry(buffer, offset, entry_size()); } @@ -49,10 +55,10 @@ public: const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; static size_t calc_entry_size(size_t array_size) noexcept; static size_t calc_array_size(size_t entry_size) noexcept; - static ElemType* get_entry(void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<ElemType*>(static_cast<char*>(buffer) + offset * entry_size); } - static const ElemType* get_entry(const void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<const ElemType*>(static_cast<const char*>(buffer) + offset * entry_size); } - static uint32_t get_dynamic_array_size(const void *buffer, uint32_t entry_size) noexcept { return *reinterpret_cast<const uint32_t*>(static_cast<const char*>(buffer) + entry_size - sizeof(uint32_t)); } - static void set_dynamic_array_size(void *buffer, uint32_t entry_size, uint32_t array_size) noexcept { *reinterpret_cast<uint32_t*>(static_cast<char*>(buffer) + entry_size - sizeof(uint32_t)) = array_size; } + static ElemType* get_entry(void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<ElemType*>(static_cast<char*>(buffer) + offset * entry_size + entry_bias); } + static const ElemType* get_entry(const void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<const ElemType*>(static_cast<const char*>(buffer) + offset * entry_size + entry_bias); } + static uint32_t get_dynamic_array_size(const ElemType* buffer) noexcept { return *(reinterpret_cast<const uint32_t*>(buffer) - 1); } + static void set_dynamic_array_size(ElemType* buffer, uint32_t array_size) noexcept { *(reinterpret_cast<uint32_t*>(buffer) - 1) = array_size; } }; extern template class DynamicArrayBufferType<char>; diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp index 81d8a850b84..bf3235a6b97 100644 --- a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp @@ -3,8 +3,6 @@ #pragma once #include "dynamic_array_buffer_type.h" -#include "aligner.h" -#include <algorithm> #include <cassert> namespace vespalib::datastore { @@ -26,15 +24,14 @@ template <typename ElemT> size_t DynamicArrayBufferType<ElemT>::calc_entry_size(size_t array_size) noexcept { - Aligner aligner(std::max(alignof(uint32_t), alignof(ElemType))); - return aligner.align(sizeof(ElemType) * array_size + sizeof(uint32_t)); + return EntryMinAligner::align(sizeof(ElemType) * array_size + entry_bias); } template <typename ElemT> size_t DynamicArrayBufferType<ElemT>::calc_array_size(size_t entry_size) noexcept { - return (entry_size - sizeof(uint32_t)) / sizeof(ElemType); + return (entry_size - entry_bias) / sizeof(ElemType); } template <typename ElemT> @@ -59,7 +56,7 @@ DynamicArrayBufferType<ElemT>::fallback_copy(void* new_buffer, const void* old_b for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) { auto d = get_entry(new_buffer, entry_idx); auto s = get_entry(old_buffer, entry_idx); - set_dynamic_array_size(d, entry_size(), get_dynamic_array_size(s, entry_size())); + set_dynamic_array_size(d, get_dynamic_array_size(s)); for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { new (static_cast<void*>(d)) ElemType(*s); ++s; @@ -76,7 +73,7 @@ DynamicArrayBufferType<ElemT>::initialize_reserved_entries(void* buffer, EntryCo const auto& empty = empty_entry(); for (uint32_t entry_idx = 0; entry_idx < reserved_entries; ++entry_idx) { auto e = get_entry(buffer, entry_idx); - set_dynamic_array_size(e, entry_size(), 0); + set_dynamic_array_size(e, 0); for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { new (static_cast<void*>(e)) ElemType(empty); ++e; @@ -92,7 +89,7 @@ DynamicArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCoun const auto& empty = empty_entry(); for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) { auto e = get_entry(buffer, offset + entry_idx); - auto array_size = get_dynamic_array_size(e, entry_size()); + auto array_size = get_dynamic_array_size(e); assert(array_size <= max_array_size); for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { *e = empty; diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp index 6f3e0bc9911..25eb50750ba 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp @@ -112,7 +112,7 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::alloc_dynamic_array(ConstArrayRef a for (size_t i = 0; i < array.size(); ++i) { *(buf + i) = array[i]; } - BufferType::set_dynamic_array_size(buf, entry_size, array.size()); + BufferType::set_dynamic_array_size(buf, array.size()); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp index c6d93e92828..c6e164bd2d4 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp @@ -42,7 +42,7 @@ FreeListRawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size) auto entry_size = _store.get_entry_size(_typeId); assert(_store.getBufferState(ref.bufferId()).getArraySize() >= array_size); EntryT* entry = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); - BufferType::set_dynamic_array_size(entry, entry_size, array_size); + BufferType::set_dynamic_array_size(entry, array_size); return HandleType(ref, entry); } diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp index 5dde8aaa622..91e21aa26e1 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp @@ -41,7 +41,7 @@ RawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size) RefT ref(state.size(), buffer_id); auto entry_size = _store.get_entry_size(_typeId); EntryT* buffer = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); - BufferType::set_dynamic_array_size(buffer, entry_size, array_size); + BufferType::set_dynamic_array_size(buffer, array_size); state.stats().pushed_back(1); return HandleType(ref, buffer); } |