Store dynamic array size size at start of entry.

8 files changed, 23 insertions, 20 deletions

diff --git a/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp
index c2933dccf18..a703d9b18eb 100644
--- a/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp
+++ b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp
@@ -163,7 +163,7 @@ std::vector<int>
 DynamicArrayBufferTypeTest::get_vector(const void* buffer, uint32_t offset)
 {
     auto e = BufferType::get_entry(buffer, offset, _entry_size);
-    auto array_size = BufferType::get_dynamic_array_size(e, _entry_size);
+    auto array_size = BufferType::get_dynamic_array_size(e);
     EXPECT_GE(_buffer_type.getArraySize(), array_size);
     return get_vector(buffer, offset, array_size);
 }
@@ -179,7 +179,7 @@ void
 DynamicArrayBufferTypeTest::write_entry1()
 {
     auto e1 = BufferType::get_entry(_buf.get(), 1, _entry_size);
-    BufferType::set_dynamic_array_size(e1, _entry_size, 2);
+    BufferType::set_dynamic_array_size(e1, 2);
     new (static_cast<void *>(e1)) WrapInt32(42);
     new (static_cast<void *>(e1 + 1)) WrapInt32(47);
     new (static_cast<void *>(e1 + 2)) WrapInt32(49); // Not cleaned by clean_hold
diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp
index f80ba607ce7..fe425b6a45c 100644
--- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp
@@ -88,7 +88,7 @@ Allocator<EntryT, RefT>::alloc_dynamic_array(ConstArrayRef array)
     for (size_t i = array.size(); i < max_array_size; ++i) {
         new (static_cast<void *>(buf + i)) EntryT();
     }
-    BufferType::set_dynamic_array_size(buf, entry_size, array.size());
+    BufferType::set_dynamic_array_size(buf, array.size());
      state.stats().pushed_back(1);
     return HandleType(ref, buf);
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h
index bff3a5cc5b2..f5c30c90c5b 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.h
@@ -101,7 +101,7 @@ private:
     template <typename BufferType>
     ConstArrayRef get_dynamic_array(const void* buffer, size_t offset, uint32_t entry_size) const {
         auto entry = BufferType::get_entry(buffer, offset, entry_size);
-        auto size = BufferType::get_dynamic_array_size(entry, entry_size);
+        auto size = BufferType::get_dynamic_array_size(entry);
         return ConstArrayRef(entry, size);
     }
     ConstArrayRef getLargeArray(RefT ref) const {
@@ -122,7 +122,7 @@ public:
         const BufferAndMeta & bufferAndMeta = _store.getBufferMeta(internalRef.bufferId());
         if (bufferAndMeta.getTypeId() != _largeArrayTypeId) [[likely]] {
             if constexpr (has_dynamic_buffer_type) {
-                if (_mapper.is_dynamic_buffer(bufferAndMeta.getTypeId())) [[unlikely]] {
+                if (_mapper.is_dynamic_buffer(bufferAndMeta.getTypeId())) {
                         return get_dynamic_array<typename TypeMapper::DynamicBufferType>(bufferAndMeta.get_buffer_acquire(), internalRef.offset(), bufferAndMeta.get_entry_size());
                  }
             }
diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h
index 7d67d0cddf7..b7666ec3dc6 100644
--- a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h
+++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h
@@ -3,7 +3,9 @@
 #pragma once
 
 #include "buffer_type.h"
+#include "aligner.h"
 #include "array_store_config.h"
+#include <algorithm>
 #include <memory>
 
 namespace vespalib::datastore {
@@ -26,6 +28,10 @@ class DynamicArrayBufferType : public BufferTypeBase
     std::shared_ptr<alloc::MemoryAllocator> _memory_allocator;
 public:
     using ElemType = ElemT;
+
+    static constexpr size_t entry_min_align = std::max(alignof(uint32_t), alignof(ElemT));
+    using EntryMinAligner = Aligner<entry_min_align>;
+    static constexpr size_t entry_bias = EntryMinAligner::align(sizeof(uint32_t));
 protected:
     static const ElemType& empty_entry() noexcept;
     ElemType* get_entry(void *buffer, size_t offset) noexcept { return get_entry(buffer, offset, entry_size()); }
@@ -49,10 +55,10 @@ public:
     const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
     static size_t calc_entry_size(size_t array_size) noexcept;
     static size_t calc_array_size(size_t entry_size) noexcept;
-    static ElemType* get_entry(void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<ElemType*>(static_cast<char*>(buffer) + offset * entry_size); }
-    static const ElemType* get_entry(const void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<const ElemType*>(static_cast<const char*>(buffer) + offset * entry_size); }
-    static uint32_t get_dynamic_array_size(const void *buffer, uint32_t entry_size) noexcept { return *reinterpret_cast<const uint32_t*>(static_cast<const char*>(buffer) + entry_size - sizeof(uint32_t)); }
-    static void set_dynamic_array_size(void *buffer, uint32_t entry_size, uint32_t array_size) noexcept { *reinterpret_cast<uint32_t*>(static_cast<char*>(buffer) + entry_size - sizeof(uint32_t)) = array_size; }
+    static ElemType* get_entry(void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<ElemType*>(static_cast<char*>(buffer) + offset * entry_size + entry_bias); }
+    static const ElemType* get_entry(const void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<const ElemType*>(static_cast<const char*>(buffer) + offset * entry_size + entry_bias); }
+    static uint32_t get_dynamic_array_size(const void *buffer) noexcept { return *(static_cast<const uint32_t*>(buffer) - 1); }
+    static void set_dynamic_array_size(void *buffer, uint32_t array_size) noexcept { *(static_cast<uint32_t*>(buffer) - 1) = array_size; }
 };
 
 extern template class DynamicArrayBufferType<char>;
diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp
index 81d8a850b84..bf3235a6b97 100644
--- a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp
@@ -3,8 +3,6 @@
 #pragma once
 
 #include "dynamic_array_buffer_type.h"
-#include "aligner.h"
-#include <algorithm>
 #include <cassert>
 
 namespace vespalib::datastore {
@@ -26,15 +24,14 @@ template <typename ElemT>
 size_t
 DynamicArrayBufferType<ElemT>::calc_entry_size(size_t array_size) noexcept
 {
-    Aligner aligner(std::max(alignof(uint32_t), alignof(ElemType)));
-    return aligner.align(sizeof(ElemType) * array_size + sizeof(uint32_t));
+    return EntryMinAligner::align(sizeof(ElemType) * array_size + entry_bias);
 }
 
 template <typename ElemT>
 size_t
 DynamicArrayBufferType<ElemT>::calc_array_size(size_t entry_size) noexcept
 {
-    return (entry_size - sizeof(uint32_t)) / sizeof(ElemType);
+    return (entry_size - entry_bias) / sizeof(ElemType);
 }
 
 template <typename ElemT>
@@ -59,7 +56,7 @@ DynamicArrayBufferType<ElemT>::fallback_copy(void* new_buffer, const void* old_b
     for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) {
         auto d = get_entry(new_buffer, entry_idx);
         auto s = get_entry(old_buffer, entry_idx);
-        set_dynamic_array_size(d, entry_size(), get_dynamic_array_size(s, entry_size()));
+        set_dynamic_array_size(d, get_dynamic_array_size(s));
         for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) {
             new (static_cast<void*>(d)) ElemType(*s);
             ++s;
@@ -76,7 +73,7 @@ DynamicArrayBufferType<ElemT>::initialize_reserved_entries(void* buffer, EntryCo
     const auto& empty = empty_entry();
     for (uint32_t entry_idx = 0; entry_idx < reserved_entries; ++entry_idx) {
         auto e = get_entry(buffer, entry_idx);
-        set_dynamic_array_size(e, entry_size(), 0);
+        set_dynamic_array_size(e, 0);
         for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) {
             new (static_cast<void*>(e)) ElemType(empty);
             ++e;
@@ -92,7 +89,7 @@ DynamicArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCoun
     const auto& empty = empty_entry();
     for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) {
         auto e = get_entry(buffer, offset + entry_idx);
-        auto array_size = get_dynamic_array_size(e, entry_size());
+        auto array_size = get_dynamic_array_size(e);
         assert(array_size <= max_array_size);
         for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) {
             *e = empty;
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
index 6f3e0bc9911..25eb50750ba 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
@@ -112,7 +112,7 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::alloc_dynamic_array(ConstArrayRef a
     for (size_t i = 0; i < array.size(); ++i) {
         *(buf + i) = array[i];
     }
-    BufferType::set_dynamic_array_size(buf, entry_size, array.size());
+    BufferType::set_dynamic_array_size(buf, array.size());
     return HandleType(ref, buf);
 }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
index c6d93e92828..c6e164bd2d4 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
@@ -42,7 +42,7 @@ FreeListRawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size)
     auto entry_size = _store.get_entry_size(_typeId);
     assert(_store.getBufferState(ref.bufferId()).getArraySize() >= array_size);
     EntryT* entry = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size);
-    BufferType::set_dynamic_array_size(entry, entry_size, array_size);
+    BufferType::set_dynamic_array_size(entry, array_size);
     return HandleType(ref, entry);
 }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
index 5dde8aaa622..91e21aa26e1 100644
--- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
@@ -41,7 +41,7 @@ RawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size)
     RefT ref(state.size(), buffer_id);
     auto entry_size = _store.get_entry_size(_typeId);
     EntryT* buffer = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size);
-    BufferType::set_dynamic_array_size(buffer, entry_size, array_size);
+    BufferType::set_dynamic_array_size(buffer, array_size);
     state.stats().pushed_back(1);
     return HandleType(ref, buffer);
 }