diff options
author | Tor Egge <Tor.Egge@yahooinc.com> | 2023-06-16 21:19:57 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-16 21:19:57 +0200 |
commit | 056b844af08214c90ec5226afac58824a828f255 (patch) | |
tree | fb7b20da8474e2f78d7ebb68f8fe6dca78baa784 /vespalib/src | |
parent | 5dd1b71adefe6107d5bfed1228080c78f3e577fb (diff) | |
parent | 25517d0ff47d6d7cc4e23c7f7ee4c624adab38d2 (diff) |
Merge pull request #27453 from vespa-engine/toregge/wire-in-use-of-dynamic-buffer-type-as-needed-in-array-store
Wire in use of dynamic array buffer type as needed in ArrayStore.
Diffstat (limited to 'vespalib/src')
15 files changed, 418 insertions, 138 deletions
diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 37d5fc66c8b..c9f1230346c 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp> +#include <vespa/vespalib/datastore/dynamic_array_buffer_type.hpp> #include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> @@ -27,38 +29,57 @@ namespace { constexpr float ALLOC_GROW_FACTOR = 0.2; +template <typename ElemT> +class MyArrayStoreSimpleTypeMapper : public ArrayStoreSimpleTypeMapper<ElemT> { +public: + MyArrayStoreSimpleTypeMapper(uint32_t, double) + : ArrayStoreSimpleTypeMapper<ElemT>() + { + } +}; + } -template <typename TestT, typename ElemT, typename RefT = EntryRefT<19> > +template <typename TestT, typename ElemT, typename RefT = EntryRefT<19>, typename TypeMapper = ArrayStoreDynamicTypeMapper<ElemT>> struct ArrayStoreTest : public TestT { using EntryRefType = RefT; - using ArrayStoreType = ArrayStore<ElemT, RefT>; + using ArrayStoreType = ArrayStore<ElemT, RefT, TypeMapper>; using LargeArray = typename ArrayStoreType::LargeArray; using ConstArrayRef = typename ArrayStoreType::ConstArrayRef; using ElemVector = std::vector<ElemT>; using value_type = ElemT; using ReferenceStore = vespalib::hash_map<EntryRef, ElemVector>; + using TypeMapperType = TypeMapper; + static constexpr bool simple_type_mapper = std::is_same_v<TypeMapperType,ArrayStoreSimpleTypeMapper<ElemT>>; + using TypeMapperWrappedType = std::conditional_t<simple_type_mapper,MyArrayStoreSimpleTypeMapper<ElemT>,TypeMapperType>; AllocStats stats; + TypeMapperWrappedType type_mapper; ArrayStoreType store; ReferenceStore refStore; generation_t generation; bool add_using_allocate; - ArrayStoreTest(uint32_t maxSmallArraySize = 3, bool enable_free_lists = true, bool add_using_allocate_in = false) - : store(ArrayStoreConfig(maxSmallArraySize, + double type_mapper_grow_factor; + ArrayStoreTest(uint32_t maxSmallArraySize = 3, bool enable_free_lists = true, bool add_using_allocate_in = false, double type_mapper_grow_factor_in = 2.0) + : type_mapper(maxSmallArraySize, type_mapper_grow_factor_in), + store(ArrayStoreConfig(maxSmallArraySize, ArrayStoreConfig::AllocSpec(16, RefT::offsetSize(), 8_Ki, ALLOC_GROW_FACTOR)).enable_free_lists(enable_free_lists), - std::make_unique<MemoryAllocatorObserver>(stats)), + std::make_unique<MemoryAllocatorObserver>(stats), + TypeMapperType(type_mapper)), refStore(), generation(1), - add_using_allocate(add_using_allocate_in) + add_using_allocate(add_using_allocate_in), + type_mapper_grow_factor(type_mapper_grow_factor_in) {} explicit ArrayStoreTest(const ArrayStoreConfig &storeCfg) - : store(storeCfg, std::make_unique<MemoryAllocatorObserver>(stats)), + : type_mapper(storeCfg.maxSmallArrayTypeId(), 2.0), + store(storeCfg, std::make_unique<MemoryAllocatorObserver>(stats), TypeMapperType(type_mapper)), refStore(), generation(1), - add_using_allocate(false) + add_using_allocate(false), + type_mapper_grow_factor(2.0) {} ~ArrayStoreTest() override; void assertAdd(const ElemVector &input) { @@ -163,39 +184,78 @@ struct ArrayStoreTest : public TestT } size_t elem_size() const { return sizeof(ElemT); } size_t largeArraySize() const { return sizeof(LargeArray); } + bool simple_buffers() const { return simple_type_mapper || type_mapper_grow_factor == 1.0; } }; -template <typename TestT, typename ElemT, typename RefT> -ArrayStoreTest<TestT, ElemT, RefT>::~ArrayStoreTest() = default; +template <typename TestT, typename ElemT, typename RefT, typename TypeMapper> +ArrayStoreTest<TestT, ElemT, RefT, TypeMapper>::~ArrayStoreTest() = default; -template <typename TestT, typename ElemT, typename RefT> +template <typename TestT, typename ElemT, typename RefT, typename TypeMapper> size_t -ArrayStoreTest<TestT, ElemT, RefT>::reference_store_count(EntryRef ref) const +ArrayStoreTest<TestT, ElemT, RefT, TypeMapper>::reference_store_count(EntryRef ref) const { return refStore.count(ref); } -struct TestParam { - bool add_using_allocate; - TestParam(bool add_using_allocate_in) : add_using_allocate(add_using_allocate_in) {} +struct SimpleTypeMapperAdd { + using TypeMapper = ArrayStoreSimpleTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = false; + static constexpr double type_mapper_grow_factor = 1.0; }; -std::ostream& operator<<(std::ostream& os, const TestParam& param) -{ - os << (param.add_using_allocate ? "add_using_allocate" : "basic_add"); - return os; -} +struct SimpleTypeMapperAllocate { + using TypeMapper = ArrayStoreSimpleTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = true; + static constexpr double type_mapper_grow_factor = 1.0; +}; + +struct DynamicTypeMapperAddGrow1 { + using TypeMapper = ArrayStoreDynamicTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = false; + static constexpr double type_mapper_grow_factor = 1.0; +}; + +struct DynamicTypeMapperAllocateGrow1 { + using TypeMapper = ArrayStoreDynamicTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = true; + static constexpr double type_mapper_grow_factor = 1.0; +}; + +struct DynamicTypeMapperAddGrow2 { + using TypeMapper = ArrayStoreDynamicTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = false; + static constexpr double type_mapper_grow_factor = 2.0; +}; + +struct DynamicTypeMapperAllocateGrow2 { + using TypeMapper = ArrayStoreDynamicTypeMapper<uint32_t>; + static constexpr bool add_using_allocate = true; + static constexpr double type_mapper_grow_factor = 2.0; +}; -using NumberStoreTestWithParam = ArrayStoreTest<testing::TestWithParam<TestParam>, uint32_t>; +template <typename TypeMapper> +using NumberStoreTestWithParam = ArrayStoreTest<testing::Test, uint32_t, EntryRefT<19>, TypeMapper>; -struct NumberStoreTest : public NumberStoreTestWithParam { - NumberStoreTest() : NumberStoreTestWithParam(3, true, GetParam().add_using_allocate) {} +template <typename Param> +struct NumberStoreTest : public NumberStoreTestWithParam<typename Param::TypeMapper> { + using Parent = NumberStoreTestWithParam<typename Param::TypeMapper>; + NumberStoreTest() : Parent(3, true, Param::add_using_allocate, Param::type_mapper_grow_factor) {} }; -struct NumberStoreFreeListsDisabledTest : public NumberStoreTestWithParam { - NumberStoreFreeListsDisabledTest() : NumberStoreTestWithParam(3, false, GetParam().add_using_allocate) {} +template <typename Param> +struct NumberStoreFreeListsDisabledTest : public NumberStoreTestWithParam<typename Param::TypeMapper> { + using Parent = NumberStoreTestWithParam<typename Param::TypeMapper>; + NumberStoreFreeListsDisabledTest() : Parent(3, false, Param::add_using_allocate, Param::type_mapper_grow_factor) {} }; +using NumberStoreTestTypes = testing::Types<SimpleTypeMapperAdd, SimpleTypeMapperAllocate, + DynamicTypeMapperAddGrow1, DynamicTypeMapperAllocateGrow1, + DynamicTypeMapperAddGrow2, DynamicTypeMapperAllocateGrow2>; + +TYPED_TEST_SUITE(NumberStoreTest, NumberStoreTestTypes); + +TYPED_TEST_SUITE(NumberStoreFreeListsDisabledTest, NumberStoreTestTypes); + using NumberStoreBasicTest = ArrayStoreTest<testing::Test, uint32_t>; using StringStoreTest = ArrayStoreTest<testing::Test, std::string>; using SmallOffsetNumberStoreTest = ArrayStoreTest<testing::Test, uint32_t, EntryRefT<10>>; @@ -206,32 +266,54 @@ TEST(BasicStoreTest, test_with_trivial_and_non_trivial_types) EXPECT_FALSE(vespalib::can_skip_destruction<StringStoreTest::value_type>); } -INSTANTIATE_TEST_SUITE_P(NumberStoreMultiTest, - NumberStoreTest, - testing::Values(TestParam(false), TestParam(true)), - testing::PrintToStringParamName()); - -INSTANTIATE_TEST_SUITE_P(NumberStoreFreeListsDisabledMultiTest, - NumberStoreFreeListsDisabledTest, - testing::Values(TestParam(false), TestParam(true)), - testing::PrintToStringParamName()); - -TEST_P(NumberStoreTest, control_static_sizes) { +TYPED_TEST(NumberStoreTest, control_static_sizes) { static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque; - EXPECT_EQ(416u + sizeof_deque, sizeof(store)); - EXPECT_EQ(240u + sizeof_deque, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); - EXPECT_EQ(112u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); - MemoryUsage usage = store.getMemoryUsage(); - EXPECT_EQ(202140u, usage.allocatedBytes()); - EXPECT_EQ(197680u, usage.usedBytes()); + if constexpr (TestFixture::simple_type_mapper) { + EXPECT_EQ(416u + sizeof_deque, sizeof(this->store)); + } else { + EXPECT_EQ(464u + sizeof_deque, sizeof(this->store)); + } + EXPECT_EQ(240u + sizeof_deque, sizeof(typename TestFixture::ArrayStoreType::DataStoreType)); + EXPECT_EQ(112u, sizeof(typename TestFixture::ArrayStoreType::SmallBufferType)); + MemoryUsage usage = this->store.getMemoryUsage(); + if (this->simple_buffers()) { + EXPECT_EQ(202140u, usage.allocatedBytes()); + EXPECT_EQ(197680u, usage.usedBytes()); + } else { + EXPECT_EQ(202388u, usage.allocatedBytes()); + EXPECT_EQ(197568u, usage.usedBytes()); + } } -TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) +TYPED_TEST(NumberStoreTest, control_type_mapper) { - assertAdd({}); - assertAdd({1}); - assertAdd({2,3}); - assertAdd({3,4,5}); + if constexpr (TestFixture::simple_type_mapper) { + GTEST_SKIP() << "Skipping test due to using simple type mapper"; + } else { + EXPECT_EQ(3, this->type_mapper.get_max_small_array_type_id(1000)); + EXPECT_FALSE(this->type_mapper.is_dynamic_buffer(0)); + EXPECT_FALSE(this->type_mapper.is_dynamic_buffer(1)); + EXPECT_EQ(1, this->type_mapper.get_array_size(1)); + EXPECT_FALSE(this->type_mapper.is_dynamic_buffer(2)); + EXPECT_EQ(2, this->type_mapper.get_array_size(2)); + if (this->type_mapper_grow_factor == 1.0) { + EXPECT_FALSE(this->type_mapper.is_dynamic_buffer(3)); + EXPECT_EQ(3, this->type_mapper.get_array_size(3)); + EXPECT_EQ(0, this->type_mapper.count_dynamic_buffer_types(3)); + } else { + EXPECT_TRUE(this->type_mapper.is_dynamic_buffer(3)); + EXPECT_EQ(4, this->type_mapper.get_array_size(3)); + EXPECT_EQ(1, this->type_mapper.count_dynamic_buffer_types(3)); + } + } +} + +TYPED_TEST(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) +{ + this->assertAdd({}); + this->assertAdd({1}); + this->assertAdd({2,3}); + this->assertAdd({3,4,5}); } TEST_F(StringStoreTest, add_and_get_small_arrays_of_non_trivial_type) @@ -242,60 +324,60 @@ TEST_F(StringStoreTest, add_and_get_small_arrays_of_non_trivial_type) assertAdd({"ddd", "eeee", "fffff"}); } -TEST_P(NumberStoreTest, add_and_get_large_arrays_of_simple_type) +TYPED_TEST(NumberStoreTest, add_and_get_large_arrays_of_simple_type) { - assertAdd({1,2,3,4}); - assertAdd({2,3,4,5,6}); + this->assertAdd({1,2,3,4,5}); + this->assertAdd({2,3,4,5,6,7}); } TEST_F(StringStoreTest, add_and_get_large_arrays_of_non_trivial_type) { - assertAdd({"aa", "bb", "cc", "dd"}); - assertAdd({"ddd", "eee", "ffff", "gggg", "hhhh"}); + assertAdd({"aa", "bb", "cc", "dd", "ee"}); + assertAdd({"ddd", "eee", "ffff", "gggg", "hhhh", "iiii"}); } -TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_small_array_is_removed) +TYPED_TEST(NumberStoreTest, entries_are_put_on_hold_when_a_small_array_is_removed) { - EntryRef ref = add({1,2,3}); - assertBufferState(ref, MemStats().used(1).hold(0)); - store.remove(ref); - assertBufferState(ref, MemStats().used(1).hold(1)); + EntryRef ref = this->add({1,2,3}); + this->assertBufferState(ref, MemStats().used(1).hold(0)); + this->store.remove(ref); + this->assertBufferState(ref, MemStats().used(1).hold(1)); } -TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_large_array_is_removed) +TYPED_TEST(NumberStoreTest, entries_are_put_on_hold_when_a_large_array_is_removed) { - EntryRef ref = add({1,2,3,4}); + EntryRef ref = this->add({1,2,3,4,5}); // Note: The first buffer has the first element reserved -> we expect 2 elements used here. - assertBufferState(ref, MemStats().used(2).hold(0).dead(1)); - store.remove(ref); - assertBufferState(ref, MemStats().used(2).hold(1).dead(1)); + this->assertBufferState(ref, MemStats().used(2).hold(0).dead(1)); + this->store.remove(ref); + this->assertBufferState(ref, MemStats().used(2).hold(1).dead(1)); } -TEST_P(NumberStoreTest, small_arrays_are_allocated_from_free_lists_when_enabled) { - assert_ref_reused({1,2,3}, {4,5,6}, true); +TYPED_TEST(NumberStoreTest, small_arrays_are_allocated_from_free_lists_when_enabled) { + this->assert_ref_reused({1,2,3}, {4,5,6}, true); } -TEST_P(NumberStoreTest, large_arrays_are_allocated_from_free_lists_when_enabled) { - assert_ref_reused({1,2,3,4}, {5,6,7,8}, true); +TYPED_TEST(NumberStoreTest, large_arrays_are_allocated_from_free_lists_when_enabled) { + this->assert_ref_reused({1,2,3,4,5}, {5,6,7,8,9}, true); } -TEST_P(NumberStoreFreeListsDisabledTest, small_arrays_are_NOT_allocated_from_free_lists_when_disabled) { - assert_ref_reused({1,2,3}, {4,5,6}, false); +TYPED_TEST(NumberStoreFreeListsDisabledTest, small_arrays_are_NOT_allocated_from_free_lists_when_disabled) { + this->assert_ref_reused({1,2,3}, {4,5,6}, false); } -TEST_P(NumberStoreFreeListsDisabledTest, large_arrays_are_NOT_allocated_from_free_lists_when_disabled) { - assert_ref_reused({1,2,3,4}, {5,6,7,8}, false); +TYPED_TEST(NumberStoreFreeListsDisabledTest, large_arrays_are_NOT_allocated_from_free_lists_when_disabled) { + this->assert_ref_reused({1,2,3,4,5}, {5,6,7,8,9}, false); } -TEST_P(NumberStoreTest, track_size_of_large_array_allocations_with_free_lists_enabled) { - EntryRef ref = add({1,2,3,4}); - assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(1).extra_used(16)); - remove({1,2,3,4}); - assert_buffer_stats(ref, TestBufferStats().used(2).hold(1).dead(1).extra_hold(16).extra_used(16)); - reclaim_memory(); - assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(2).extra_used(0)); - add({5,6,7,8,9}); - assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(1).extra_used(20)); +TYPED_TEST(NumberStoreTest, track_size_of_large_array_allocations_with_free_lists_enabled) { + EntryRef ref = this->add({1,2,3,4,5}); + this->assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(1).extra_used(20)); + this->remove({1,2,3,4,5}); + this->assert_buffer_stats(ref, TestBufferStats().used(2).hold(1).dead(1).extra_hold(20).extra_used(20)); + this->reclaim_memory(); + this->assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(2).extra_used(0)); + this->add({5,6,7,8,9,10}); + this->assert_buffer_stats(ref, TestBufferStats().used(2).hold(0).dead(1).extra_used(24)); } TEST_F(SmallOffsetNumberStoreTest, new_underlying_buffer_is_allocated_when_current_is_full) @@ -361,7 +443,8 @@ TEST_F(NumberStoreTwoSmallBufferTypesTest, buffer_with_most_dead_space_is_compac namespace { -void testCompaction(NumberStoreTest &f, bool compactMemory, bool compactAddressSpace) +template <typename Fixture> +void testCompaction(Fixture &f, bool compactMemory, bool compactAddressSpace) { EntryRef size1Ref = f.add({1}); EntryRef size2Ref = f.add({2,2}); @@ -422,52 +505,53 @@ void testCompaction(NumberStoreTest &f, bool compactMemory, bool compactAddressS } -TEST_P(NumberStoreTest, compactWorst_selects_on_only_memory) { - testCompaction(*this, true, false); +TYPED_TEST(NumberStoreTest, compactWorst_selects_on_only_memory) { + testCompaction<typename TestFixture::Parent>(*this, true, false); } -TEST_P(NumberStoreTest, compactWorst_selects_on_only_address_space) { - testCompaction(*this, false, true); +TYPED_TEST(NumberStoreTest, compactWorst_selects_on_only_address_space) { + testCompaction<typename TestFixture::Parent>(*this, false, true); } -TEST_P(NumberStoreTest, compactWorst_selects_on_both_memory_and_address_space) { - testCompaction(*this, true, true); +TYPED_TEST(NumberStoreTest, compactWorst_selects_on_both_memory_and_address_space) { + testCompaction<typename TestFixture::Parent>(*this, true, true); } -TEST_P(NumberStoreTest, compactWorst_selects_on_neither_memory_nor_address_space) { - testCompaction(*this, false, false); +TYPED_TEST(NumberStoreTest, compactWorst_selects_on_neither_memory_nor_address_space) { + testCompaction<typename TestFixture::Parent>(*this, false, false); } -TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_small_arrays) +TYPED_TEST(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_small_arrays) { - MemStats exp(store.getMemoryUsage()); - add({1,2,3}); - assertMemoryUsage(exp.used(elem_size() * 3)); - remove({1,2,3}); - assertMemoryUsage(exp.hold(elem_size() * 3)); - reclaim_memory(); - assertMemoryUsage(exp.holdToDead(elem_size() * 3)); + MemStats exp(this->store.getMemoryUsage()); + this->add({1,2,3}); + uint32_t exp_entry_size = this->simple_buffers() ? (this->elem_size() * 3) : (this->elem_size() * 4 + 4); + this->assertMemoryUsage(exp.used(exp_entry_size)); + this->remove({1,2,3}); + this->assertMemoryUsage(exp.hold(exp_entry_size)); + this->reclaim_memory(); + this->assertMemoryUsage(exp.holdToDead(exp_entry_size)); } -TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_large_arrays) +TYPED_TEST(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_large_arrays) { - MemStats exp(store.getMemoryUsage()); - add({1,2,3,4}); - assertMemoryUsage(exp.used(largeArraySize() + elem_size() * 4)); - remove({1,2,3,4}); - assertMemoryUsage(exp.hold(largeArraySize() + elem_size() * 4)); - reclaim_memory(); - assertMemoryUsage(exp.decUsed(elem_size() * 4).decHold(largeArraySize() + elem_size() * 4). - dead(largeArraySize())); + MemStats exp(this->store.getMemoryUsage()); + this->add({1,2,3,4,5}); + this->assertMemoryUsage(exp.used(this->largeArraySize() + this->elem_size() * 5)); + this->remove({1,2,3,4,5}); + this->assertMemoryUsage(exp.hold(this->largeArraySize() + this->elem_size() * 5)); + this->reclaim_memory(); + this->assertMemoryUsage(exp.decUsed(this->elem_size() * 5).decHold(this->largeArraySize() + this->elem_size() * 5). + dead(this->largeArraySize())); } -TEST_P(NumberStoreTest, address_space_usage_is_ratio_between_used_arrays_and_number_of_possible_arrays) +TYPED_TEST(NumberStoreTest, address_space_usage_is_ratio_between_used_arrays_and_number_of_possible_arrays) { - add({2,2}); - add({3,3,3}); + this->add({2,2}); + this->add({3,3,3}); // 1 array is reserved (buffer 0, offset 0). - EXPECT_EQ(3u, store.addressSpaceUsage().used()); - EXPECT_EQ(1u, store.addressSpaceUsage().dead()); + EXPECT_EQ(3u, this->store.addressSpaceUsage().used()); + EXPECT_EQ(1u, this->store.addressSpaceUsage().dead()); size_t fourgig = (1ull << 32); /* * Expected limit is sum of allocated arrays for active buffers and @@ -479,14 +563,18 @@ TEST_P(NumberStoreTest, address_space_usage_is_ratio_between_used_arrays_and_num * 16 * 3 * sizeof(int) = 192 -> 256. * allocated elements = 256 / sizeof(int) = 64. * limit = 64 / 3 = 21. + * + * For dynamic buffer 3, we have 16 * 5 * sizeof(int) => 320 -> 512 + * limit = 512 / (5 * 4) = 25 */ - size_t expLimit = fourgig - 4 * NumberStoreTest::EntryRefType::offsetSize() + 3 * 16 + 21; - EXPECT_EQ(static_cast<double>(2)/ expLimit, store.addressSpaceUsage().usage()); - EXPECT_EQ(expLimit, store.addressSpaceUsage().limit()); + size_t type_id_3_entries = this->simple_buffers() ? 21 : 25; + size_t expLimit = fourgig - 4 * TestFixture::EntryRefType::offsetSize() + 3 * 16 + type_id_3_entries; + EXPECT_EQ(static_cast<double>(2)/ expLimit, this->store.addressSpaceUsage().usage()); + EXPECT_EQ(expLimit, this->store.addressSpaceUsage().limit()); } -struct ByteStoreTest : public ArrayStoreTest<testing::Test, uint8_t> { - ByteStoreTest() : ArrayStoreTest<testing::Test, uint8_t>(ByteStoreTest::ArrayStoreType:: +struct ByteStoreTest : public ArrayStoreTest<testing::Test, uint8_t, EntryRefT<19>, ArrayStoreSimpleTypeMapper<uint8_t>> { + ByteStoreTest() : ArrayStoreTest<testing::Test, uint8_t, EntryRefT<19>, ArrayStoreSimpleTypeMapper<uint8_t>>(ByteStoreTest::ArrayStoreType:: optimizedConfigForHugePage(1023, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, @@ -503,9 +591,9 @@ TEST_F(ByteStoreTest, offset_in_EntryRefT_is_within_bounds_when_allocating_memor assertStoreContent(); } -TEST_P(NumberStoreTest, provided_memory_allocator_is_used) +TYPED_TEST(NumberStoreTest, provided_memory_allocator_is_used) { - EXPECT_EQ(AllocStats(4, 0), stats); + EXPECT_EQ(AllocStats(4, 0), this->stats); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.h b/vespalib/src/vespa/vespalib/datastore/allocator.h index 30938bdc1c1..432232a8879 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/allocator.h @@ -31,6 +31,8 @@ public: HandleType allocArray(ConstArrayRef array); HandleType allocArray(); + template <typename BufferType> + HandleType alloc_dynamic_array(ConstArrayRef array); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp index fa97ef9a5f5..f80ba607ce7 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp @@ -68,5 +68,30 @@ Allocator<EntryT, RefT>::allocArray() return HandleType(ref, buf); } +template <typename EntryT, typename RefT> +template <typename BufferType> +typename Allocator<EntryT, RefT>::HandleType +Allocator<EntryT, RefT>::alloc_dynamic_array(ConstArrayRef array) +{ + _store.ensure_buffer_capacity(_typeId, 1); + uint32_t buffer_id = _store.primary_buffer_id(_typeId); + BufferState &state = _store.getBufferState(buffer_id); + assert(state.isActive()); + auto max_array_size = state.getArraySize(); + assert(max_array_size >= array.size()); + RefT ref(state.size(), buffer_id); + auto entry_size = _store.get_entry_size(_typeId); + EntryT* buf = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); + for (size_t i = 0; i < array.size(); ++i) { + new (static_cast<void *>(buf + i)) EntryT(array[i]); + } + for (size_t i = array.size(); i < max_array_size; ++i) { + new (static_cast<void *>(buf + i)) EntryT(); + } + BufferType::set_dynamic_array_size(buf, entry_size, array.size()); + state.stats().pushed_back(1); + return HandleType(ref, buf); +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 66e6c19fcb0..4a40c94766a 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -15,19 +15,32 @@ #include "large_array_buffer_type.h" #include "small_array_buffer_type.h" #include <vespa/vespalib/util/array.h> +#include <type_traits> namespace vespalib::datastore { /** - * Datastore for storing arrays of type ElemT that is accessed via a 32-bit EntryRef. + * Datastore for storing arrays of type ElemT that is accessed via a 32-bit + * EntryRef. * - * The default EntryRef type uses 19 bits for offset (524288 values) and 13 bits for buffer id (8192 buffers). + * The default EntryRef type uses 19 bits for offset (524288 values) and 13 + * bits for buffer id (8192 buffers). * - * Buffer type ids [1,maxSmallArrayTypeId] are used to allocate small arrays in datastore buffers. - * The default type mapper uses a 1-to-1 mapping between type id and array size. - * Buffer type id 0 is used to heap allocate large arrays as vespalib::Array instances. + * Buffer type ids [1,maxSmallArrayTypeId] are used to allocate small + * arrays in datastore buffers. * - * The max value of maxSmallArrayTypeId is (2^bufferBits - 1). + * The simple type mapper (ArrayStoreSimpleTypeMapper) uses a 1-to-1 + * mapping between type id and array size. + * + * If the type mapper has defined a DynamicBufferType type + * (e.g. ArrayStoreDynamicTypeMapper) then the last part of the buffer type + * ids range might be for dynamic buffers where maximum array size can + * grow exponentially as buffer type id increases. + * + * Buffer type id 0 is used to heap allocate large arrays as + * vespalib::Array instances. + * + * The max value of maxSmallArrayTypeId is (2^(bufferBits - 3) - 1). */ template <typename ElemT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<ElemT> > class ArrayStore : public ICompactable @@ -43,6 +56,22 @@ public: using RefType = RefT; using SmallBufferType = typename TypeMapperT::SmallBufferType; using TypeMapper = TypeMapperT; + struct no_vector { }; + + template <class, class = void> + struct check_dynamic_buffer_type_member { + static constexpr bool value = false; + using vector_type = no_vector; + }; + + template <class T> + struct check_dynamic_buffer_type_member<T, std::void_t<typename T::DynamicBufferType>> { + static constexpr bool value = true; + using vector_type = std::vector<typename T::DynamicBufferType>; + }; + + static constexpr bool has_dynamic_buffer_type = check_dynamic_buffer_type_member<TypeMapper>::value; + using DynamicBufferTypeVector = check_dynamic_buffer_type_member<TypeMapper>::vector_type; private: uint32_t _largeArrayTypeId; uint32_t _maxSmallArrayTypeId; @@ -50,19 +79,31 @@ private: DataStoreType _store; TypeMapper _mapper; std::vector<SmallBufferType> _smallArrayTypes; + [[no_unique_address]] DynamicBufferTypeVector _dynamicArrayTypes; LargeBufferType _largeArrayType; CompactionSpec _compaction_spec; using generation_t = vespalib::GenerationHandler::generation_t; + BufferTypeBase* initArrayType(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, uint32_t type_id); void initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator); - EntryRef addSmallArray(ConstArrayRef array); - EntryRef allocate_small_array(size_t array_size); + EntryRef addSmallArray(ConstArrayRef array, uint32_t type_id); + EntryRef allocate_small_array(uint32_t type_id); + template <typename BufferType> + EntryRef add_dynamic_array(ConstArrayRef array, uint32_t type_id); + template <typename BufferType> + EntryRef allocate_dynamic_array(size_t array_size, uint32_t type_id); EntryRef addLargeArray(ConstArrayRef array); EntryRef allocate_large_array(size_t array_size); ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const { const ElemT *buf = _store.template getEntryArray<ElemT>(ref, arraySize); return ConstArrayRef(buf, arraySize); } + template <typename BufferType> + ConstArrayRef get_dynamic_array(const void* buffer, size_t offset, uint32_t entry_size) const { + auto entry = BufferType::get_entry(buffer, offset, entry_size); + auto size = BufferType::get_dynamic_array_size(entry, entry_size); + return ConstArrayRef(entry, size); + } ConstArrayRef getLargeArray(RefT ref) const { const LargeArray *buf = _store.template getEntry<LargeArray>(ref); return ConstArrayRef(&(*buf)[0], buf->size()); @@ -80,6 +121,11 @@ public: RefT internalRef(ref); const BufferAndMeta & bufferAndMeta = _store.getBufferMeta(internalRef.bufferId()); if (bufferAndMeta.getTypeId() != _largeArrayTypeId) [[likely]] { + if constexpr (has_dynamic_buffer_type) { + if (_mapper.is_dynamic_buffer(bufferAndMeta.getTypeId())) [[unlikely]] { + return get_dynamic_array<typename TypeMapper::DynamicBufferType>(bufferAndMeta.get_buffer_acquire(), internalRef.offset(), bufferAndMeta.get_entry_size()); + } + } return getSmallArray(internalRef, bufferAndMeta.getArraySize()); } else { return getLargeArray(internalRef); diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index c0caab7b7db..8957e1f60aa 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -16,17 +16,35 @@ namespace vespalib::datastore { template <typename ElemT, typename RefT, typename TypeMapperT> +BufferTypeBase* +ArrayStore<ElemT, RefT, TypeMapperT>::initArrayType(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, uint32_t type_id) +{ + const AllocSpec &spec = cfg.spec_for_type_id(type_id); + size_t array_size = _mapper.get_array_size(type_id); + if constexpr (has_dynamic_buffer_type) { + if (_mapper.is_dynamic_buffer(type_id)) { + return &_dynamicArrayTypes.emplace_back(array_size, spec, std::move(memory_allocator), _mapper); + } + } + return &_smallArrayTypes.emplace_back(array_size, spec, std::move(memory_allocator), _mapper); +} + +template <typename ElemT, typename RefT, typename TypeMapperT> void ArrayStore<ElemT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); _smallArrayTypes.reserve(_maxSmallArrayTypeId); + if constexpr (has_dynamic_buffer_type) { + auto dynamic_buffer_types = _mapper.count_dynamic_buffer_types(_maxSmallArrayTypeId); + _smallArrayTypes.reserve(_maxSmallArrayTypeId - dynamic_buffer_types); + _dynamicArrayTypes.reserve(dynamic_buffer_types); + } else { + _smallArrayTypes.reserve(_maxSmallArrayTypeId); + } for (uint32_t type_id = 1; type_id <= _maxSmallArrayTypeId; ++type_id) { - const AllocSpec &spec = cfg.spec_for_type_id(type_id); - size_t arraySize = _mapper.get_array_size(type_id); - _smallArrayTypes.emplace_back(arraySize, spec, memory_allocator, _mapper); - uint32_t act_type_id = _store.addType(&_smallArrayTypes.back()); + uint32_t act_type_id = _store.addType(initArrayType(cfg, memory_allocator, type_id)); assert(type_id == act_type_id); } } @@ -80,7 +98,13 @@ ArrayStore<ElemT, RefT, TypeMapperT>::add(ConstArrayRef array) return EntryRef(); } if (array.size() <= _maxSmallArraySize) { - return addSmallArray(array); + uint32_t type_id = _mapper.get_type_id(array.size()); + if constexpr (has_dynamic_buffer_type) { + if (_mapper.is_dynamic_buffer(type_id)) [[unlikely]] { + return add_dynamic_array<typename TypeMapper::DynamicBufferType>(array, type_id); + } + } + return addSmallArray(array, type_id); } else { return addLargeArray(array); } @@ -94,7 +118,13 @@ ArrayStore<ElemT, RefT, TypeMapperT>::allocate(size_t array_size) return EntryRef(); } if (array_size <= _maxSmallArraySize) { - return allocate_small_array(array_size); + uint32_t type_id = _mapper.get_type_id(array_size); + if constexpr (has_dynamic_buffer_type) { + if (_mapper.is_dynamic_buffer(type_id)) [[unlikely]] { + return allocate_dynamic_array<typename TypeMapper::DynamicBufferType>(array_size, type_id); + } + } + return allocate_small_array(type_id); } else { return allocate_large_array(array_size); } @@ -102,22 +132,37 @@ ArrayStore<ElemT, RefT, TypeMapperT>::allocate(size_t array_size) template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<ElemT, RefT, TypeMapperT>::addSmallArray(ConstArrayRef array) +ArrayStore<ElemT, RefT, TypeMapperT>::addSmallArray(ConstArrayRef array, uint32_t type_id) { - uint32_t typeId = _mapper.get_type_id(array.size()); using NoOpReclaimer = DefaultReclaimer<ElemT>; - return _store.template freeListAllocator<ElemT, NoOpReclaimer>(typeId).allocArray(array).ref; + return _store.template freeListAllocator<ElemT, NoOpReclaimer>(type_id).allocArray(array).ref; } template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<ElemT, RefT, TypeMapperT>::allocate_small_array(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_small_array(uint32_t type_id) { - uint32_t type_id = _mapper.get_type_id(array_size); return _store.template freeListRawAllocator<ElemT>(type_id).alloc(1).ref; } template <typename ElemT, typename RefT, typename TypeMapperT> +template <typename BufferType> +EntryRef +ArrayStore<ElemT, RefT, TypeMapperT>::add_dynamic_array(ConstArrayRef array, uint32_t type_id) +{ + using NoOpReclaimer = DefaultReclaimer<ElemT>; + return _store.template freeListAllocator<ElemT, NoOpReclaimer>(type_id).template alloc_dynamic_array<BufferType>(array).ref; +} + +template <typename ElemT, typename RefT, typename TypeMapperT> +template <typename BufferType> +EntryRef +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_dynamic_array(size_t array_size, uint32_t type_id) +{ + return _store.template freeListRawAllocator<ElemT>(type_id).template alloc_dynamic_array<BufferType>(array_size).ref; +} + +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef ArrayStore<ElemT, RefT, TypeMapperT>::addLargeArray(ConstArrayRef array) { diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h index e02f57eaea3..73c998e82a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h @@ -35,10 +35,13 @@ public: using DynamicBufferType = vespalib::datastore::DynamicArrayBufferType<ElemT>; using LargeBufferType = vespalib::datastore::LargeArrayBufferType<ElemT>; + ArrayStoreDynamicTypeMapper(); ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor); ~ArrayStoreDynamicTypeMapper(); void setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor); size_t get_entry_size(uint32_t type_id) const; + bool is_dynamic_buffer(uint32_t type_id) const noexcept { return type_id > _max_static_array_buffer_type_id; } + uint32_t count_dynamic_buffer_types(uint32_t max_type_id) const noexcept { return (max_type_id > _max_static_array_buffer_type_id) ? (max_type_id - _max_static_array_buffer_type_id) : 0u; } }; extern template class ArrayStoreDynamicTypeMapper<char>; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp index f529ecccb46..e74cd92e6aa 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp @@ -11,6 +11,13 @@ namespace vespalib::datastore { template <typename ElemT> +ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper() + : ArrayStoreTypeMapper(), + _max_static_array_buffer_type_id(0) +{ +} + +template <typename ElemT> ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor) : ArrayStoreTypeMapper(), _max_static_array_buffer_type_id(0) diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index f714f8e24d5..289be32e19b 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -140,6 +140,7 @@ public: uint32_t getArraySize() const { return _arraySize; } BufferState * get_state_relaxed() { return _state.load(std::memory_order_relaxed); } const BufferState * get_state_acquire() const { return _state.load(std::memory_order_acquire); } + uint32_t get_entry_size() const { return get_state_acquire()->getTypeHandler()->entry_size(); } void setTypeId(uint32_t typeId) { _typeId = typeId; } void setArraySize(uint32_t arraySize) { _arraySize = arraySize; } void set_state(BufferState * state) { _state.store(state, std::memory_order_release); } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index e5a38e3fd41..dbcdbeb12b9 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -184,12 +184,14 @@ public: */ virtual void reclaim_entry_refs(generation_t oldest_used_gen) = 0; + uint32_t get_entry_size(uint32_t type_id) { return _typeHandlers[type_id]->entry_size(); } + + void* getBuffer(uint32_t bufferId) { return _buffers[bufferId].get_buffer_relaxed(); } + protected: DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries); virtual ~DataStoreBase(); - void* getBuffer(uint32_t bufferId) { return _buffers[bufferId].get_buffer_relaxed(); } - struct EntryRefHoldElem { EntryRef ref; size_t num_entries; diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h index dc2d1ea3c34..f488a4f0e0f 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h @@ -30,6 +30,8 @@ public: HandleType allocArray(ConstArrayRef array); HandleType allocArray(); + template <typename BufferType> + HandleType alloc_dynamic_array(ConstArrayRef array); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp index 4e69db08a3c..6f3e0bc9911 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp @@ -96,5 +96,25 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray() return HandleType(ref, buf); } +template <typename EntryT, typename RefT, typename ReclaimerT> +template <typename BufferType> +typename Allocator<EntryT, RefT>::HandleType +FreeListAllocator<EntryT, RefT, ReclaimerT>::alloc_dynamic_array(ConstArrayRef array) +{ + auto& free_list = _store.getFreeList(_typeId); + if (free_list.empty()) { + return ParentType::template alloc_dynamic_array<BufferType>(array); + } + RefT ref = free_list.pop_entry(); + assert(_store.getBufferState(ref.bufferId()).getArraySize() >= array.size()); + auto entry_size = _store.get_entry_size(_typeId); + EntryT* buf = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); + for (size_t i = 0; i < array.size(); ++i) { + *(buf + i) = array[i]; + } + BufferType::set_dynamic_array_size(buf, entry_size, array.size()); + return HandleType(ref, buf); +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h index 29684267546..ead192156bd 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h @@ -28,6 +28,8 @@ public: FreeListRawAllocator(DataStoreBase &store, uint32_t typeId); HandleType alloc(size_t num_entries); + template <typename BufferType> + HandleType alloc_dynamic_array(size_t array_size); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp index 7680cd8a9a5..c6d93e92828 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp @@ -29,5 +29,22 @@ FreeListRawAllocator<EntryT, RefT>::alloc(size_t num_entries) return HandleType(ref, entry); } +template <typename EntryT, typename RefT> +template <typename BufferType> +typename FreeListRawAllocator<EntryT, RefT>::HandleType +FreeListRawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size) +{ + auto& free_list = _store.getFreeList(_typeId); + if (free_list.empty()) { + return ParentType::template alloc_dynamic_array<BufferType>(array_size); + } + RefT ref = free_list.pop_entry(); + auto entry_size = _store.get_entry_size(_typeId); + assert(_store.getBufferState(ref.bufferId()).getArraySize() >= array_size); + EntryT* entry = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); + BufferType::set_dynamic_array_size(entry, entry_size, array_size); + return HandleType(ref, entry); +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h index e7a59fadcf8..6af608164bc 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h @@ -29,6 +29,8 @@ public: return alloc(num_entries, 0); } HandleType alloc(size_t num_entries, size_t extra_entries); + template <typename BufferType> + HandleType alloc_dynamic_array(size_t array_size); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp index 9de361a8b19..5dde8aaa622 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp @@ -28,5 +28,23 @@ RawAllocator<EntryT, RefT>::alloc(size_t num_entries, size_t extra_entries) return HandleType(ref, buffer); } +template <typename EntryT, typename RefT> +template <typename BufferType> +typename RawAllocator<EntryT, RefT>::HandleType +RawAllocator<EntryT, RefT>::alloc_dynamic_array(size_t array_size) +{ + _store.ensure_buffer_capacity(_typeId, 1); + uint32_t buffer_id = _store.primary_buffer_id(_typeId); + BufferState &state = _store.getBufferState(buffer_id); + assert(state.isActive()); + assert(state.getArraySize() >= array_size); + RefT ref(state.size(), buffer_id); + auto entry_size = _store.get_entry_size(_typeId); + EntryT* buffer = BufferType::get_entry(_store.getBuffer(ref.bufferId()), ref.offset(), entry_size); + BufferType::set_dynamic_array_size(buffer, entry_size, array_size); + state.stats().pushed_back(1); + return HandleType(ref, buffer); +} + } |