diff options
Diffstat (limited to 'vespalib/src')
4 files changed, 37 insertions, 24 deletions
diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 797dc97c963..6e433e48d88 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -32,7 +32,7 @@ constexpr float ALLOC_GROW_FACTOR = 0.2; template <typename ElemT> class MyArrayStoreSimpleTypeMapper : public ArrayStoreSimpleTypeMapper<ElemT> { public: - MyArrayStoreSimpleTypeMapper(uint32_t, double) + MyArrayStoreSimpleTypeMapper(uint32_t, double, size_t) : ArrayStoreSimpleTypeMapper<ElemT>() { } @@ -62,7 +62,7 @@ struct ArrayStoreTest : public TestT bool add_using_allocate; double type_mapper_grow_factor; ArrayStoreTest(uint32_t max_type_id = 3, bool enable_free_lists = true, bool add_using_allocate_in = false, double type_mapper_grow_factor_in = 2.0) - : type_mapper(max_type_id, type_mapper_grow_factor_in), + : type_mapper(max_type_id, type_mapper_grow_factor_in, ArrayStoreConfig::default_max_buffer_size), store(ArrayStoreConfig(max_type_id, ArrayStoreConfig::AllocSpec(16, RefT::offsetSize(), 8_Ki, ALLOC_GROW_FACTOR)).enable_free_lists(enable_free_lists), @@ -74,7 +74,7 @@ struct ArrayStoreTest : public TestT type_mapper_grow_factor(type_mapper_grow_factor_in) {} explicit ArrayStoreTest(const ArrayStoreConfig &storeCfg) - : type_mapper(storeCfg.max_type_id(), 2.0), + : type_mapper(storeCfg.max_type_id(), 2.0, ArrayStoreConfig::default_max_buffer_size), store(storeCfg, std::make_unique<MemoryAllocatorObserver>(stats), TypeMapperType(type_mapper)), refStore(), generation(1), diff --git a/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp index 3ab38a4113d..f53f5a8ff22 100644 --- a/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp +++ b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp @@ -2,10 +2,18 @@ #include <vespa/vespalib/datastore/array_store_dynamic_type_mapper.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/size_literals.h> +#include <limits> using vespalib::datastore::ArrayStoreDynamicTypeMapper; +namespace { + constexpr double default_grow_factor = 1.03; +constexpr size_t default_max_buffer_size = 256_Mi; +constexpr size_t small_max_buffer_size = 256_Ki; +constexpr size_t max_max_buffer_size = std::numeric_limits<uint32_t>::max(); +} template <typename ElemT> class TestBase : public testing::Test @@ -19,13 +27,13 @@ protected: std::vector<size_t> get_large_array_sizes(uint32_t num_large_arrays); void select_type_ids(std::vector<size_t> array_sizes); void setup_mapper(uint32_t max_buffer_type_id, double grow_factor); - static uint32_t calc_max_buffer_type_id(double grow_factor); + static uint32_t calc_max_buffer_type_id(double grow_factor, size_t max_buffer_size = default_max_buffer_size); }; template <typename ElemT> TestBase<ElemT>::TestBase() : testing::Test(), - _mapper(5, default_grow_factor) + _mapper(5, default_grow_factor, default_max_buffer_size) { } @@ -36,7 +44,7 @@ template <typename ElemT> void TestBase<ElemT>::setup_mapper(uint32_t max_buffer_type_id, double grow_factor) { - _mapper = ArrayStoreDynamicTypeMapper<ElemT>(max_buffer_type_id, grow_factor); + _mapper = ArrayStoreDynamicTypeMapper<ElemT>(max_buffer_type_id, grow_factor, default_max_buffer_size); } template <typename ElemT> @@ -108,9 +116,9 @@ TestBase<ElemT>::select_type_ids(std::vector<size_t> array_sizes) template <typename ElemT> uint32_t -TestBase<ElemT>::calc_max_buffer_type_id(double grow_factor) +TestBase<ElemT>::calc_max_buffer_type_id(double grow_factor, size_t max_buffer_size) { - ArrayStoreDynamicTypeMapper<ElemT> mapper(1000, grow_factor); + ArrayStoreDynamicTypeMapper<ElemT> mapper(1000, grow_factor, max_buffer_size); return mapper.get_max_type_id(1000); } @@ -139,11 +147,13 @@ TEST_F(ArrayStoreDynamicTypeMapperCharTest, large_arrays_grows_exponentially) TEST_F(ArrayStoreDynamicTypeMapperCharTest, avoid_entry_size_overflow) { - EXPECT_EQ(32, calc_max_buffer_type_id(2.0)); - EXPECT_EQ(410, calc_max_buffer_type_id(1.05)); - EXPECT_EQ(507, calc_max_buffer_type_id(1.04)); - EXPECT_EQ(661, calc_max_buffer_type_id(1.03)); - EXPECT_EQ(968, calc_max_buffer_type_id(1.02)); + EXPECT_EQ(29, calc_max_buffer_type_id(2.0)); + EXPECT_EQ(367, calc_max_buffer_type_id(1.05)); + EXPECT_EQ(454, calc_max_buffer_type_id(1.04)); + EXPECT_EQ(591, calc_max_buffer_type_id(1.03)); + EXPECT_EQ(357, calc_max_buffer_type_id(1.03, small_max_buffer_size)); + EXPECT_EQ(661, calc_max_buffer_type_id(1.03, max_max_buffer_size)); + EXPECT_EQ(863, calc_max_buffer_type_id(1.02)); } using ArrayStoreDynamicTypeMapperInt32Test = TestBase<int32_t>; @@ -159,11 +169,13 @@ TEST_F(ArrayStoreDynamicTypeMapperInt32Test, array_sizes_are_calculated) TEST_F(ArrayStoreDynamicTypeMapperInt32Test, avoid_entry_size_overflow) { - EXPECT_EQ(30, calc_max_buffer_type_id(2.0)); - EXPECT_EQ(379, calc_max_buffer_type_id(1.05)); - EXPECT_EQ(462, calc_max_buffer_type_id(1.04)); - EXPECT_EQ(596, calc_max_buffer_type_id(1.03)); - EXPECT_EQ(849, calc_max_buffer_type_id(1.02)); + EXPECT_EQ(27, calc_max_buffer_type_id(2.0)); + EXPECT_EQ(337, calc_max_buffer_type_id(1.05)); + EXPECT_EQ(409, calc_max_buffer_type_id(1.04)); + EXPECT_EQ(525, calc_max_buffer_type_id(1.03)); + EXPECT_EQ(291, calc_max_buffer_type_id(1.03, small_max_buffer_size)); + EXPECT_EQ(596, calc_max_buffer_type_id(1.03, max_max_buffer_size)); + EXPECT_EQ(744, calc_max_buffer_type_id(1.02)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h index 73c998e82a5..6797b2a79b4 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h @@ -36,9 +36,9 @@ public: using LargeBufferType = vespalib::datastore::LargeArrayBufferType<ElemT>; ArrayStoreDynamicTypeMapper(); - ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor); + ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor, size_t max_buffer_size); ~ArrayStoreDynamicTypeMapper(); - void setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor); + void setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor, size_t max_buffer_size); size_t get_entry_size(uint32_t type_id) const; bool is_dynamic_buffer(uint32_t type_id) const noexcept { return type_id > _max_static_array_buffer_type_id; } uint32_t count_dynamic_buffer_types(uint32_t max_type_id) const noexcept { return (max_type_id > _max_static_array_buffer_type_id) ? (max_type_id - _max_static_array_buffer_type_id) : 0u; } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp index e74cd92e6aa..48de5cf5332 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp @@ -18,16 +18,16 @@ ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper() } template <typename ElemT> -ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor) +ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor, size_t max_buffer_size) : ArrayStoreTypeMapper(), _max_static_array_buffer_type_id(0) { - setup_array_sizes(max_buffer_type_id, grow_factor); + setup_array_sizes(max_buffer_type_id, grow_factor, max_buffer_size); } template <typename ElemT> void -ArrayStoreDynamicTypeMapper<ElemT>::setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor) +ArrayStoreDynamicTypeMapper<ElemT>::setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor, size_t max_buffer_size) { _array_sizes.clear(); _array_sizes.reserve(max_buffer_type_id + 1); @@ -49,7 +49,8 @@ ArrayStoreDynamicTypeMapper<ElemT>::setup_array_sizes(uint32_t max_buffer_type_i entry_size = array_size * sizeof(ElemT); } } - if (entry_size > std::numeric_limits<uint32_t>::max()) { + if (entry_size > std::numeric_limits<uint32_t>::max() || + entry_size >= 2 * max_buffer_size) { break; } _array_sizes.emplace_back(array_size); |