diff options
14 files changed, 113 insertions, 50 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 307d1a0d112..25ca7729a32 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -79,11 +79,12 @@ public: static vespalib::datastore::ArrayStoreConfig optimizedConfigForHugePage(size_t max_type_id, - size_t hugePageSize, - size_t smallPageSize, - size_t min_num_entries_for_new_buffer, - float allocGrowFactor, - bool enable_free_lists); + size_t hugePageSize, + size_t smallPageSize, + size_t max_buffer_size, + size_t min_num_entries_for_new_buffer, + float allocGrowFactor, + bool enable_free_lists); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 99808b11e92..3c9a52f2e5c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -66,14 +66,15 @@ MultiValueMapping<ElemT, RefT>::getAddressSpaceUsage() const { template <typename ElemT, typename RefT> vespalib::datastore::ArrayStoreConfig MultiValueMapping<ElemT, RefT>::optimizedConfigForHugePage(size_t max_type_id, - size_t hugePageSize, - size_t smallPageSize, - size_t min_num_entries_for_new_buffer, - float allocGrowFactor, - bool enable_free_lists) + size_t hugePageSize, + size_t smallPageSize, + size_t max_buffer_size, + size_t min_num_entries_for_new_buffer, + float allocGrowFactor, + bool enable_free_lists) { ArrayStoreTypeMapper mapper(max_type_id, array_store_grow_factor); - auto result = ArrayStore::optimizedConfigForHugePage(max_type_id, mapper, hugePageSize, smallPageSize, min_num_entries_for_new_buffer, allocGrowFactor); + auto result = ArrayStore::optimizedConfigForHugePage(max_type_id, mapper, hugePageSize, smallPageSize, max_buffer_size, min_num_entries_for_new_buffer, allocGrowFactor); result.enable_free_lists(enable_free_lists); return result; } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index d8ada97fa2c..3cf75b450af 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -28,6 +28,7 @@ MultiValueAttribute(const vespalib::string &baseFileName, _mvMapping(MultiValueMapping::optimizedConfigForHugePage(MultiValueMapping::array_store_max_type_id, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, + vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, 8 * 1024, cfg.getGrowStrategy().getMultiValueAllocGrowFactor(), multivalueattribute::enable_free_lists), diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp index cd9e0508344..00c195b9eb7 100644 --- a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp @@ -20,6 +20,7 @@ RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> TypeMapper(max_small_buffer_type_id, grow_factor), MemoryAllocator::HUGEPAGE_SIZE, MemoryAllocator::PAGE_SIZE, + vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, 8_Ki, ALLOC_GROW_FACTOR), std::move(allocator), TypeMapper(max_small_buffer_type_id, grow_factor)) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 748a747d515..22a33270a27 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -31,6 +31,7 @@ namespace search::tensor { using search::AddressSpaceComponents; using search::StateExplorerUtils; using search::queryeval::GlobalFilter; +using vespalib::datastore::ArrayStoreConfig; using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; using vespalib::GenericHeader; @@ -145,25 +146,27 @@ PreparedAddDoc::PreparedAddDoc(PreparedAddDoc&& other) noexcept = default; } template <HnswIndexType type> -vespalib::datastore::ArrayStoreConfig +ArrayStoreConfig HnswIndex<type>::make_default_level_array_store_config() { return LevelArrayStore::optimizedConfigForHugePage(max_level_array_size, - vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, - vespalib::alloc::MemoryAllocator::PAGE_SIZE, + vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + vespalib::alloc::MemoryAllocator::PAGE_SIZE, + ArrayStoreConfig::default_max_buffer_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true); } template <HnswIndexType type> -vespalib::datastore::ArrayStoreConfig +ArrayStoreConfig HnswIndex<type>::make_default_link_array_store_config() { return LinkArrayStore::optimizedConfigForHugePage(max_link_array_size, - vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, - vespalib::alloc::MemoryAllocator::PAGE_SIZE, - min_num_arrays_for_new_buffer, - alloc_grow_factor).enable_free_lists(true); + vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + vespalib::alloc::MemoryAllocator::PAGE_SIZE, + ArrayStoreConfig::default_max_buffer_size, + min_num_arrays_for_new_buffer, + alloc_grow_factor).enable_free_lists(true); } template <HnswIndexType type> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp index a78d9cefc64..cf30d62a0b8 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp @@ -49,6 +49,7 @@ HnswNodeidMapping::HnswNodeidMapping() _nodeids(NodeidStore::optimizedConfigForHugePage(max_type_id, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, + vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true), {}), _hold_list(), diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp index ff39c33fc5d..29f20e27d09 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp @@ -38,6 +38,7 @@ TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_p TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops), MemoryAllocator::HUGEPAGE_SIZE, MemoryAllocator::PAGE_SIZE, + vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, 8_Ki, ALLOC_GROW_FACTOR), std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops)) { diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index baec5494b36..efa7e18aa33 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -40,6 +40,7 @@ vespalib::datastore::ArrayStoreConfig make_default_array_store_config() { return ReplicaStore::optimizedConfigForHugePage(1023, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, + vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, 8_Ki, 0.2).enable_free_lists(true); } diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 2674acf1ce9..797dc97c963 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -578,6 +578,7 @@ struct ByteStoreTest : public ArrayStoreTest<testing::Test, uint8_t, EntryRefT<1 optimizedConfigForHugePage(1023, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, + ArrayStoreConfig::default_max_buffer_size, 8_Ki, ALLOC_GROW_FACTOR)) {} }; diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp index 71c1341ae74..83430c760bb 100644 --- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp +++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp @@ -22,15 +22,20 @@ struct Fixture Fixture(uint32_t max_type_id, size_t hugePageSize, size_t smallPageSize, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer) : cfg(ArrayStoreConfig::optimizeForHugePage(max_type_id, [](size_t type_id) noexcept { return type_id * sizeof(int); }, hugePageSize, smallPageSize, EntryRefType::offsetSize(), + max_buffer_size, min_num_entries_for_new_buffer, ALLOC_GROW_FACTOR)) { } void assertSpec(uint32_t type_id, uint32_t num_entries_for_new_buffer) { - assertSpec(type_id, AllocSpec(0, EntryRefType::offsetSize(), + assertSpec(type_id, EntryRefType::offsetSize(), num_entries_for_new_buffer); + } + void assertSpec(uint32_t type_id, uint32_t max_entries, uint32_t num_entries_for_new_buffer) { + assertSpec(type_id, AllocSpec(0, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR)); } void assertSpec(uint32_t type_id, const AllocSpec &expSpec) { @@ -50,9 +55,6 @@ makeSpec(size_t min_entries_in_buffer, return AllocSpec(min_entries_in_buffer, max_entries_in_buffer, num_entries_for_new_buffer, ALLOC_GROW_FACTOR); } -constexpr size_t KB = 1024; -constexpr size_t MB = KB * KB; - TEST_F("require that default allocation spec is given for all array sizes", Fixture(3, makeSpec(4, 32, 8))) { EXPECT_EQUAL(3u, f.cfg.max_type_id()); @@ -62,26 +64,54 @@ TEST_F("require that default allocation spec is given for all array sizes", Fixt TEST_DO(f.assertSpec(3, makeSpec(4, 32, 8))); } -TEST_F("require that we can generate config optimized for a given huge page", Fixture(1024, - 2 * MB, - 4 * KB, - 8 * KB)) +struct BigBuffersFixture : public Fixture { + BigBuffersFixture() : Fixture(1023, 2_Mi, 4_Ki, 1024_Gi, 8_Ki) { } +}; + +TEST_F("require that we can generate config optimized for a given huge page without capped buffer sizes", BigBuffersFixture()) +{ + EXPECT_EQUAL(1023u, f.cfg.max_type_id()); + TEST_DO(f.assertSpec(0, 8_Ki)); // large arrays + TEST_DO(f.assertSpec(1, 256_Ki)); + TEST_DO(f.assertSpec(2, 256_Ki)); + TEST_DO(f.assertSpec(3, 168_Ki)); + TEST_DO(f.assertSpec(4, 128_Ki)); + TEST_DO(f.assertSpec(5, 100_Ki)); + TEST_DO(f.assertSpec(6, 84_Ki)); + + TEST_DO(f.assertSpec(32, 16_Ki)); + TEST_DO(f.assertSpec(33, 12_Ki)); + TEST_DO(f.assertSpec(42, 12_Ki)); + TEST_DO(f.assertSpec(43, 8_Ki)); + TEST_DO(f.assertSpec(1022, 8_Ki)); + TEST_DO(f.assertSpec(1023, 8_Ki)); +} + +struct CappedBuffersFixture : public Fixture { + CappedBuffersFixture() : Fixture(1023, 2_Mi, 4_Ki, 256_Mi, 8_Ki) { } + size_t max_entries(size_t array_size) { + auto entry_size = array_size * sizeof(int); + return std::min(EntryRefType::offsetSize(), (256_Mi + entry_size - 1) / entry_size); + } +}; + +TEST_F("require that we can generate config optimized for a given huge page with capped buffer sizes", CappedBuffersFixture()) { - EXPECT_EQUAL(1_Ki, f.cfg.max_type_id()); - TEST_DO(f.assertSpec(0, 8 * KB)); // large arrays - TEST_DO(f.assertSpec(1, 256 * KB)); - TEST_DO(f.assertSpec(2, 256 * KB)); - TEST_DO(f.assertSpec(3, 168 * KB)); - TEST_DO(f.assertSpec(4, 128 * KB)); - TEST_DO(f.assertSpec(5, 100 * KB)); - TEST_DO(f.assertSpec(6, 84 * KB)); + EXPECT_EQUAL(1023u, f.cfg.max_type_id()); + TEST_DO(f.assertSpec(0, f.max_entries(1023), 8_Ki)); // large arrays + TEST_DO(f.assertSpec(1, 256_Ki)); + TEST_DO(f.assertSpec(2, 256_Ki)); + TEST_DO(f.assertSpec(3, 168_Ki)); + TEST_DO(f.assertSpec(4, 128_Ki)); + TEST_DO(f.assertSpec(5, 100_Ki)); + TEST_DO(f.assertSpec(6, 84_Ki)); - TEST_DO(f.assertSpec(32, 16 * KB)); - TEST_DO(f.assertSpec(33, 12 * KB)); - TEST_DO(f.assertSpec(42, 12 * KB)); - TEST_DO(f.assertSpec(43, 8 * KB)); - TEST_DO(f.assertSpec(1022, 8 * KB)); - TEST_DO(f.assertSpec(1023, 8 * KB)); + TEST_DO(f.assertSpec(32, 16_Ki)); + TEST_DO(f.assertSpec(33, 12_Ki)); + TEST_DO(f.assertSpec(42, 12_Ki)); + TEST_DO(f.assertSpec(43, 8_Ki)); + TEST_DO(f.assertSpec(1022, f.max_entries(1022), 8_Ki)); + TEST_DO(f.assertSpec(1023, f.max_entries(1023), 8_Ki)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 0490687aeb8..7ee63be3848 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -196,6 +196,7 @@ public: static ArrayStoreConfig optimizedConfigForHugePage(uint32_t max_type_id, size_t hugePageSize, size_t smallPageSize, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer, float allocGrowFactor); @@ -203,6 +204,7 @@ public: const TypeMapper& mapper, size_t hugePageSize, size_t smallPageSize, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 211176b8ad0..bfd4ff0430a 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -252,6 +252,7 @@ ArrayStoreConfig ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t max_type_id, size_t hugePageSize, size_t smallPageSize, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer, float allocGrowFactor) { @@ -260,6 +261,7 @@ ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t max_ty mapper, hugePageSize, smallPageSize, + max_buffer_size, min_num_entries_for_new_buffer, allocGrowFactor); } @@ -267,17 +269,19 @@ ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t max_ty template <typename ElemT, typename RefT, typename TypeMapperT> ArrayStoreConfig ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t max_type_id, - const TypeMapper& mapper, - size_t hugePageSize, - size_t smallPageSize, - size_t min_num_entries_for_new_buffer, - float allocGrowFactor) + const TypeMapper& mapper, + size_t hugePageSize, + size_t smallPageSize, + size_t max_buffer_size, + size_t min_num_entries_for_new_buffer, + float allocGrowFactor) { return ArrayStoreConfig::optimizeForHugePage(mapper.get_max_type_id(max_type_id), [&](uint32_t type_id) noexcept { return mapper.get_entry_size(type_id); }, hugePageSize, smallPageSize, RefT::offsetSize(), + max_buffer_size, min_num_entries_for_new_buffer, allocGrowFactor); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp index c7f0b69a85e..37f6fab96dc 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "array_store_config.h" +#include <algorithm> #include <cassert> namespace vespalib::datastore { @@ -42,6 +43,13 @@ alignToSmallPageSize(size_t value, size_t minLimit, size_t smallPageSize) return ((value - minLimit) / smallPageSize) * smallPageSize + minLimit; } +size_t +cap_max_entries(size_t max_entries, size_t max_buffer_size, size_t entry_size) +{ + size_t dynamic_max_entries = (max_buffer_size + (entry_size - 1)) / entry_size; + return std::min(max_entries, dynamic_max_entries); +} + } ArrayStoreConfig @@ -50,17 +58,21 @@ ArrayStoreConfig::optimizeForHugePage(uint32_t max_type_id, size_t hugePageSize, size_t smallPageSize, size_t maxEntryRefOffset, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer, float allocGrowFactor) { AllocSpecVector allocSpecs; - allocSpecs.emplace_back(0, maxEntryRefOffset, min_num_entries_for_new_buffer, allocGrowFactor); // large array spec; + auto entry_size = type_id_to_entry_size(max_type_id); + auto capped_max_entries = cap_max_entries(maxEntryRefOffset, max_buffer_size, entry_size); + allocSpecs.emplace_back(0, capped_max_entries, min_num_entries_for_new_buffer, allocGrowFactor); // large array spec; for (uint32_t type_id = 1; type_id <= max_type_id; ++type_id) { - size_t entry_size = type_id_to_entry_size(type_id); + entry_size = type_id_to_entry_size(type_id); + capped_max_entries = cap_max_entries(maxEntryRefOffset, max_buffer_size, entry_size); size_t num_entries_for_new_buffer = hugePageSize / entry_size; - num_entries_for_new_buffer = capToLimits(num_entries_for_new_buffer, min_num_entries_for_new_buffer, maxEntryRefOffset); + num_entries_for_new_buffer = capToLimits(num_entries_for_new_buffer, min_num_entries_for_new_buffer, capped_max_entries); num_entries_for_new_buffer = alignToSmallPageSize(num_entries_for_new_buffer, min_num_entries_for_new_buffer, smallPageSize); - allocSpecs.emplace_back(0, maxEntryRefOffset, num_entries_for_new_buffer, allocGrowFactor); + allocSpecs.emplace_back(0, capped_max_entries, num_entries_for_new_buffer, allocGrowFactor); } return ArrayStoreConfig(allocSpecs); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.h b/vespalib/src/vespa/vespalib/datastore/array_store_config.h index 3b62609d0f1..3967996c64d 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/util/size_literals.h> #include <cstddef> #include <cstdint> #include <functional> @@ -39,6 +40,8 @@ public: using AllocSpecVector = std::vector<AllocSpec>; + static constexpr size_t default_max_buffer_size = 256_Mi; + private: AllocSpecVector _allocSpecs; bool _enable_free_lists; @@ -77,6 +80,7 @@ public: size_t hugePageSize, size_t smallPageSize, size_t maxEntryRefOffset, + size_t max_buffer_size, size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; |