Cap number of entries in a buffer to avoid very large buffers.

7 files changed, 28 insertions, 19 deletions

diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
index 307d1a0d112..25ca7729a32 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
@@ -79,11 +79,12 @@ public:
 
 
     static vespalib::datastore::ArrayStoreConfig optimizedConfigForHugePage(size_t max_type_id,
-                                                                  size_t hugePageSize,
-                                                                  size_t smallPageSize,
-                                                                  size_t min_num_entries_for_new_buffer,
-                                                                  float allocGrowFactor,
-                                                                  bool enable_free_lists);
+                                                                            size_t hugePageSize,
+                                                                            size_t smallPageSize,
+                                                                            size_t max_buffer_size,
+                                                                            size_t min_num_entries_for_new_buffer,
+                                                                            float allocGrowFactor,
+                                                                            bool enable_free_lists);
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
index 99808b11e92..3c9a52f2e5c 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
@@ -66,14 +66,15 @@ MultiValueMapping<ElemT, RefT>::getAddressSpaceUsage() const {
 template <typename ElemT, typename RefT>
 vespalib::datastore::ArrayStoreConfig
 MultiValueMapping<ElemT, RefT>::optimizedConfigForHugePage(size_t max_type_id,
-                                                             size_t hugePageSize,
-                                                             size_t smallPageSize,
-                                                             size_t min_num_entries_for_new_buffer,
-                                                             float allocGrowFactor,
-                                                             bool enable_free_lists)
+                                                           size_t hugePageSize,
+                                                           size_t smallPageSize,
+                                                           size_t max_buffer_size,
+                                                           size_t min_num_entries_for_new_buffer,
+                                                           float allocGrowFactor,
+                                                           bool enable_free_lists)
 {
     ArrayStoreTypeMapper mapper(max_type_id, array_store_grow_factor);
-    auto result = ArrayStore::optimizedConfigForHugePage(max_type_id, mapper, hugePageSize, smallPageSize, min_num_entries_for_new_buffer, allocGrowFactor);
+    auto result = ArrayStore::optimizedConfigForHugePage(max_type_id, mapper, hugePageSize, smallPageSize, max_buffer_size, min_num_entries_for_new_buffer, allocGrowFactor);
     result.enable_free_lists(enable_free_lists);
     return result;
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
index d8ada97fa2c..3cf75b450af 100644
--- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
@@ -28,6 +28,7 @@ MultiValueAttribute(const vespalib::string &baseFileName,
       _mvMapping(MultiValueMapping::optimizedConfigForHugePage(MultiValueMapping::array_store_max_type_id,
                                                                vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
                                                                vespalib::alloc::MemoryAllocator::PAGE_SIZE,
+                                                               vespalib::datastore::ArrayStoreConfig::default_max_buffer_size,
                                                                8 * 1024,
                                                                cfg.getGrowStrategy().getMultiValueAllocGrowFactor(),
                                                                multivalueattribute::enable_free_lists),
diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp
index cd9e0508344..00c195b9eb7 100644
--- a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp
@@ -20,6 +20,7 @@ RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator>
                                                               TypeMapper(max_small_buffer_type_id, grow_factor),
                                                               MemoryAllocator::HUGEPAGE_SIZE,
                                                               MemoryAllocator::PAGE_SIZE,
+                                                              vespalib::datastore::ArrayStoreConfig::default_max_buffer_size,
                                                               8_Ki, ALLOC_GROW_FACTOR),
                    std::move(allocator), TypeMapper(max_small_buffer_type_id, grow_factor))
 {
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
index 748a747d515..22a33270a27 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
@@ -31,6 +31,7 @@ namespace search::tensor {
 using search::AddressSpaceComponents;
 using search::StateExplorerUtils;
 using search::queryeval::GlobalFilter;
+using vespalib::datastore::ArrayStoreConfig;
 using vespalib::datastore::CompactionStrategy;
 using vespalib::datastore::EntryRef;
 using vespalib::GenericHeader;
@@ -145,25 +146,27 @@ PreparedAddDoc::PreparedAddDoc(PreparedAddDoc&& other) noexcept = default;
 }
 
 template <HnswIndexType type>
-vespalib::datastore::ArrayStoreConfig
+ArrayStoreConfig
 HnswIndex<type>::make_default_level_array_store_config()
 {
     return LevelArrayStore::optimizedConfigForHugePage(max_level_array_size,
-                                                 vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
-                                                 vespalib::alloc::MemoryAllocator::PAGE_SIZE,
+                                                       vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+                                                       vespalib::alloc::MemoryAllocator::PAGE_SIZE,
+                                                       ArrayStoreConfig::default_max_buffer_size,
                                                  min_num_arrays_for_new_buffer,
                                                  alloc_grow_factor).enable_free_lists(true);
 }
 
 template <HnswIndexType type>
-vespalib::datastore::ArrayStoreConfig
+ArrayStoreConfig
 HnswIndex<type>::make_default_link_array_store_config()
 {
     return LinkArrayStore::optimizedConfigForHugePage(max_link_array_size,
-                                                 vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
-                                                 vespalib::alloc::MemoryAllocator::PAGE_SIZE,
-                                                 min_num_arrays_for_new_buffer,
-                                                 alloc_grow_factor).enable_free_lists(true);
+                                                      vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+                                                      vespalib::alloc::MemoryAllocator::PAGE_SIZE,
+                                                      ArrayStoreConfig::default_max_buffer_size,
+                                                      min_num_arrays_for_new_buffer,
+                                                      alloc_grow_factor).enable_free_lists(true);
 }
 
 template <HnswIndexType type>
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp
index a78d9cefc64..cf30d62a0b8 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp
@@ -49,6 +49,7 @@ HnswNodeidMapping::HnswNodeidMapping()
       _nodeids(NodeidStore::optimizedConfigForHugePage(max_type_id,
                                                        vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
                                                        vespalib::alloc::MemoryAllocator::PAGE_SIZE,
+                                                       vespalib::datastore::ArrayStoreConfig::default_max_buffer_size,
                                                        min_num_arrays_for_new_buffer,
                                                        alloc_grow_factor).enable_free_lists(true), {}),
       _hold_list(),
diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp
index ff39c33fc5d..29f20e27d09 100644
--- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp
@@ -38,6 +38,7 @@ TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_p
                                                               TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops),
                                                               MemoryAllocator::HUGEPAGE_SIZE,
                                                               MemoryAllocator::PAGE_SIZE,
+                                                              vespalib::datastore::ArrayStoreConfig::default_max_buffer_size,
                                                               8_Ki, ALLOC_GROW_FACTOR),
                    std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops))
 {