Merge pull request #21114 from vespa-engine/balder/use-mmap-for-large-vectors-4

Automatically switch to mmap for large allocations.

1 files changed, 32 insertions, 20 deletions

diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp
index 591350347e1..a0a96bc4497 100644
--- a/eval/src/vespa/eval/eval/fast_value.hpp
+++ b/eval/src/vespa/eval/eval/fast_value.hpp
@@ -168,29 +168,19 @@ struct FastCells {
     static constexpr size_t elem_size = sizeof(T);
     size_t capacity;
     size_t size;
-    void *memory;
-    FastCells(size_t initial_capacity)
-        : capacity(roundUp2inN(initial_capacity)),
-          size(0),
-          memory(malloc(elem_size * capacity))
-    {
-        static_assert(std::is_trivially_copyable_v<T>);
-        static_assert(can_skip_destruction<T>::value);
-    }
-    ~FastCells() {
-        free(memory);
-    }
+    mutable alloc::Alloc memory;
+    FastCells(size_t initial_capacity);
+    FastCells(const FastCells &) = delete;
+    FastCells & operator = (const FastCells &) = delete;
+    ~FastCells();
     void ensure_free(size_t need) {
         if (__builtin_expect((size + need) > capacity, false)) {
-            capacity = roundUp2inN(size + need);
-            void *new_memory = malloc(elem_size * capacity);
-            memcpy(new_memory, memory, elem_size * size);
-            free(memory);
-            memory = new_memory;
+            reallocate(need);
         }
     }
+    void reallocate(size_t need);
     constexpr T *get(size_t offset) const {
-        return reinterpret_cast<T*>(memory) + offset;
+        return reinterpret_cast<T*>(memory.get()) + offset;
     }
     void push_back_fast(T value) {
         *get(size++) = value;
@@ -209,6 +199,28 @@ struct FastCells {
     }
 };
 
+template <typename T>
+FastCells<T>::FastCells(size_t initial_capacity)
+    : capacity(roundUp2inN(initial_capacity)),
+      size(0),
+      memory(alloc::Alloc::alloc(elem_size * capacity))
+{
+    static_assert(std::is_trivially_copyable_v<T>);
+    static_assert(can_skip_destruction<T>::value);
+}
+
+template <typename T>
+void
+FastCells<T>::reallocate(size_t need) {
+    capacity = roundUp2inN(size + need);
+    alloc::Alloc new_memory = alloc::Alloc::alloc(elem_size * capacity);
+    memcpy(new_memory.get(), memory.get(), elem_size * size);
+    memory = std::move(new_memory);
+}
+
+template <typename T>
+FastCells<T>::~FastCells() = default;
+
 //-----------------------------------------------------------------------------
 
 template <typename T, bool transient>
@@ -237,7 +249,7 @@ struct FastValue final : Value, ValueBuilder<T> {
             // not called
             abort();
         } else {
-            return TypedCells(my_cells.memory, get_cell_type<T>(), my_cells.size);
+            return TypedCells(my_cells.memory.get(), get_cell_type<T>(), my_cells.size);
         }
     }
     void add_mapping(ConstArrayRef<vespalib::stringref> addr) {
@@ -333,7 +345,7 @@ struct FastDenseValue final : Value, ValueBuilder<T> {
     ~FastDenseValue() override;
     const ValueType &type() const override { return my_type; }
     const Value::Index &index() const override { return TrivialIndex::get(); }
-    TypedCells cells() const override { return TypedCells(my_cells.memory, get_cell_type<T>(), my_cells.size); }
+    TypedCells cells() const override { return TypedCells(my_cells.memory.get(), get_cell_type<T>(), my_cells.size); }
     ArrayRef<T> add_subspace(ConstArrayRef<vespalib::stringref>) override {
         return ArrayRef<T>(my_cells.get(0), my_cells.size);
     }