Use a smart allocator for allocating memory for large 'long' lived

vectors. Large vectors will be allocated directly with mmap.
This cancels the main reason for using vespalib::Array.

10 files changed, 84 insertions, 22 deletions

diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp
index d23c2c6b68c..e86a9ad020a 100644
--- a/vespalib/src/tests/stllike/hash_test.cpp
+++ b/vespalib/src/tests/stllike/hash_test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/stllike/hash_set.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/stllike/hash_map_equal.hpp>
+#include <vespa/vespalib/stllike/allocator.h>
 #include <cstddef>
 #include <algorithm>
 
@@ -553,4 +554,17 @@ TEST("test that begin and end are identical with empty hashtables") {
     EXPECT_TRUE(empty_but_reserved.begin() == empty_but_reserved.end());
 }
 
+TEST ("test that large_allocator works fine with std::vector") {
+    using V = std::vector<uint64_t, allocator_large<uint64_t>>;
+    V a;
+    a.push_back(1);
+    a.reserve(14);
+    for (size_t i(0); i < 400000; i++) {
+        a.push_back(i);
+    }
+    V b = std::move(a);
+    V c = b;
+    ASSERT_EQUAL(b.size(), c.size());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
index 6b9a4f7201e..2c495ca9a1e 100644
--- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
@@ -44,9 +44,9 @@ public:
     virtual void move_entries(ICompactable& compactable) = 0;
     virtual uint32_t get_num_uniques() const = 0;
     virtual vespalib::MemoryUsage get_memory_usage() const = 0;
-    virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0;
+    virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0;
     virtual void build(vespalib::ConstArrayRef<EntryRef> refs) = 0;
-    virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0;
+    virtual void build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) = 0;
     virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0;
 };
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
index f2be4e1237b..cbafdbafd4f 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "unique_store_allocator.h"
+#include <vespa/vespalib/stllike/allocator.h>
 
 namespace vespalib::datastore {
 
@@ -21,8 +22,8 @@ class UniqueStoreBuilder {
 
     Allocator& _allocator;
     IUniqueStoreDictionary& _dict;
-    std::vector<EntryRef> _refs;
-    std::vector<uint32_t> _refCounts;
+    std::vector<EntryRef, allocator_large<EntryRef>> _refs;
+    std::vector<uint32_t, allocator_large<uint32_t>> _refCounts;
 
 public:
     UniqueStoreBuilder(Allocator& allocator, IUniqueStoreDictionary& dict, uint32_t uniqueValuesHint);
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp
index 0925702ddcb..d54b86558be 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp
@@ -20,9 +20,7 @@ UniqueStoreBuilder<Allocator>::UniqueStoreBuilder(Allocator& allocator, IUniqueS
 }
 
 template <typename Allocator>
-UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder()
-{
-}
+UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder() = default;
 
 template <typename Allocator>
 void
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
index 4ec98dc2100..8f7a344a99f 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
@@ -46,9 +46,9 @@ public:
     void move_entries(ICompactable& compactable) override;
     uint32_t get_num_uniques() const override;
     vespalib::MemoryUsage get_memory_usage() const override;
-    void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override;
+    void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override;
     void build(vespalib::ConstArrayRef<EntryRef> refs) override;
-    void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override;
+    void build_with_payload(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> payloads) override;
     std::unique_ptr<ReadSnapshot> get_read_snapshot() const override;
 };
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
index bce11591970..8ecf71d08c7 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
@@ -158,8 +158,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::get_memory_usage() const
 
 template <typename DictionaryT, typename ParentT>
 void
-UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> &refs,
-                                                   const std::vector<uint32_t> &ref_counts,
+UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<EntryRef> refs,
+                                                   vespalib::ConstArrayRef<uint32_t> ref_counts,
                                                    std::function<void(EntryRef)> hold)
 {
     assert(refs.size() == ref_counts.size());
@@ -188,8 +188,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<Entry
 
 template <typename DictionaryT, typename ParentT>
 void
-UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs,
-                                                                const std::vector<uint32_t>& payloads)
+UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(vespalib::ConstArrayRef<EntryRef> refs,
+                                                                vespalib::ConstArrayRef<uint32_t> payloads)
 {
     assert(refs.size() == payloads.size());
     typename DictionaryType::Builder builder(_dict.getAllocator());
diff --git a/vespalib/src/vespa/vespalib/stllike/allocator.h b/vespalib/src/vespa/vespalib/stllike/allocator.h
new file mode 100644
index 00000000000..3aaa05a7d8f
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/stllike/allocator.h
@@ -0,0 +1,28 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib {
+
+/**
+ * std compliant allocator that will use a smart allocator
+ * that uses mmap prefering huge pages for large allocations.
+ * This is a good fit for use with std::vector and std::deque.
+ */
+template <typename T>
+class allocator_large {
+    using PtrAndSize = alloc::MemoryAllocator::PtrAndSize;
+public:
+    allocator_large() : _allocator(alloc::MemoryAllocator::select_allocator()) {}
+    using value_type = T;
+    T * allocate(std::size_t n) {
+        return static_cast<T *>(_allocator->alloc(n*sizeof(T)).first);
+    }
+    void deallocate(T * p, std::size_t n) {
+        _allocator->free(p, n*sizeof(T));
+    }
+private:
+    const alloc::MemoryAllocator * _allocator;
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp
index 4c0675198bb..c550964a19b 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.cpp
+++ b/vespalib/src/vespa/vespalib/util/alloc.cpp
@@ -162,6 +162,7 @@ public:
     AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { }
     PtrAndSize alloc(size_t sz) const override;
     void free(PtrAndSize alloc) const override;
+    void free(void * ptr, size_t sz) const override;
     size_t resize_inplace(PtrAndSize current, size_t newSize) const override;
     static MemoryAllocator & getDefault();
     static MemoryAllocator & getAllocator(size_t mmapLimit, size_t alignment);
@@ -171,13 +172,11 @@ private:
             ? MMapAllocator::roundUpToHugePages(sz)
             : sz;
     }
-    bool isMMapped(size_t sz) const { return (sz >= _mmapLimit); }
     bool useMMap(size_t sz) const {
-        if (_mmapLimit >= HUGEPAGE_SIZE) {
-            return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit;
-        } else {
-            return (sz >= _mmapLimit);
-        }
+        return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit;
+    }
+    bool isMMapped(size_t sz) const {
+        return sz >= _mmapLimit;
     }
     size_t _mmapLimit;
     size_t _alignment;
@@ -424,7 +423,7 @@ void MMapAllocator::sfree(PtrAndSize alloc)
 
 size_t
 AutoAllocator::resize_inplace(PtrAndSize current, size_t newSize) const {
-    if (isMMapped(current.second) && useMMap(newSize)) {
+    if (useMMap(current.second) && useMMap(newSize)) {
         newSize = roundUpToHugePages(newSize);
         return MMapAllocator::sresize_inplace(current, newSize);
     } else {
@@ -455,8 +454,21 @@ AutoAllocator::free(PtrAndSize alloc) const {
     }
 }
 
+void
+AutoAllocator::free(void * ptr, size_t sz) const {
+    if (useMMap(sz)) {
+        return MMapAllocator::sfree(PtrAndSize(ptr, roundUpToHugePages(sz)));
+    } else {
+        return HeapAllocator::sfree(PtrAndSize(ptr, sz));
+    }
+}
+
 }
 
+const MemoryAllocator *
+MemoryAllocator::select_allocator(size_t mmapLimit, size_t alignment) {
+    return & AutoAllocator::getAllocator(mmapLimit, alignment);
+}
 
 Alloc
 Alloc::allocHeap(size_t sz)
diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h
index 03ebc2807f9..449cdde5fc7 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.h
+++ b/vespalib/src/vespa/vespalib/util/alloc.h
@@ -17,6 +17,10 @@ public:
     virtual ~MemoryAllocator() { }
     virtual PtrAndSize alloc(size_t sz) const = 0;
     virtual void free(PtrAndSize alloc) const = 0;
+    // Allow for freeing memory there size is the size requested, and not the size allocated.
+    virtual void free(void * ptr, size_t sz) const {
+        free(PtrAndSize(ptr, sz));
+    }
     /*
      * If possible the allocations will be resized. If it was possible it will return the real size,
      * if not it shall return 0.
@@ -30,6 +34,7 @@ public:
     static size_t roundUpToHugePages(size_t sz) {
         return (sz+(HUGEPAGE_SIZE-1)) & ~(HUGEPAGE_SIZE-1);
     }
+    static const MemoryAllocator * select_allocator(size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0);
 };
 
 /**
diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h
index 186316c10d3..749395ff574 100644
--- a/vespalib/src/vespa/vespalib/util/arrayref.h
+++ b/vespalib/src/vespa/vespalib/util/arrayref.h
@@ -15,11 +15,13 @@ class ArrayRef {
 public:
     ArrayRef() : _v(nullptr), _sz(0) { }
     ArrayRef(T * v, size_t sz) : _v(v), _sz(sz) { }
-    ArrayRef(std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { }
+    template<typename A=std::allocator<T>>
+    ArrayRef(std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { }
     ArrayRef(Array<T> &v) : _v(&v[0]), _sz(v.size()) { }
     T & operator [] (size_t i) { return _v[i]; }
     const T & operator [] (size_t i) const { return _v[i]; }
     size_t size() const { return _sz; }
+    bool empty() const { return _sz == 0; }
     T *begin() { return _v; }
     T *end() { return _v + _sz; }
 private:
@@ -31,12 +33,14 @@ template <typename T>
 class ConstArrayRef {
 public:
     ConstArrayRef(const T *v, size_t sz) : _v(v), _sz(sz) { }
-    ConstArrayRef(const std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { }
+    template<typename A=std::allocator<T>>
+    ConstArrayRef(const std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { }
     ConstArrayRef(const ArrayRef<T> & v) : _v(&v[0]), _sz(v.size()) { }
     ConstArrayRef(const Array<T> &v) : _v(&v[0]), _sz(v.size()) { }
     ConstArrayRef() : _v(nullptr), _sz(0) {}
     const T & operator [] (size_t i) const { return _v[i]; }
     size_t size() const { return _sz; }
+    bool empty() const { return _sz == 0; }
     const T *cbegin() const { return _v; }
     const T *cend() const { return _v + _sz; }
     const T *begin() const { return _v; }