Add support for mmapping large allocations.

11 files changed, 186 insertions, 23 deletions

diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp
index 0723f8cca85..77c07a52918 100644
--- a/vespamalloc/src/tests/test1/new_test.cpp
+++ b/vespamalloc/src/tests/test1/new_test.cpp
@@ -1,5 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/size_literals.h>
 #include <vespa/log/log.h>
 #include <malloc.h>
 #include <dlfcn.h>
@@ -170,7 +171,19 @@ TEST("verify mallopt") {
     if (env == MallocLibrary::UNKNOWN) return;
     EXPECT_EQUAL(0, mallopt(M_MMAP_MAX, 0x1000000));
     EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x1000000));
-    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, -1));
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi));
+}
+
+TEST("verify mmap_limit") {
+    MallocLibrary env = detectLibrary();
+    if (env == MallocLibrary::UNKNOWN) return;
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x100000));
+    auto small = std::make_unique<char[]>(16_Ki);
+    auto large_1 = std::make_unique<char[]>(1200_Ki);
+    EXPECT_GREATER(size_t(labs(small.get() - large_1.get())), 1_Ti);
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi));
+    auto large_2 = std::make_unique<char[]>(1200_Ki);
+    EXPECT_LESS(size_t(labs(small.get() - large_2.get())), 1_Ti);
 }
 
 
diff --git a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt
index cfa8018e25f..acac1aa5b85 100644
--- a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt
+++ b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(vespamalloc_malloc OBJECT
     malloc.cpp
     allocchunk.cpp
     common.cpp
+    mmappool.cpp
     threadproxy.cpp
     memblock.cpp
     datasegment.cpp
@@ -17,6 +18,7 @@ vespa_add_library(vespamalloc_mallocd OBJECT
     mallocd.cpp
     allocchunk.cpp
     common.cpp
+    mmappool.cpp
     threadproxy.cpp
     memblockboundscheck.cpp
     memblockboundscheck_d.cpp
@@ -31,6 +33,7 @@ vespa_add_library(vespamalloc_mallocdst16 OBJECT
     mallocdst16.cpp
     allocchunk.cpp
     common.cpp
+    mmappool.cpp
     threadproxy.cpp
     memblockboundscheck.cpp
     memblockboundscheck_dst.cpp
@@ -46,6 +49,7 @@ vespa_add_library(vespamalloc_mallocdst16_nl OBJECT
     mallocdst16_nl.cpp
     allocchunk.cpp
     common.cpp
+    mmappool.cpp
     threadproxy.cpp
     memblockboundscheck.cpp
     memblockboundscheck_dst.cpp
diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h
index 4b0af3199a1..65a86b89bf6 100644
--- a/vespamalloc/src/vespamalloc/malloc/common.h
+++ b/vespamalloc/src/vespamalloc/malloc/common.h
@@ -55,7 +55,7 @@ static constexpr uint32_t NUM_THREADS = 16384;
 using OSMemory = MmapMemory;
 using SizeClassT = int;
 
-constexpr size_t ALWAYS_REUSE_LIMIT = 0x200000ul;
+constexpr size_t ALWAYS_REUSE_LIMIT = 0x100000ul;
    
 inline constexpr int msbIdx(uint64_t v) {
     return (sizeof(v)*8 - 1) - __builtin_clzl(v);
diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h
index dc81178150a..5c65f3703a7 100644
--- a/vespamalloc/src/vespamalloc/malloc/datasegment.h
+++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h
@@ -21,6 +21,7 @@ public:
     void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline));
     void returnBlock(void *ptr) __attribute__((noinline));
     SizeClassT sizeClass(const void * ptr)    const { return _blockList[blockId(ptr)].sizeClass(); }
+    bool containsPtr(const void * ptr)       const { return blockId(ptr) < BlockCount; }
     size_t getMaxSize(const void * ptr)       const { return _blockList[blockId(ptr)].getMaxSize(); }
     const void * start()                      const { return _osMemory.getStart(); }
     const void * end()                        const { return _osMemory.getEnd(); }
diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h
index 2fb0f81826d..4225691329d 100644
--- a/vespamalloc/src/vespamalloc/malloc/malloc.h
+++ b/vespamalloc/src/vespamalloc/malloc/malloc.h
@@ -30,13 +30,25 @@ public:
     void *malloc(size_t sz, std::align_val_t);
     void *realloc(void *oldPtr, size_t sz);
     void free(void *ptr) {
-        freeSC(ptr, _segment.sizeClass(ptr));
+        if (_segment.containsPtr(ptr)) {
+            freeSC(ptr, _segment.sizeClass(ptr));
+        } else {
+            _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr());
+        }
     }
     void free(void *ptr, size_t sz) {
-        freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz)));
+        if (_segment.containsPtr(ptr)) {
+            freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz)));
+        } else {
+            _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr());
+        }
     }
     void free(void *ptr, size_t sz, std::align_val_t alignment) {
-        freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment)));
+        if (_segment.containsPtr(ptr)) {
+            freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment)));
+        } else {
+            _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr());
+        }
     }
     size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); }
     size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); }
@@ -73,6 +85,7 @@ private:
     size_t                     _prAllocLimit;
     DataSegment<MemBlockPtrT>  _segment;
     AllocPool                  _allocPool;
+    MMapPool                   _mmapPool;
     ThreadListT                _threadList;
 };
 
@@ -82,7 +95,8 @@ MemoryManager<MemBlockPtrT, ThreadListT>::MemoryManager(size_t logLimitAtStart)
     _prAllocLimit(logLimitAtStart),
     _segment(),
     _allocPool(_segment),
-    _threadList(_allocPool)
+    _mmapPool(),
+    _threadList(_allocPool, _mmapPool)
 {
     setAllocatorForThreads(this);
     initThisThread();
diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp
new file mode 100644
index 00000000000..eb17f5b9666
--- /dev/null
+++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp
@@ -0,0 +1,88 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespamalloc/malloc/mmappool.h>
+#include <vespamalloc/malloc/common.h>
+#include <cassert>
+#include <sys/mman.h>
+
+namespace vespamalloc {
+
+MMapPool::MMapPool()
+    : _page_size(getpagesize()),
+      _huge_flags((getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0),
+      _count(0),
+      _mutex(),
+      _mappings()
+{
+
+}
+
+MMapPool::~MMapPool() {
+    assert(_mappings.empty());
+}
+void *
+MMapPool::mmap(size_t sz) {
+    void * buf(nullptr);
+    assert((sz & (_page_size - 1)) == 0);
+    if (sz > 0) {
+        const int flags(MAP_ANON | MAP_PRIVATE);
+        const int prot(PROT_READ | PROT_WRITE);
+        size_t mmapId = _count.fetch_add(1);
+        if (sz >= _G_bigBlockLimit) {
+            fprintf(_G_logFile, "mmap %ld of size %ld from : ", mmapId, sz);
+            logStackTrace();
+        }
+        buf = ::mmap(nullptr, sz, prot, flags | _huge_flags, -1, 0);
+        if (buf == MAP_FAILED) {
+            if (!_has_hugepage_failure_just_happened) {
+                _has_hugepage_failure_just_happened = true;
+            }
+            buf = ::mmap(nullptr, sz, prot, flags, -1, 0);
+            if (buf == MAP_FAILED) {
+                fprintf(_G_logFile, "Failed mmaping anonymous of size %ld errno(%d) from : ", sz, errno);
+                logStackTrace();
+                abort();
+            }
+        } else {
+            if (_has_hugepage_failure_just_happened) {
+                _has_hugepage_failure_just_happened = false;
+            }
+        }
+#ifdef __linux__
+        if (sz >= _G_bigBlockLimit) {
+            if (madvise(buf, sz, MADV_DONTDUMP) != 0) {
+                std::error_code ec(errno, std::system_category());
+                fprintf(_G_logFile, "Failed madvise(%p, %ld, MADV_DONTDUMP) = '%s'\n", buf, sz,
+                    ec.message().c_str());
+            }
+        }
+#endif
+        std::lock_guard guard(_mutex);
+        auto [it, inserted] = _mappings.insert(std::make_pair(buf, MMapInfo(mmapId, sz)));
+        assert(inserted);
+        if (sz >= _G_bigBlockLimit) {
+            size_t sum(0);
+            std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; });
+            fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), sum);
+        }
+    }
+    return buf;
+}
+
+void
+MMapPool::unmap(void * ptr) {
+    size_t sz;
+    {
+        std::lock_guard guard(_mutex);
+        auto found = _mappings.find(ptr);
+        if (found == _mappings.end()) {
+            fprintf(_G_logFile, "Not able to unmap %p as it is not registered: ", ptr);
+            logStackTrace();
+            abort();
+        }
+        sz = found->second._sz;
+    }
+    int munmap_ok = ::munmap(ptr, sz);
+    assert(munmap_ok == 0);
+}
+
+}
diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h
new file mode 100644
index 00000000000..7dc38c12826
--- /dev/null
+++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h
@@ -0,0 +1,31 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <atomic>
+#include <map>
+
+namespace vespamalloc {
+
+class MMapPool {
+public:
+    MMapPool();
+    MMapPool(const MMapPool &) = delete;
+    MMapPool & operator =(const MMapPool &) = delete;
+    ~MMapPool();
+    void * mmap(size_t sz);
+    void unmap(void *);
+private:
+    struct MMapInfo {
+        MMapInfo(size_t id, size_t sz) : _id(id), _sz(sz) { }
+        size_t _id;
+        size_t _sz;
+    };
+    const size_t        _page_size;
+    const int           _huge_flags;
+    std::atomic<size_t> _count;
+    std::atomic<bool>   _has_hugepage_failure_just_happened;
+    std::mutex          _mutex;
+    std::map<const void *, MMapInfo> _mappings;
+};
+
+}
diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.h b/vespamalloc/src/vespamalloc/malloc/threadlist.h
index c95760dc015..ca3a58483c9 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadlist.h
+++ b/vespamalloc/src/vespamalloc/malloc/threadlist.h
@@ -17,7 +17,9 @@ class ThreadListT
 public:
     using ThreadPool = ThreadPoolT<MemBlockPtrT, ThreadStatT >;
     using AllocPool = AllocPoolT<MemBlockPtrT>;
-    ThreadListT(AllocPool & pool);
+    ThreadListT(AllocPool & allocPool, MMapPool & mmapPool);
+    ThreadListT(const ThreadListT & tl) = delete;
+    ThreadListT & operator = (const ThreadListT & tl) = delete;
     ~ThreadListT();
     void setParams(size_t threadCacheLimit) {
         ThreadPool::setParams(threadCacheLimit);
@@ -33,13 +35,12 @@ public:
     void info(FILE * os, size_t level=0);
     size_t getMaxNumThreads() const { return NELEMS(_threadVector); }
 private:
-    ThreadListT(const ThreadListT & tl);
-    ThreadListT & operator = (const ThreadListT & tl);
     std::atomic_flag           _isThreaded;
     std::atomic<uint32_t>      _threadCount;
     std::atomic<uint32_t>      _threadCountAccum;
     ThreadPool                 _threadVector[NUM_THREADS];
     AllocPoolT<MemBlockPtrT> & _allocPool;
+    MMapPool                 & _mmapPool;
     static thread_local ThreadPool * _myPool TLS_LINKAGE;
 };
 
diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp
index 8a2cb1de879..5f65e98d0ac 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp
+++ b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp
@@ -6,14 +6,15 @@
 namespace vespamalloc {
 
 template <typename MemBlockPtrT, typename ThreadStatT>
-ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & pool) :
+ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & allocPool, MMapPool & mmapPool) :
     _isThreaded(false),
     _threadCount(0),
     _threadCountAccum(0),
-    _allocPool(pool)
+    _allocPool(allocPool),
+    _mmapPool(mmapPool)
 {
     for (size_t i = 0; i < getMaxNumThreads(); i++) {
-        _threadVector[i].setPool(_allocPool);
+        _threadVector[i].setPool(_allocPool, _mmapPool);
     }
 }
 
diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.h b/vespamalloc/src/vespamalloc/malloc/threadpool.h
index ec89079a415..1276d0129b5 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadpool.h
+++ b/vespamalloc/src/vespamalloc/malloc/threadpool.h
@@ -5,6 +5,7 @@
 #include <vespamalloc/malloc/common.h>
 #include <vespamalloc/malloc/allocchunk.h>
 #include <vespamalloc/malloc/globalpool.h>
+#include <vespamalloc/malloc/mmappool.h>
 
 namespace vespamalloc {
 
@@ -12,12 +13,13 @@ template <typename MemBlockPtrT, typename ThreadStatT >
 class ThreadPoolT
 {
 public:
-    typedef AFList<MemBlockPtrT> ChunkSList;
-    typedef AllocPoolT<MemBlockPtrT> AllocPool;
+    using ChunkSList = AFList<MemBlockPtrT>;
+    using AllocPool = AllocPoolT<MemBlockPtrT>;
     ThreadPoolT();
     ~ThreadPoolT();
-    void setPool(AllocPool & pool) {
-        _allocPool = & pool;
+    void setPool(AllocPool & allocPool, MMapPool & mmapPool) {
+        _allocPool = & allocPool;
+        _mmapPool = & mmapPool;
     }
     int mallopt(int param, int value);
     void malloc(size_t sz, MemBlockPtrT & mem);
@@ -66,7 +68,8 @@ private:
     static constexpr bool alwaysReuse(SizeClassT sc) { return sc > ALWAYS_REUSE_SC_LIMIT; }
 
     AllocPool   * _allocPool;
-    ssize_t       _mmapLimit;
+    MMapPool    * _mmapPool;
+    size_t        _mmapLimit;
     AllocFree     _memList[NUM_SIZE_CLASSES];
     ThreadStatT   _stat[NUM_SIZE_CLASSES];
     uint32_t      _threadId;
diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
index e9b9fabebdc..202a6529c99 100644
--- a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
+++ b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp
@@ -71,13 +71,19 @@ mallocHelper(size_t exactSize,
                 PARANOID_CHECK2( *(int *)2 = 2; );
             }
         } else {
-            af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom);
-            _stat[sc].incExactAlloc();
-            if (af._allocFrom) {
-                af._allocFrom->sub(mem);
-                PARANOID_CHECK2( if (!mem.ptr()) { *(int *)3 = 3; } );
+            if (exactSize > _mmapLimit) {
+                mem = MemBlockPtrT(_mmapPool->mmap(MemBlockPtrT::classSize(sc)), MemBlockPtrT::classSize(sc));
+                mem.setExact(exactSize);
+                mem.free();
             } else {
-                PARANOID_CHECK2( *(int *)4 = 4; );
+                af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom);
+                _stat[sc].incExactAlloc();
+                if (af._allocFrom) {
+                    af._allocFrom->sub(mem);
+                    PARANOID_CHECK2(if (!mem.ptr()) { *(int *) 3 = 3; });
+                } else {
+                    PARANOID_CHECK2(*(int *) 4 = 4;);
+                }
             }
         }
     }
@@ -86,6 +92,7 @@ mallocHelper(size_t exactSize,
 template <typename MemBlockPtrT, typename ThreadStatT >
 ThreadPoolT<MemBlockPtrT, ThreadStatT>::ThreadPoolT() :
     _allocPool(nullptr),
+    _mmapPool(nullptr),
     _mmapLimit(0x40000000),
     _threadId(0),
     _osThreadId(0)