- Add interface to get the size of the mmapped area.

- Special handle realloc

8 files changed, 121 insertions, 69 deletions

diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt
index cbb851e878d..6c9b4a01e59 100644
--- a/vespamalloc/src/tests/allocfree/CMakeLists.txt
+++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt
@@ -8,10 +8,6 @@ vespa_add_executable(vespamalloc_allocfree_shared_test_app
     allocfree.cpp
     producerconsumer.cpp
 )
-vespa_add_executable(vespamalloc_realloc_test_app
-    SOURCES
-    realloc.cpp
-)
 vespa_add_executable(vespamalloc_linklist_test_app
     SOURCES
     linklist.cpp
@@ -24,5 +20,5 @@ vespa_add_executable(vespamalloc_linklist_test_app
     ${VESPA_ATOMIC_LIB}
 )
 vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK
-               DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app
+               DEPENDS vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app
                        vespamalloc vespamallocd)
diff --git a/vespamalloc/src/tests/allocfree/allocfree_test.sh b/vespamalloc/src/tests/allocfree/allocfree_test.sh
index fc9899a7c6c..aca710b0df6 100755
--- a/vespamalloc/src/tests/allocfree/allocfree_test.sh
+++ b/vespamalloc/src/tests/allocfree/allocfree_test.sh
@@ -7,8 +7,6 @@ TIME=/usr/bin/time
 VESPA_MALLOC_SO=../../../src/vespamalloc/libvespamalloc.so
 VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamallocd.so
 
-LD_PRELOAD=$VESPA_MALLOC_SO ./vespamalloc_realloc_test_app
-LD_PRELOAD=$VESPA_MALLOC_SO_D ./vespamalloc_realloc_test_app
 $TIME ./vespamalloc_linklist_test_app 3
 LD_PRELOAD=$VESPA_MALLOC_SO $TIME ./vespamalloc_allocfree_shared_test_app 3
 LD_PRELOAD=$VESPA_MALLOC_SO_D $TIME ./vespamalloc_allocfree_shared_test_app 3
diff --git a/vespamalloc/src/tests/allocfree/realloc.cpp b/vespamalloc/src/tests/allocfree/realloc.cpp
deleted file mode 100644
index 739664989c8..00000000000
--- a/vespamalloc/src/tests/allocfree/realloc.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/testapp.h>
-
-TEST_SETUP(Test);
-
-char *ptr_add(char *ptr, size_t offset) __attribute__((noinline));
-char *ptr_add(char *ptr, size_t offset) { return ptr + offset; }
-
-int Test::Main() {
-    char * v = static_cast<char *>(malloc(0x400001));
-    char * nv = static_cast<char *>(realloc(v, 0x500001));
-    ASSERT_TRUE(v == nv);
-    v = static_cast<char *>(realloc(nv, 0x600001));
-    ASSERT_TRUE(v != nv);
-    free(v);
-
-    char *t = static_cast<char *>(malloc(70));
-    free (ptr_add(t, 7));
-    t = static_cast<char *>(malloc(0x400001));
-    free (ptr_add(t, 7));
-    return 0;
-}
diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp
index 77c07a52918..bd4d4f55cc3 100644
--- a/vespamalloc/src/tests/test1/new_test.cpp
+++ b/vespamalloc/src/tests/test1/new_test.cpp
@@ -122,20 +122,43 @@ TEST("verify reallocarray") {
 }
 #endif
 
-void verify_vespamalloc_usable_size() {
-    struct AllocInfo { size_t requested; size_t usable;};
-    AllocInfo allocInfo[] = {{0x7, 0x20}, {0x27, 0x40}, {0x47, 0x80}, {0x87, 0x100}, {0x107, 0x200}, {0x207, 0x400},
-                            {0x407, 0x800}, {0x807, 0x1000}, {0x1007, 0x2000}, {0x2007, 0x4000}, {0x4007, 0x8000},
-                            {0x8007, 0x10000}, {0x10007, 0x20000}, {0x20007, 0x40000}, {0x40007, 0x80000}, {0x80007, 0x100000},
-                            {0x100007, 0x200000}, {0x200007, 0x400000}, {0x400007, 0x600000}};
-    for (const AllocInfo & info : allocInfo) {
+namespace {
+
+void
+verify_vespamalloc_usable_size() {
+    struct AllocInfo {
+        size_t requested;
+        size_t usable;
+    };
+    AllocInfo allocInfo[] = {{0x7,      0x20},
+                             {0x27,     0x40},
+                             {0x47,     0x80},
+                             {0x87,     0x100},
+                             {0x107,    0x200},
+                             {0x207,    0x400},
+                             {0x407,    0x800},
+                             {0x807,    0x1000},
+                             {0x1007,   0x2000},
+                             {0x2007,   0x4000},
+                             {0x4007,   0x8000},
+                             {0x8007,   0x10000},
+                             {0x10007,  0x20000},
+                             {0x20007,  0x40000},
+                             {0x40007,  0x80000},
+                             {0x80007,  0x100000},
+                             {0x100007, 0x200000},
+                             {0x200007, 0x400000},
+                             {0x400007, 0x600000}};
+    for (const AllocInfo &info: allocInfo) {
         std::unique_ptr<char[]> buf = std::make_unique<char[]>(info.requested);
         size_t usable_size = malloc_usable_size(buf.get());
         EXPECT_EQUAL(info.usable, usable_size);
     }
 }
 
-enum class MallocLibrary { UNKNOWN, VESPA_MALLOC, VESPA_MALLOC_D};
+enum class MallocLibrary {
+    UNKNOWN, VESPA_MALLOC, VESPA_MALLOC_D
+};
 
 MallocLibrary
 detectLibrary() {
@@ -148,15 +171,27 @@ detectLibrary() {
     return MallocLibrary::UNKNOWN;
 }
 
+MallocLibrary _Env = detectLibrary();
+
+size_t
+cmp(const char * v, char c, size_t count) {
+    size_t errors = 0;
+    for (size_t i(0); i < count; i++) {
+        if (v[i] != c) errors++;
+    }
+    return errors;
+}
+
+}
+
 TEST("verify malloc_usable_size is sane") {
     constexpr size_t SZ = 33;
     std::unique_ptr<char[]> buf = std::make_unique<char[]>(SZ);
     size_t usable_size = malloc_usable_size(buf.get());
-    MallocLibrary env = detectLibrary();
-    if (env == MallocLibrary::VESPA_MALLOC_D) {
+    if (_Env == MallocLibrary::VESPA_MALLOC_D) {
         // Debug variants will never have more memory available as there is pre/postamble for error detection.
         EXPECT_EQUAL(SZ, usable_size);
-    } else if (env == MallocLibrary::VESPA_MALLOC) {
+    } else if (_Env == MallocLibrary::VESPA_MALLOC) {
         // Normal production vespamalloc will round up
         EXPECT_EQUAL(64u, usable_size);
         verify_vespamalloc_usable_size();
@@ -167,16 +202,14 @@ TEST("verify malloc_usable_size is sane") {
 }
 
 TEST("verify mallopt") {
-    MallocLibrary env = detectLibrary();
-    if (env == MallocLibrary::UNKNOWN) return;
+    if (_Env == MallocLibrary::UNKNOWN) return;
     EXPECT_EQUAL(0, mallopt(M_MMAP_MAX, 0x1000000));
     EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x1000000));
     EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi));
 }
 
 TEST("verify mmap_limit") {
-    MallocLibrary env = detectLibrary();
-    if (env == MallocLibrary::UNKNOWN) return;
+    if (_Env == MallocLibrary::UNKNOWN) return;
     EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x100000));
     auto small = std::make_unique<char[]>(16_Ki);
     auto large_1 = std::make_unique<char[]>(1200_Ki);
@@ -186,5 +219,35 @@ TEST("verify mmap_limit") {
     EXPECT_LESS(size_t(labs(small.get() - large_2.get())), 1_Ti);
 }
 
+void
+verifyReallocLarge(char * initial, bool expect_vespamalloc_optimization) {
+    const size_t INITIAL_SIZE = 0x400001;
+    const size_t SECOND_SIZE = 0x500001;
+    const size_t THIRD_SIZE = 0x600001;
+    char *v = static_cast<char *>(realloc(initial, INITIAL_SIZE));
+    memset(v, 0x5b, INITIAL_SIZE);
+    char *nv = static_cast<char *>(realloc(v, SECOND_SIZE));
+    if (expect_vespamalloc_optimization) {
+        ASSERT_TRUE(v == nv);
+    }
+    EXPECT_EQUAL(0u, cmp(nv, 0x5b, INITIAL_SIZE));
+    memset(nv, 0xbe, SECOND_SIZE);
+    v = static_cast<char *>(realloc(nv, THIRD_SIZE));
+    if (expect_vespamalloc_optimization) {
+        ASSERT_TRUE(v != nv);
+    }
+    EXPECT_EQUAL(0u, cmp(v, 0xbe, INITIAL_SIZE));
+    free(v);
+}
+TEST("test realloc large buffers") {
+    verifyReallocLarge(nullptr, _Env != MallocLibrary::UNKNOWN);
+    verifyReallocLarge(static_cast<char *>(malloc(2000)), _Env != MallocLibrary::UNKNOWN);
+    if (_Env == MallocLibrary::UNKNOWN) return;
+
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Mi));
+    verifyReallocLarge(nullptr, false);
+    verifyReallocLarge(static_cast<char *>(malloc(2000)), false);
+    EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi));
+}
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp
index 76402c2570d..5ce52330f77 100644
--- a/vespamalloc/src/tests/test1/testatomic.cpp
+++ b/vespamalloc/src/tests/test1/testatomic.cpp
@@ -29,16 +29,20 @@ TEST("test explicit mmap/munmap") {
     void * mmap1 = mmapPool.mmap(0xe000);
     EXPECT_EQUAL(1u, mmapPool.getNumMappings());
     EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes());
+    EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1));
     mmapPool.unmap(mmap1);
     EXPECT_EQUAL(0u, mmapPool.getNumMappings());
     EXPECT_EQUAL(0u, mmapPool.getMmappedBytes());
     mmap1 = mmapPool.mmap(0xe000);
     EXPECT_EQUAL(1u, mmapPool.getNumMappings());
     EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes());
+    EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1));
 
     void * mmap2 = mmapPool.mmap(0x1e000);
     EXPECT_EQUAL(2u, mmapPool.getNumMappings());
     EXPECT_EQUAL(0x2c000u, mmapPool.getMmappedBytes());
+    EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1));
+    EXPECT_EQUAL(0x1e000u, mmapPool.get_size(mmap2));
     mmapPool.unmap(mmap1);
     EXPECT_EQUAL(1u, mmapPool.getNumMappings());
     EXPECT_EQUAL(0x1e000u, mmapPool.getMmappedBytes());
diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h
index cc69d7469c3..614d140c212 100644
--- a/vespamalloc/src/vespamalloc/malloc/malloc.h
+++ b/vespamalloc/src/vespamalloc/malloc/malloc.h
@@ -76,7 +76,7 @@ public:
         return MemBlockPtrT::usable_size(ptr, _segment.getMaxSize<MemBlockPtrT>(ptr));
     }
 
-    void *calloc(size_t nelm, size_t esz) {
+    void * calloc(size_t nelm, size_t esz) {
         void * ptr = malloc(nelm * esz);
         if (ptr) {
             memset(ptr, 0, nelm * esz);
@@ -235,35 +235,41 @@ void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc)
 template <typename MemBlockPtrT, typename ThreadListT>
 void * MemoryManager<MemBlockPtrT, ThreadListT>::realloc(void *oldPtr, size_t sz)
 {
-    void *ptr;
-    if (oldPtr) {
-        MemBlockPtrT mem(oldPtr);
-        mem.readjustAlignment(_segment);
-        if (! mem.validAlloc()) {
-            fprintf(stderr, "Someone has tamper with my pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size());
-            crash();
-        }
-        SizeClassT sc(_segment.sizeClass(oldPtr));
-        if (sc >= 0) {
-            size_t oldSz(_segment.getMaxSize<MemBlockPtrT>(oldPtr));
-            if (sz > oldSz) {
-                ptr = malloc(sz);
-                if (ptr) {
-                    memcpy(ptr, oldPtr, oldSz);
-                    free(oldPtr);
-                }
-            } else {
-                mem.setExact(sz);
-                ptr = oldPtr;
-            }
-        } else {
+    if (oldPtr == nullptr) return malloc(sz);
+    if ( ! _segment.containsPtr(oldPtr)) {
+        void * ptr = malloc(sz);
+        size_t oldBlockSize = _mmapPool.get_size(MemBlockPtrT(oldPtr).rawPtr());
+        memcpy(ptr, oldPtr, MemBlockPtrT::unAdjustSize(oldBlockSize));
+        _mmapPool.unmap(MemBlockPtrT(oldPtr).rawPtr());
+        return ptr;
+    }
+
+    MemBlockPtrT mem(oldPtr);
+    mem.readjustAlignment(_segment);
+    if (! mem.validAlloc()) {
+        fprintf(stderr, "Someone has tamper with my pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size());
+        crash();
+    }
+
+    SizeClassT sc(_segment.sizeClass(oldPtr));
+    void * ptr;
+    if (sc >= 0) {
+        size_t oldSz(_segment.getMaxSize<MemBlockPtrT>(oldPtr));
+        if (sz > oldSz) {
             ptr = malloc(sz);
             if (ptr) {
-                memcpy(ptr, oldPtr, sz);
+                memcpy(ptr, oldPtr, oldSz);
+                free(oldPtr);
             }
+        } else {
+            mem.setExact(sz);
+            ptr = oldPtr;
         }
     } else {
         ptr = malloc(sz);
+        if (ptr) {
+            memcpy(ptr, oldPtr, sz);
+        }
     }
     PARANOID_CHECK2( { MemBlockPtrT mem(ptr); mem.readjustAlignment(_segment); if (! mem.validAlloc()) { crash(); } });
     return ptr;
diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp
index e693ce0de46..eae4b9562d6 100644
--- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp
+++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp
@@ -101,4 +101,11 @@ MMapPool::unmap(void * ptr) {
     assert(munmap_ok == 0);
 }
 
+size_t
+MMapPool::get_size(void * ptr) const {
+    std::lock_guard guard(_mutex);
+    auto found = _mappings.find(ptr);
+    return found->second._sz;
+}
+
 }
diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h
index c07ef9dce15..aa8679c8171 100644
--- a/vespamalloc/src/vespamalloc/malloc/mmappool.h
+++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h
@@ -14,6 +14,7 @@ public:
     ~MMapPool();
     void * mmap(size_t sz);
     void unmap(void *);
+    size_t get_size(void *) const;
     size_t getNumMappings() const;
     size_t getMmappedBytes() const;
 private: