Handle alignment in vespamallocd too.

6 files changed, 83 insertions, 29 deletions

diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h
index b21a2f63ed5..36f1bd0521d 100644
--- a/vespamalloc/src/vespamalloc/malloc/common.h
+++ b/vespamalloc/src/vespamalloc/malloc/common.h
@@ -66,6 +66,7 @@ template <size_t MinClassSizeC>
 class CommonT
 {
 public:
+    static constexpr size_t MAX_ALIGN = 0x200000ul;
     enum {MinClassSize = MinClassSizeC};
     static inline SizeClassT sizeClass(size_t sz) {
         SizeClassT tmp(msbIdx(sz - 1) - (MinClassSizeC - 1));
diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h
index b3131ac5cbb..c3d67e3a5ae 100644
--- a/vespamalloc/src/vespamalloc/malloc/malloc.h
+++ b/vespamalloc/src/vespamalloc/malloc/malloc.h
@@ -16,7 +16,7 @@ class MemoryManager : public IAllocator
 {
 public:
     MemoryManager(size_t logLimitAtStart);
-    ~MemoryManager();
+    ~MemoryManager() override;
     bool initThisThread() override;
     bool quitThisThread() override;
     void enableThreadSupport() override;
@@ -26,9 +26,17 @@ public:
     size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); }
 
     void *malloc(size_t sz);
+    void *malloc(size_t sz, std::align_val_t);
     void *realloc(void *oldPtr, size_t sz);
-    void free(void *ptr) { freeSC(ptr, _segment.sizeClass(ptr)); }
-    void free(void *ptr, size_t sz) { freeSC(ptr, MemBlockPtrT::sizeClass(sz)); }
+    void free(void *ptr) {
+        freeSC(ptr, _segment.sizeClass(ptr));
+    }
+    void free(void *ptr, size_t sz) {
+        freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz)));
+    }
+    void free(void *ptr, size_t sz, std::align_val_t alignment) {
+        freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment)));
+    }
     size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); }
     size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); }
 
@@ -159,13 +167,27 @@ void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz)
         fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size());
         crash();
     }
-    PARANOID_CHECK2(if (!mem.validFree() && mem.ptr()) { crash(); } );
     mem.setExact(sz);
     mem.alloc(_prAllocLimit<=mem.adjustSize(sz));
     return mem.ptr();
 }
 
 template <typename MemBlockPtrT, typename ThreadListT>
+void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz, std::align_val_t alignment)
+{
+    MemBlockPtrT mem;
+    ThreadPool & tp = _threadList.getCurrent();
+    tp.malloc(mem.adjustSize(sz, alignment), mem);
+    if (!mem.validFree()) {
+        fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size());
+        crash();
+    }
+    mem.setExact(sz);
+    mem.alloc(_prAllocLimit<=mem.adjustSize(sz, alignment));
+    return mem.ptr(alignment);
+}
+
+template <typename MemBlockPtrT, typename ThreadListT>
 void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc)
 {
     if (MemBlockPtrT::verifySizeClass(sc)) {
diff --git a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp
index 8e8bb642efc..47c12b4f186 100644
--- a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp
+++ b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp
@@ -8,11 +8,11 @@ typedef ThreadListT<MemBlockBoundsCheck, Stat> ThreadList;
 typedef MemoryWatcher<MemBlockBoundsCheck, ThreadList> Allocator;
 
 static char _Gmem[sizeof(Allocator)];
-static Allocator *_GmemP = NULL;
+static Allocator *_GmemP = nullptr;
 
 static Allocator * createAllocator()
 {
-    if (_GmemP == NULL) {
+    if (_GmemP == nullptr) {
         _GmemP = new (_Gmem) Allocator(-1, 0x7fffffffffffffffl);
     }
     return _GmemP;
diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.h b/vespamalloc/src/vespamalloc/malloc/memblock.h
index 118fb0e046c..8d20f064d39 100644
--- a/vespamalloc/src/vespamalloc/malloc/memblock.h
+++ b/vespamalloc/src/vespamalloc/malloc/memblock.h
@@ -10,14 +10,14 @@ namespace vespamalloc {
 template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC>
 class MemBlockT : public CommonT<MinSizeClassC>
 {
-    static const size_t MAX_ALIGN= 0x200000ul;
 public:
-    typedef StackEntry<StackReturnEntry> Stack;
+    using Parent = CommonT<MinSizeClassC>;
+    using Stack = StackEntry<StackReturnEntry>;
     enum {
         MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC,
         SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC)
     };
-    MemBlockT() : _ptr(NULL) { }
+    MemBlockT() : _ptr(nullptr) { }
     MemBlockT(void * p) : _ptr(p) { }
     MemBlockT(void * p, size_t /*sz*/) : _ptr(p) { }
     MemBlockT(void * p, size_t, bool) : _ptr(p) { }
@@ -25,6 +25,7 @@ public:
     void readjustAlignment(const T & segment)  { (void) segment; }
     void *rawPtr()            { return _ptr; }
     void *ptr()               { return _ptr; }
+    void *ptr(std::align_val_t) { return _ptr; }
     const void *ptr()   const { return _ptr; }
     bool validAlloc()   const { return true; }
     bool validFree()    const { return true; }
@@ -36,12 +37,13 @@ public:
     bool allocated()    const { return false; }
     int threadId()      const { return 0; }
     void info(FILE *, unsigned level=0) const  { (void) level; }
-    Stack * callStack()                   { return NULL; }
+    Stack * callStack()                   { return nullptr; }
     size_t callStackLen()           const { return 0; }
     void fillMemory(size_t)               { }
     void logBigBlock(size_t exact, size_t adjusted, size_t gross) const __attribute__((noinline));
 
     static size_t adjustSize(size_t sz)   { return sz; }
+    static size_t adjustSize(size_t sz, std::align_val_t)   { return sz; }
     static size_t unAdjustSize(size_t sz) { return sz; }
     static void dumpInfo(size_t level);
     static void dumpFile(FILE * fp)       { _logFile = fp; }
@@ -49,9 +51,9 @@ public:
     static void setFill(uint8_t ) { }
     static bool verifySizeClass(int sc) { (void) sc; return true; }
     static size_t getMinSizeForAlignment(size_t align, size_t sz) {
-        return (sz < MAX_ALIGN)
+        return (sz < Parent::MAX_ALIGN)
                    ? std::max(sz, align)
-                   : (align < MAX_ALIGN) ? sz : sz + align;
+                   : (align < Parent::MAX_ALIGN) ? sz : sz + align;
     }
 private:
     void * _ptr;
@@ -64,4 +66,3 @@ template <> void MemBlock::dumpInfo(size_t level);
 extern template class MemBlockT<5, 20>;
 
 }
-
diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h
index 21e9d74c0d2..7176a83812b 100644
--- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h
+++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h
@@ -11,8 +11,22 @@ class MemBlockBoundsCheckBaseTBase : public CommonT<5>
 public:
     typedef StackEntry<StackReturnEntry> Stack;
     void * rawPtr()          { return _ptr; }
-    void *ptr()              { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; }
-    const void *ptr()  const { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; }
+    void *ptr()              {
+        char *p((char*)_ptr);
+        return p ? (p+preambleOverhead()) : nullptr;
+    }
+    const void *ptr() const {
+        const char *p((const char*)_ptr);
+        return p ? (p+preambleOverhead()) : nullptr;
+    }
+    void *ptr(std::align_val_t alignment) {
+        char *p((char*)_ptr);
+        return p ? (p+preambleOverhead(alignment)) : nullptr;
+    }
+    const void *ptr(std::align_val_t alignment) const {
+        const char *p((const char*)_ptr);
+        return p ? (p+preambleOverhead(alignment)) : nullptr;
+    }
 
     void setThreadId(int th)              { if (_ptr) { static_cast<uint32_t*>(_ptr)[2] = th; } }
     bool allocated()                const { return (static_cast<unsigned*>(_ptr)[3] == ALLOC_MAGIC); }
@@ -45,6 +59,12 @@ protected:
     void verifyFill() const __attribute__((noinline));
 
     void setSize(size_t sz) { static_cast<uint64_t *>(_ptr)[0] = sz; }
+    static size_t preambleOverhead(std::align_val_t alignment) {
+        return std::max(4*sizeof(unsigned), size_t(alignment));
+    }
+    static size_t preambleOverhead() {
+        return 4*sizeof(unsigned);
+    }
 
     enum {
         ALLOC_MAGIC = 0xF1E2D3C4,
@@ -121,13 +141,23 @@ public:
         }
         return StackTraceLen;
     }
-    static size_t adjustSize(size_t sz)   { return sz + ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); }
-    static size_t unAdjustSize(size_t sz) { return sz - ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); }
+    static size_t adjustSize(size_t sz)   { return sz + overhead(); }
+    static size_t adjustSize(size_t sz, std::align_val_t alignment)   { return sz + overhead(alignment); }
+    static size_t unAdjustSize(size_t sz) { return sz - overhead(); }
     static void dumpInfo(size_t level) __attribute__((noinline));
     static size_t getMinSizeForAlignment(size_t align, size_t sz) { return sz + align; }
     void info(FILE * os, unsigned level=0) const __attribute__((noinline));
 
 protected:
+    static size_t postambleOverhead() {
+        return sizeof(unsigned) + StackTraceLen*sizeof(void *);
+    }
+    static size_t overhead() {
+        return preambleOverhead() + postambleOverhead();
+    }
+    static size_t overhead(std::align_val_t alignment) {
+        return preambleOverhead(alignment) + postambleOverhead();
+    }
     void setTailMagic() { *(reinterpret_cast<unsigned *> ((char*)_ptr + size() + 4*sizeof(unsigned) + StackTraceLen*sizeof(void *))) = TAIL_MAGIC; }
     void init(size_t sz) {
         if (_ptr) {
diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h
index e41a530c981..56cd8101731 100644
--- a/vespamalloc/src/vespamalloc/malloc/overload.h
+++ b/vespamalloc/src/vespamalloc/malloc/overload.h
@@ -72,11 +72,11 @@ void operator delete[](void* ptr, std::size_t sz, const std::nothrow_t&) noexcep
  * Due to allocation being power of 2 up to huge page size (2M)
  * alignment will always be satisfied. size will always be larger or equal to alignment.
  */
-void* operator new(std::size_t sz, std::align_val_t) {
-    return vespamalloc::_GmemP->malloc(sz);
+void* operator new(std::size_t sz, std::align_val_t alignment) {
+    return vespamalloc::_GmemP->malloc(sz, alignment);
 }
-void* operator new(std::size_t sz, std::align_val_t, const std::nothrow_t&) noexcept {
-    return vespamalloc::_GmemP->malloc(sz);
+void* operator new(std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+    return vespamalloc::_GmemP->malloc(sz, alignment);
 }
 void operator delete(void* ptr , std::align_val_t) noexcept {
     return vespamalloc::_GmemP->free(ptr);
@@ -84,11 +84,11 @@ void operator delete(void* ptr , std::align_val_t) noexcept {
 void operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
     return vespamalloc::_GmemP->free(ptr);
 }
-void* operator new[](std::size_t sz, std::align_val_t) {
-    return vespamalloc::_GmemP->malloc(sz);
+void* operator new[](std::size_t sz, std::align_val_t alignment) {
+    return vespamalloc::_GmemP->malloc(sz, alignment);
 }
-void* operator new[](std::size_t sz, std::align_val_t, const std::nothrow_t&) noexcept {
-    return vespamalloc::_GmemP->malloc(sz);
+void* operator new[](std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+    return vespamalloc::_GmemP->malloc(sz, alignment);
 }
 void operator delete[](void* ptr, std::align_val_t) noexcept {
     return vespamalloc::_GmemP->free(ptr);
@@ -96,11 +96,11 @@ void operator delete[](void* ptr, std::align_val_t) noexcept {
 void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
     return vespamalloc::_GmemP->free(ptr);
 }
-void operator delete(void* ptr, std::size_t, std::align_val_t) noexcept {
-    return vespamalloc::_GmemP->free(ptr);
+void operator delete(void* ptr, std::size_t sz, std::align_val_t alignment) noexcept {
+    return vespamalloc::_GmemP->free(ptr, sz, alignment);
 }
-void operator delete[](void* ptr, std::size_t, std::align_val_t) noexcept {
-    return vespamalloc::_GmemP->free(ptr);
+void operator delete[](void* ptr, std::size_t sz, std::align_val_t alignment) noexcept {
+    return vespamalloc::_GmemP->free(ptr, sz, alignment);
 }
 
 extern "C" {