// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include "common.h" #include "datasegment.h" #include "allocchunk.h" #include "globalpool.h" #include "threadpool.h" #include "threadlist.h" #include "threadproxy.h" namespace vespamalloc { template class MemBlockInfoT final : public segment::IMemBlockInfo { public: MemBlockInfoT(void *ptr) : _mem(ptr, 0, false) { } bool allocated() const override { return _mem.allocated(); } uint32_t threadId() const override { return _mem.threadId(); } void info(FILE * os, int level) const override { _mem.info(os, level); } uint32_t callStackLen() const override { return _mem.callStackLen(); } const StackEntry * callStack() const override { return _mem.callStack(); } private: MemBlockPtrT _mem; }; template class MemoryManager : public IAllocator, public segment::IHelper { using DataSegment = segment::DataSegment; public: MemoryManager(size_t logLimitAtStart); ~MemoryManager() override; bool initThisThread() override; bool quitThisThread() override; void enableThreadSupport() override; void setReturnAddressStop(const void * returnAddressStop) override { MemBlockPtrT::Stack::setStopAddress(returnAddressStop); } size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); } size_t classSize(SizeClassT sc) const override { return MemBlockPtrT::classSize(sc); } void dumpInfo(int level) const override { MemBlockPtrT::dumpInfo(level); } std::unique_ptr createMemblockInfo(void * ptr) const override { return std::make_unique>(ptr); } int mallopt(int param, int value); void *malloc(size_t sz); void *malloc(size_t sz, std::align_val_t); void *realloc(void *oldPtr, size_t sz); void free(void *ptr) { if (_segment.containsPtr(ptr)) { freeSC(ptr, _segment.sizeClass(ptr)); } else { _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); } } void free(void *ptr, size_t sz) { if (_segment.containsPtr(ptr)) { freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); } else { _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); } } void free(void *ptr, size_t sz, std::align_val_t alignment) { if (_segment.containsPtr(ptr)) { freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); } else { _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); } } size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } size_t usable_size(void *ptr) const { return MemBlockPtrT::usable_size(ptr, _segment); } void * calloc(size_t nelm, size_t esz) { void * ptr = malloc(nelm * esz); if (ptr) { memset(ptr, 0, nelm * esz); } return ptr; } void info(FILE * os, size_t level=0) __attribute__ ((noinline)); void setupSegmentLog(size_t bigMemLogLevel, size_t bigLimit, size_t bigIncrement, size_t allocs2Show) { _segment.setupLog(bigMemLogLevel, bigLimit, bigIncrement, allocs2Show); } void setupLog(size_t prAllocLimit) { _prAllocLimit = prAllocLimit; } void setParams(size_t threadCacheLimit) { _threadList.setParams(threadCacheLimit); _allocPool.setParams(threadCacheLimit); } const DataSegment & dataSegment() const { return _segment; } const MMapPool & mmapPool() const { return _mmapPool; } private: void freeSC(void *ptr, SizeClassT sc); void crash() __attribute__((noinline)); using AllocPool = AllocPoolT; using ThreadPool = typename ThreadListT::ThreadPool; size_t _prAllocLimit; DataSegment _segment; AllocPool _allocPool; MMapPool _mmapPool; ThreadListT _threadList; }; template MemoryManager::MemoryManager(size_t logLimitAtStart) : IAllocator(), _prAllocLimit(logLimitAtStart), _segment(*this), _allocPool(_segment), _mmapPool(), _threadList(_allocPool, _mmapPool) { setAllocatorForThreads(this); initThisThread(); Mutex::allowRecursion(); } template MemoryManager::~MemoryManager() = default; template bool MemoryManager::initThisThread() { bool retval(_threadList.initThisThread()); if ( retval ) { // ThreadPool & tp = _threadList.getCurrent(); // tp.init(_threadList.getThreadId()); } else { abort(); } return retval; } template bool MemoryManager::quitThisThread() { return _threadList.quitThisThread(); } template void MemoryManager::enableThreadSupport() { _segment.enableThreadSupport(); _allocPool.enableThreadSupport(); _threadList.enableThreadSupport(); } template void MemoryManager::crash() { fprintf(stderr, "vespamalloc detected unrecoverable error.\n"); logStackTrace(); abort(); } template void MemoryManager::info(FILE * os, size_t level) { fprintf(os, "DataSegment at %p(%ld), AllocPool at %p(%ld), ThreadList at %p(%ld)\n", &_segment, sizeof(_segment), &_allocPool, sizeof(_allocPool), &_threadList, sizeof(_threadList)); _segment.info(os, level); _allocPool.info(os, level); _threadList.info(os, level); _mmapPool.info(os, level); fflush(os); } template int MemoryManager::mallopt(int param, int value) { return _threadList.getCurrent().mallopt(param, value); } template void * MemoryManager::malloc(size_t sz) { MemBlockPtrT mem; ThreadPool & tp = _threadList.getCurrent(); tp.malloc(mem.adjustSize(sz), mem); if (!mem.validFree()) { fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); crash(); } mem.setExact(sz); mem.alloc(_prAllocLimit<=mem.adjustSize(sz)); return mem.ptr(); } template void * MemoryManager::malloc(size_t sz, std::align_val_t alignment) { MemBlockPtrT mem; ThreadPool & tp = _threadList.getCurrent(); tp.malloc(mem.adjustSize(sz, alignment), mem); if (!mem.validFree()) { fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); crash(); } mem.setExact(sz, alignment); mem.alloc(_prAllocLimit<=mem.adjustSize(sz, alignment)); return mem.ptr(); } template void MemoryManager::freeSC(void *ptr, SizeClassT sc) { if (MemBlockPtrT::verifySizeClass(sc)) { ThreadPool & tp = _threadList.getCurrent(); MemBlockPtrT mem(ptr); mem.readjustAlignment(_segment); if (mem.validAlloc()) { mem.free(); tp.free(mem, sc); } else if (mem.validFree()) { fprintf(stderr, "Already deleted %p(%ld).\n", mem.ptr(), mem.size()); crash(); } else { fprintf(stderr, "Someone has tamper with my pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size()); crash(); } } else { fprintf(stderr, "%p not allocated here, can not be freed\n", ptr); crash(); } } template void * MemoryManager::realloc(void *oldPtr, size_t sz) { if (oldPtr == nullptr) return malloc(sz); if ( ! _segment.containsPtr(oldPtr)) { void * ptr = malloc(sz); size_t oldBlockSize = _mmapPool.get_size(MemBlockPtrT(oldPtr).rawPtr()); memcpy(ptr, oldPtr, MemBlockPtrT::unAdjustSize(oldBlockSize)); _mmapPool.unmap(MemBlockPtrT(oldPtr).rawPtr()); return ptr; } MemBlockPtrT mem(oldPtr); mem.readjustAlignment(_segment); if (! mem.validAlloc()) { fprintf(stderr, "Someone has tampered with the pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size()); crash(); } SizeClassT sc(_segment.sizeClass(oldPtr)); void * ptr; if (sc >= 0) { size_t oldSz(_segment.getMaxSize(oldPtr)); if (sz > oldSz) { ptr = malloc(sz); memcpy(ptr, oldPtr, oldSz); free(oldPtr); } else { mem.setExact(sz); ptr = oldPtr; } } else { ptr = malloc(sz); memcpy(ptr, oldPtr, sz); } PARANOID_CHECK2( { MemBlockPtrT mem(ptr); mem.readjustAlignment(_segment); if (! mem.validAlloc()) { crash(); } }); return ptr; } }