diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-01-07 12:13:54 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-01-07 12:13:54 +0100 |
commit | 95881a2d13b3db13ea4fe936d5195f8b791748ca (patch) | |
tree | b6116441b0e65f69f0486240875c936ac0e7406b | |
parent | a35ffed3918dab6e41fd7052ed82b82bdae54abb (diff) |
Revert "Revert "Balder/vespamalloc should not depend on anything else rebased 2""
26 files changed, 147 insertions, 222 deletions
diff --git a/vespamalloc/CMakeLists.txt b/vespamalloc/CMakeLists.txt index ef629698590..11bc1b60983 100644 --- a/vespamalloc/CMakeLists.txt +++ b/vespamalloc/CMakeLists.txt @@ -3,7 +3,7 @@ add_compile_options(-fvisibility=hidden) add_definitions(-DPARANOID_LEVEL=0) vespa_define_module( - DEPENDS + TEST_DEPENDS fastos vespalib vespalog diff --git a/vespamalloc/src/tests/allocfree/allocfree.cpp b/vespamalloc/src/tests/allocfree/allocfree.cpp index f1ecb74754b..7e81aaa9c1d 100644 --- a/vespamalloc/src/tests/allocfree/allocfree.cpp +++ b/vespamalloc/src/tests/allocfree/allocfree.cpp @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/testkit/testapp.h> #include "producerconsumer.h" +#include <vespa/vespalib/testkit/testapp.h> #include <map> +#include <vespa/log/log.h> +LOG_SETUP("allocfree_test"); + using vespalib::Consumer; using vespalib::Producer; using vespalib::ProducerConsumer; -LOG_SETUP("allocfree_test"); - TEST_SETUP(Test); //----------------------------------------------------------------------------- diff --git a/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp b/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp index 53de3f274cc..86da05311c6 100644 --- a/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp +++ b/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> LOG_SETUP("creatingmanythreads_test"); TEST_SETUP(Test); diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index 39cd237420b..9642c987899 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/testkit/testapp.h> #include "producerconsumer.h" +#include <vespa/vespalib/testkit/testapp.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/util/callstack.h> #include <vespa/log/log.h> +LOG_SETUP("linklist_test"); using vespalib::Consumer; using vespalib::Producer; using vespalib::ProducerConsumer; -LOG_SETUP("linklist_test"); - TEST_SETUP(Test); //----------------------------------------------------------------------------- @@ -60,9 +58,9 @@ List globalList[NumBlocks]; class LinkIn : public Consumer { public: - LinkIn(List::HeadPtr & list, uint32_t maxQueue, bool inverse); + LinkIn(List::AtomicHeadPtr & list, uint32_t maxQueue, bool inverse); private: - List::HeadPtr & _head; + List::AtomicHeadPtr & _head; virtual void consume(void * p) { List * l((List *) p); if ( ! ((l >= &globalList[0]) && (l < &globalList[NumBlocks]))) { abort(); } @@ -70,7 +68,7 @@ private: } }; -LinkIn::LinkIn(List::HeadPtr & list, uint32_t maxQueue, bool inverse) : +LinkIn::LinkIn(List::AtomicHeadPtr & list, uint32_t maxQueue, bool inverse) : Consumer (maxQueue, inverse), _head(list) { @@ -80,10 +78,10 @@ LinkIn::LinkIn(List::HeadPtr & list, uint32_t maxQueue, bool inverse) : class LinkOut : public Producer { public: - LinkOut(List::HeadPtr & list, uint32_t cnt, LinkIn &target) + LinkOut(List::AtomicHeadPtr & list, uint32_t cnt, LinkIn &target) : Producer(cnt, target), _head(list) {} private: - List::HeadPtr & _head; + List::AtomicHeadPtr & _head; virtual void * produce() { void *p = List::linkOut(_head); List *l((List *)p); @@ -96,10 +94,10 @@ private: class LinkInOutAndIn : public ProducerConsumer { public: - LinkInOutAndIn(List::HeadPtr & list, uint32_t cnt, bool inverse) + LinkInOutAndIn(List::AtomicHeadPtr & list, uint32_t cnt, bool inverse) : ProducerConsumer(cnt, inverse), _head(list) { } private: - List::HeadPtr & _head; + List::AtomicHeadPtr & _head; virtual void * produce() { void *p = List::linkOut(_head); List *l((List *)p); @@ -125,10 +123,7 @@ int Test::Main() { ASSERT_EQUAL(1024ul, sizeof(List)); FastOS_ThreadPool pool(128000); - List::HeadPtr sharedList; - sharedList._tag = 1; - List::init(); - List::enableThreadSupport(); + List::AtomicHeadPtr sharedList(List::HeadPtr(nullptr, 1)); fprintf(stderr, "Start populating list\n"); for (size_t i=0; i < NumBlocks; i++) { List * l(&globalList[i]); @@ -143,7 +138,9 @@ int Test::Main() { List *n = List::linkOut(sharedList); ASSERT_TRUE(n == NULL); - sharedList._tag = 1; + List::HeadPtr tmp(sharedList.load()); + tmp._tag = 1; + sharedList.store(tmp); fprintf(stderr, "Start populating list\n"); for (size_t i=0; i < NumBlocks; i++) { List * l(&globalList[i]); diff --git a/vespamalloc/src/tests/allocfree/realloc.cpp b/vespamalloc/src/tests/allocfree/realloc.cpp index 8cfd50d0132..efaf89f7e1b 100644 --- a/vespamalloc/src/tests/allocfree/realloc.cpp +++ b/vespamalloc/src/tests/allocfree/realloc.cpp @@ -1,9 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/testkit/testapp.h> -LOG_SETUP("realloc_test"); +#include <vespa/vespalib/testkit/testapp.h> TEST_SETUP(Test); diff --git a/vespamalloc/src/tests/doubledelete/expectsignal.cpp b/vespamalloc/src/tests/doubledelete/expectsignal.cpp index 0b2d5e154c4..f1fb6eb5694 100644 --- a/vespamalloc/src/tests/doubledelete/expectsignal.cpp +++ b/vespamalloc/src/tests/doubledelete/expectsignal.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/slaveproc.h> -LOG_SETUP("expectsignal_test"); - using namespace vespalib; class Test : public TestApp diff --git a/vespamalloc/src/tests/overwrite/expectsignal.cpp b/vespamalloc/src/tests/overwrite/expectsignal.cpp index 0b2d5e154c4..f1fb6eb5694 100644 --- a/vespamalloc/src/tests/overwrite/expectsignal.cpp +++ b/vespamalloc/src/tests/overwrite/expectsignal.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/slaveproc.h> -LOG_SETUP("expectsignal_test"); - using namespace vespalib; class Test : public TestApp diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp index d7057444505..8c35fe841fe 100644 --- a/vespamalloc/src/tests/overwrite/overwrite.cpp +++ b/vespamalloc/src/tests/overwrite/overwrite.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> -LOG_SETUP("overwrite_test"); - using namespace vespalib; class Test : public TestApp diff --git a/vespamalloc/src/tests/test.cpp b/vespamalloc/src/tests/test.cpp index 24acb3368d8..d6208fdc240 100644 --- a/vespamalloc/src/tests/test.cpp +++ b/vespamalloc/src/tests/test.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <stdio.h> #include <stdlib.h> -#include <vespa/fastos/fastos.h> +#include <vespa/fastos/thread.h> namespace vespamalloc { void info(); diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index 205de0f96fc..3120f1d39d0 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -3,5 +3,6 @@ vespa_add_executable(vespamalloc_testatomic_app TEST SOURCES testatomic.cpp DEPENDS + atomic ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp index 1222493446c..78d94429a3f 100644 --- a/vespamalloc/src/tests/test1/testatomic.cpp +++ b/vespamalloc/src/tests/test1/testatomic.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include <vespa/fastos/thread.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/atomic.h> -#include <vector> - -using vespalib::Atomic; +#include <vespamalloc/malloc/allocchunk.h> class Test : public vespalib::TestApp { @@ -39,10 +37,10 @@ void Test::testSwap(T initial) { T value(initial); - ASSERT_TRUE(Atomic::cmpSwap(&value, initial+1, initial)); + ASSERT_TRUE(vespalib::Atomic::cmpSwap(&value, initial+1, initial)); ASSERT_TRUE(value == initial+1); - ASSERT_TRUE(!Atomic::cmpSwap(&value, initial+2, initial)); + ASSERT_TRUE(!vespalib::Atomic::cmpSwap(&value, initial+2, initial)); ASSERT_TRUE(value == initial+1); } @@ -91,7 +89,7 @@ template <typename T> void Stress<T>::stressSwap(T & value) { for (T old = value; old > 0; old = value) { - if (Atomic::cmpSwap(&value, old-1, old)) { + if (vespalib::Atomic::cmpSwap(&value, old-1, old)) { _successCount++; } else { _failedCount++; @@ -103,6 +101,20 @@ int Test::Main() { TEST_INIT("atomic"); + { + std::atomic<uint32_t> uint32V; + ASSERT_TRUE(uint32V.is_lock_free()); + } + { + std::atomic<uint64_t> uint64V; + ASSERT_TRUE(uint64V.is_lock_free()); + } + { + std::atomic<vespamalloc::TaggedPtr> taggedPtr; + ASSERT_EQUAL(16, sizeof(vespamalloc::TaggedPtr)); + ASSERT_TRUE(taggedPtr.is_lock_free()); + } + testSwap<uint32_t>(6); testSwap<uint32_t>(7); testSwap<uint32_t>(uint32_t(-6)); diff --git a/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp b/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp index 1a21e6f1c14..ac9da6a26af 100644 --- a/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp +++ b/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp @@ -1,72 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/allocchunk.h> +#include "allocchunk.h" namespace vespamalloc { -char AFListBase::_atomicLinkSpace[sizeof(AFListBase::AtomicLink)]; -char AFListBase::_lockedLinkSpace[sizeof(AFListBase::LockedLink)]; -AFListBase::LinkI *AFListBase::_link = NULL; -void AFListBase::init() -{ - _link = new (_atomicLinkSpace)AtomicLink(); -} - -AFListBase::LinkI::~LinkI() -{ -} - -void AFListBase::linkInList(HeadPtr & head, AFListBase * list) +void AFListBase::linkInList(AtomicHeadPtr & head, AFListBase * list) { AFListBase * tail; for (tail = list; tail->_next != NULL ;tail = tail->_next) { } linkIn(head, list, tail); } -void AFListBase::AtomicLink::linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail) +void AFListBase::linkIn(AtomicHeadPtr & head, AFListBase * csl, AFListBase * tail) { - HeadPtr oldHead = head; + HeadPtr oldHead = head.load(std::memory_order_relaxed); HeadPtr newHead(csl, oldHead._tag + 1); tail->_next = static_cast<AFListBase *>(oldHead._ptr); - while ( ! Atomic::cmpSwap(&head, newHead, oldHead) ) { - oldHead = head; + while ( ! head.compare_exchange_weak(oldHead, newHead, std::memory_order_release, std::memory_order_relaxed) ) { newHead._tag = oldHead._tag + 1; tail->_next = static_cast<AFListBase *>(oldHead._ptr); } } -void AFListBase::LockedLink::linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail) -{ - Guard guard(_mutex); - HeadPtr newHead(csl, head._tag + 1); - tail->_next = static_cast<AFListBase *>(head._ptr); - head = newHead; -} - -AFListBase * AFListBase::LockedLink::linkOut(HeadPtr & head) -{ - Guard guard(_mutex); - HeadPtr oldHead = head; - AFListBase *csl = static_cast<AFListBase *>(oldHead._ptr); - if (csl == NULL) { - return NULL; - } - HeadPtr newHead(csl->_next, oldHead._tag + 1); - head = newHead; - csl->_next = NULL; - return csl; -} - -AFListBase * AFListBase::AtomicLink::linkOut(HeadPtr & head) +AFListBase * AFListBase::linkOut(AtomicHeadPtr & head) { - HeadPtr oldHead = head; + HeadPtr oldHead = head.load(std::memory_order_relaxed); AFListBase *csl = static_cast<AFListBase *>(oldHead._ptr); if (csl == NULL) { return NULL; } HeadPtr newHead(csl->_next, oldHead._tag + 1); - while ( ! Atomic::cmpSwap(&head, newHead, oldHead) ) { - oldHead = head; + while ( ! head.compare_exchange_weak(oldHead, newHead, std::memory_order_acquire, std::memory_order_relaxed) ) { csl = static_cast<AFListBase *>(oldHead._ptr); if (csl == NULL) { return NULL; diff --git a/vespamalloc/src/vespamalloc/malloc/allocchunk.h b/vespamalloc/src/vespamalloc/malloc/allocchunk.h index 48128e12687..6db29678b3d 100644 --- a/vespamalloc/src/vespamalloc/malloc/allocchunk.h +++ b/vespamalloc/src/vespamalloc/malloc/allocchunk.h @@ -1,53 +1,43 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/common.h> +#include "common.h" #include <algorithm> namespace vespamalloc { +/** + * @brief Pointer and tag - use instead of bare pointer for cmpSwap() + * + * When making a lock-free data structure by using cmpSwap + * on pointers, you'll often run into the "ABA problem", see + * http://en.wikipedia.org/wiki/ABA_problem for details. + * The TaggedPtr makes it easy to do the woraround with tag bits, + * but requires the double-word compare-and-swap instruction. + * Very early Amd K7/8 CPUs are lacking this and will fail (Illegal Instruction). + **/ +struct TaggedPtr { + TaggedPtr() noexcept : _ptr(nullptr), _tag(0) { } + TaggedPtr(void *h, size_t t) noexcept : _ptr(h), _tag(t) {} + + void *_ptr; + size_t _tag; +}; + class AFListBase { public: - typedef Atomic::TaggedPtr HeadPtr; + using HeadPtr = TaggedPtr; + using AtomicHeadPtr = std::atomic<HeadPtr>; AFListBase() : _next(NULL) { } void setNext(AFListBase * csl) { _next = csl; } static void init(); - static void enableThreadSupport() { _link->enableThreadSupport(); } - static void linkInList(HeadPtr & head, AFListBase * list); - static void linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail) { - _link->linkIn(head, csl, tail); - } + static void linkInList(AtomicHeadPtr & head, AFListBase * list); + static void linkIn(AtomicHeadPtr & head, AFListBase * csl, AFListBase * tail); protected: AFListBase * getNext() { return _next; } - static AFListBase * linkOut(HeadPtr & head) { return _link->linkOut(head); } + static AFListBase * linkOut(AtomicHeadPtr & head); private: - class LinkI - { - public: - virtual ~LinkI(); - virtual void enableThreadSupport() { } - virtual void linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail) = 0; - virtual AFListBase * linkOut(HeadPtr & head) = 0; - }; - class AtomicLink : public LinkI - { - private: - virtual void linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail); - virtual AFListBase * linkOut(HeadPtr & head); - }; - class LockedLink : public LinkI - { - public: - virtual void enableThreadSupport() { _mutex.init(); } - private: - virtual void linkIn(HeadPtr & head, AFListBase * csl, AFListBase * tail); - virtual AFListBase * linkOut(HeadPtr & head); - Mutex _mutex; - }; - static char _atomicLinkSpace[sizeof(AtomicLink)]; - static char _lockedLinkSpace[sizeof(LockedLink)]; - static LinkI *_link; AFListBase *_next; }; @@ -74,7 +64,7 @@ public: bool full() const { return (_count == NumBlocks); } size_t fill(void * mem, SizeClassT sc, size_t blocksPerChunk = NumBlocks); AFList * getNext() { return static_cast<AFList *>(AFListBase::getNext()); } - static AFList * linkOut(HeadPtr & head) { + static AFList * linkOut(AtomicHeadPtr & head) { return static_cast<AFList *>(AFListBase::linkOut(head)); } private: diff --git a/vespamalloc/src/vespamalloc/malloc/common.cpp b/vespamalloc/src/vespamalloc/malloc/common.cpp index d14a0317630..ed0551cd853 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.cpp +++ b/vespamalloc/src/vespamalloc/malloc/common.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/common.h> +#include "common.h" #include <pthread.h> namespace vespamalloc { -uint32_t Mutex::_threadCount = 0; +std::atomic<uint32_t> Mutex::_threadCount(0); bool Mutex::_stopRecursion = true; void Mutex::lock() diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index ee08cfbafaa..5041b545b4d 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/atomic.h> -#include <vespa/vespalib/util/optimized.h> #include <new> +#include <atomic> #include <vespamalloc/util/osmem.h> -using vespalib::Atomic; - extern "C" void MallocRecurseOnSuspend(bool recurse) __attribute__ ((noinline)); namespace vespamalloc { @@ -58,13 +55,18 @@ typedef MmapMemory OSMemory; typedef int SizeClassT; + +inline int msbIdx(uint64_t v) { + return v ? 63 - __builtin_clzl(v) : 0; +} + template <size_t MinClassSizeC> class CommonT { public: enum {MinClassSize = MinClassSizeC}; static inline SizeClassT sizeClass(size_t sz) { - SizeClassT tmp(vespalib::Optimized::msbIdx(sz - 1) - (MinClassSizeC - 1)); + SizeClassT tmp(msbIdx(sz - 1) - (MinClassSizeC - 1)); return (sz <= (1 << MinClassSizeC )) ? 0 : tmp; } static inline size_t classSize(SizeClassT sc) { return (size_t(1) << (sc + MinClassSizeC)); } @@ -82,14 +84,14 @@ public: ~Mutex() { quit(); } void lock(); void unlock(); - static void addThread() { Atomic::postInc(&_threadCount); } - static void subThread() { Atomic::postDec(&_threadCount); } + static void addThread() { _threadCount.fetch_add(1); } + static void subThread() { _threadCount.fetch_sub(1); } static void stopRecursion() { _stopRecursion = true; } static void allowRecursion() { _stopRecursion = false; } void init(); void quit(); private: - static uint32_t _threadCount; + static std::atomic<uint32_t> _threadCount; static bool _stopRecursion; Mutex(const Mutex & org); Mutex & operator = (const Mutex & org); diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h index c50d43dc1d8..50f19233f04 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.h +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <limits.h> +#include <climits> #include <memory> #include <vespamalloc/malloc/common.h> #include <vespamalloc/util/traceutil.h> diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.h b/vespamalloc/src/vespamalloc/malloc/globalpool.h index 0669780b796..9757441524e 100644 --- a/vespamalloc/src/vespamalloc/malloc/globalpool.h +++ b/vespamalloc/src/vespamalloc/malloc/globalpool.h @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/common.h> -#include <vespamalloc/malloc/allocchunk.h> -#include <vespamalloc/malloc/datasegment.h> +#include "common.h" +#include "allocchunk.h" +#include "datasegment.h" #include <algorithm> #define USE_STAT2(a) a @@ -46,8 +46,8 @@ private: { public: AllocFree() : _full(), _empty() { } - typename ChunkSList::HeadPtr _full; - typename ChunkSList::HeadPtr _empty; + typename ChunkSList::AtomicHeadPtr _full; + typename ChunkSList::AtomicHeadPtr _empty; }; class Stat { @@ -58,13 +58,13 @@ private: _exchangeFree(0), _exactAlloc(0), _return(0),_malloc(0) { } - size_t _getAlloc; - size_t _getFree; - size_t _exchangeAlloc; - size_t _exchangeFree; - size_t _exactAlloc; - size_t _return; - size_t _malloc; + std::atomic<size_t> _getAlloc; + std::atomic<size_t> _getFree; + std::atomic<size_t> _exchangeAlloc; + std::atomic<size_t> _exchangeFree; + std::atomic<size_t> _exactAlloc; + std::atomic<size_t> _return; + std::atomic<size_t> _malloc; bool isUsed() const { // Do not count _getFree. return (_getAlloc || _exchangeAlloc || _exchangeFree || _exactAlloc || _return || _malloc); @@ -73,11 +73,11 @@ private: Mutex _mutex; ChunkSList * _chunkPool; - AllocFree _scList[NUM_SIZE_CLASSES] VESPALIB_ATOMIC_TAGGEDPTR_ALIGNMENT; + AllocFree _scList[NUM_SIZE_CLASSES]; DataSegment<MemBlockPtrT> & _dataSegment; - size_t _getChunks; - size_t _getChunksSum; - size_t _allocChunkList; + std::atomic<size_t> _getChunks; + std::atomic<size_t> _getChunksSum; + std::atomic<size_t> _allocChunkList; Stat _stat[NUM_SIZE_CLASSES]; static size_t _threadCacheLimit __attribute__((visibility("hidden"))); static size_t _alwaysReuseLimit __attribute__((visibility("hidden"))); diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp index b620c388fb6..d84305ac2ee 100644 --- a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp +++ b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/globalpool.h> +#include "globalpool.h" #define USE_STAT2(a) a -using vespalib::Atomic; - namespace vespamalloc { template <typename MemBlockPtrT> @@ -22,7 +20,6 @@ AllocPoolT<MemBlockPtrT>::AllocPoolT(DataSegment<MemBlockPtrT> & ds) _getChunksSum(0), _allocChunkList(0) { - ChunkSList::init(); memset(_scList, 0, sizeof(_scList)); } @@ -34,7 +31,6 @@ AllocPoolT<MemBlockPtrT>::~AllocPoolT() template <typename MemBlockPtrT> void AllocPoolT<MemBlockPtrT>::enableThreadSupport() { - ChunkSList::enableThreadSupport(); _mutex.init(); } @@ -42,11 +38,11 @@ template <typename MemBlockPtrT> typename AllocPoolT<MemBlockPtrT>::ChunkSList * AllocPoolT<MemBlockPtrT>::getFree(SizeClassT sc) { - typename ChunkSList::HeadPtr & empty = _scList[sc]._empty; + typename ChunkSList::AtomicHeadPtr & empty = _scList[sc]._empty; ChunkSList * csl(NULL); while ((csl = ChunkSList::linkOut(empty)) == NULL) { Guard sync(_mutex); - if (empty._ptr == NULL) { + if (empty.load(std::memory_order_relaxed)._ptr == NULL) { ChunkSList * ncsl(getChunks(sync, 1)); if (ncsl) { ChunkSList::linkInList(empty, ncsl); @@ -65,10 +61,10 @@ typename AllocPoolT<MemBlockPtrT>::ChunkSList * AllocPoolT<MemBlockPtrT>::getAlloc(SizeClassT sc) { ChunkSList * csl(NULL); - typename ChunkSList::HeadPtr & full = _scList[sc]._full; + typename ChunkSList::AtomicHeadPtr & full = _scList[sc]._full; while ((csl = ChunkSList::linkOut(full)) == NULL) { Guard sync(_mutex); - if (full._ptr == NULL) { + if (full.load(std::memory_order_relaxed)._ptr == NULL) { ChunkSList * ncsl(malloc(sync, sc)); if (ncsl) { ChunkSList::linkInList(full, ncsl); @@ -76,7 +72,7 @@ AllocPoolT<MemBlockPtrT>::getAlloc(SizeClassT sc) return NULL; } } - USE_STAT2(Atomic::postInc(&_stat[sc]._getAlloc)); + USE_STAT2(_stat[sc]._getAlloc.fetch_add(1, std::memory_order_relaxed)); } PARANOID_CHECK1( if (csl->empty() || (csl->count() > ChunkSList::NumBlocks)) { *(int*)0 = 0; } ); return csl; @@ -87,7 +83,7 @@ typename AllocPoolT<MemBlockPtrT>::ChunkSList * AllocPoolT<MemBlockPtrT>::getFree(SizeClassT sc, size_t UNUSED(minBlocks)) { ChunkSList * csl = getFree(sc); - USE_STAT2(Atomic::postInc(&_stat[sc]._getFree)); + USE_STAT2(_stat[sc]._getFree.fetch_add(1, std::memory_order_relaxed)); return csl; } @@ -99,7 +95,7 @@ AllocPoolT<MemBlockPtrT>::exchangeFree(SizeClassT sc, typename AllocPoolT<MemBlo AllocFree & af = _scList[sc]; ChunkSList::linkIn(af._full, csl, csl); ChunkSList *ncsl = getFree(sc); - USE_STAT2(Atomic::postInc(&_stat[sc]._exchangeFree)); + USE_STAT2(_stat[sc]._exchangeFree.fetch_add(1, std::memory_order_relaxed)); return ncsl; } @@ -111,7 +107,7 @@ AllocPoolT<MemBlockPtrT>::exchangeAlloc(SizeClassT sc, typename AllocPoolT<MemBl AllocFree & af = _scList[sc]; ChunkSList::linkIn(af._empty, csl, csl); ChunkSList * ncsl = getAlloc(sc); - USE_STAT2(Atomic::postInc(&_stat[sc]._exchangeAlloc)); + USE_STAT2(_stat[sc]._exchangeAlloc.fetch_add(1, std::memory_order_relaxed)); PARANOID_CHECK1( if (ncsl->empty() || (ncsl->count() > ChunkSList::NumBlocks)) { *(int*)0 = 0; } ); return ncsl; } @@ -126,7 +122,7 @@ AllocPoolT<MemBlockPtrT>::exactAlloc(size_t exactSize, SizeClassT sc, MemBlockPtrT mem(exactBlock, MemBlockPtrT::unAdjustSize(adjustedSize)); csl->add(mem); ChunkSList * ncsl = csl; - USE_STAT2(Atomic::postInc(&_stat[sc]._exactAlloc)); + USE_STAT2(_stat[sc]._exactAlloc.fetch_add(1, std::memory_order_relaxed)); mem.logBigBlock(exactSize, mem.adjustSize(exactSize), MemBlockPtrT::classSize(sc)); PARANOID_CHECK1( if (ncsl->empty() || (ncsl->count() > ChunkSList::NumBlocks)) { *(int*)0 = 0; } ); return ncsl; @@ -149,7 +145,7 @@ AllocPoolT<MemBlockPtrT>::returnMemory(SizeClassT sc, } completelyEmpty = csl; #endif - USE_STAT2(Atomic::postInc(&_stat[sc]._return)); + USE_STAT2(_stat[sc]._return.fetch_add(1, std::memory_order_relaxed)); return completelyEmpty; } @@ -193,7 +189,7 @@ AllocPoolT<MemBlockPtrT>::malloc(const Guard & guard, SizeClassT sc) } } PARANOID_CHECK1( for (ChunkSList * c(csl); c; c = c->getNext()) { if (c->empty()) { *(int*)1 = 1; } } ); - USE_STAT2(Atomic::postInc(&_stat[sc]._malloc)); + USE_STAT2(_stat[sc]._malloc.fetch_add(1, std::memory_order_relaxed)); return csl; } @@ -223,8 +219,8 @@ AllocPoolT<MemBlockPtrT>::getChunks(const Guard & guard, size_t numChunks) } else { csl = NULL; } - USE_STAT2(Atomic::postInc(&_getChunks)); - USE_STAT2(_getChunksSum+=numChunks); + USE_STAT2(_getChunks.fetch_add(1, std::memory_order_relaxed)); + USE_STAT2(_getChunksSum.fetch_add(numChunks, std::memory_order_relaxed)); PARANOID_CHECK1( for (ChunkSList * c(csl); c; c = c->getNext()) { if ( ! c->empty()) { *(int*)1 = 1; } } ); return csl; } @@ -245,7 +241,7 @@ AllocPoolT<MemBlockPtrT>::allocChunkList(const Guard & guard) } newList[chunksInBlock-1].setNext(NULL); } - USE_STAT2(Atomic::postInc(&_allocChunkList)); + USE_STAT2(_allocChunkList.fetch_add(1, std::memory_order_relaxed)); return newList; } @@ -254,16 +250,16 @@ void AllocPoolT<MemBlockPtrT>::info(FILE * os, size_t level) { if (level > 0) { fprintf(os, "GlobalPool getChunks(%ld, %ld) allocChunksList(%ld):\n", - _getChunks, _getChunksSum, _allocChunkList); + _getChunks.load(), _getChunksSum.load(), _allocChunkList.load()); for (size_t i = 0; i < NELEMS(_stat); i++) { const Stat & s = _stat[i]; if (s.isUsed()) { fprintf(os, "SC %2ld(%10ld) GetAlloc(%6ld) GetFree(%6ld) " "ExChangeAlloc(%6ld) ExChangeFree(%6ld) ExactAlloc(%6ld) " "Returned(%6ld) Malloc(%6ld)\n", - i, MemBlockPtrT::classSize(i), s._getAlloc, s._getFree, - s._exchangeAlloc, s._exchangeFree, s._exactAlloc, - s._return, s._malloc); + i, MemBlockPtrT::classSize(i), s._getAlloc.load(), s._getFree.load(), + s._exchangeAlloc.load(), s._exchangeFree.load(), s._exactAlloc.load(), + s._return.load(), s._malloc.load()); } } } diff --git a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h index 9bbfa38c416..aceef34ed09 100644 --- a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h +++ b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h @@ -7,7 +7,6 @@ #include <sys/stat.h> #include <ctype.h> #include <fcntl.h> -#include <vespa/defaults.h> #include <vespamalloc/malloc/malloc.h> #include <vespamalloc/util/callstack.h> diff --git a/vespamalloc/src/vespamalloc/malloc/mmap.cpp b/vespamalloc/src/vespamalloc/malloc/mmap.cpp index 6da13d9ac92..22a2b8fffce 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmap.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmap.cpp @@ -4,7 +4,6 @@ #include <dlfcn.h> #include <stdlib.h> #include <stdio.h> -#include <vespa/vespalib/util/backtrace.h> extern "C" { @@ -57,7 +56,7 @@ void * local_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t } } if ((length >= getLogLimit()) && !isFromVespaMalloc(addr)) { - fprintf (stderr, "mmap requesting block of size %ld from %s\n", length, vespalib::getStackTrace(0).c_str()); + fprintf (stderr, "mmap requesting block of size %ld from %s\n", length, "no backtrace"); } return (*real_func)(addr, length, prot, flags, fd, offset); } @@ -73,7 +72,7 @@ void * local_mmap64(void *addr, size_t length, int prot, int flags, int fd, off6 } } if (length >= getLogLimit() && !isFromVespaMalloc(addr)) { - fprintf (stderr, "mmap requesting block of size %ld from %s\n", length, vespalib::getStackTrace(0).c_str()); + fprintf (stderr, "mmap requesting block of size %ld from %s\n", length, "no backtrace"); } return (*real_func)(addr, length, prot, flags, fd, offset); } @@ -89,7 +88,7 @@ int local_munmap(void *addr, size_t length) } } if ((length >= getLogLimit()) && !isFromVespaMalloc(addr)) { - fprintf (stderr, "munmap releasing block of size %ld from %s\n", length, vespalib::getStackTrace(0).c_str()); + fprintf (stderr, "munmap releasing block of size %ld from %s\n", length, "no backtrace"); } return (*real_func)(addr, length); } diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.h b/vespamalloc/src/vespamalloc/malloc/threadlist.h index 9901c9f6960..875aec2942f 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.h +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/threadpool.h> +#include "threadpool.h" namespace vespamalloc { @@ -41,8 +41,8 @@ private: ThreadListT & operator = (const ThreadListT & tl); enum {ThreadStackSize=2048*1024}; volatile bool _isThreaded; - volatile size_t _threadCount; - volatile size_t _threadCountAccum; + std::atomic<size_t> _threadCount; + std::atomic<size_t> _threadCountAccum; ThreadPool _threadVector[NUM_THREADS]; AllocPoolT<MemBlockPtrT> & _allocPool; static __thread ThreadPool * _myPool TLS_LINKAGE; diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp index a1ea517beed..8aa9b6a90b5 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/threadlist.h> +#include "threadlist.h" namespace vespamalloc { @@ -48,7 +48,7 @@ bool ThreadListT<MemBlockPtrT, ThreadStatT>::quitThisThread() { ThreadPool & tp = getCurrent(); tp.quit(); - Atomic::postDec(&_threadCount); + _threadCount.fetch_sub(1); return true; } @@ -56,8 +56,8 @@ template <typename MemBlockPtrT, typename ThreadStatT> bool ThreadListT<MemBlockPtrT, ThreadStatT>::initThisThread() { bool retval(true); - Atomic::postInc(&_threadCount); - size_t lidAccum = Atomic::postInc(&_threadCountAccum); + _threadCount.fetch_add(1); + size_t lidAccum = _threadCountAccum.fetch_add(1); long localId(-1); for(size_t i = 0; (localId < 0) && (i < getMaxNumThreads()); i++) { ThreadPool & tp = _threadVector[i]; diff --git a/vespamalloc/src/vespamalloc/malloc/threadproxy.cpp b/vespamalloc/src/vespamalloc/malloc/threadproxy.cpp index 17da09f9b35..193215ef83d 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadproxy.cpp +++ b/vespamalloc/src/vespamalloc/malloc/threadproxy.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/threadproxy.h> + +#include "threadproxy.h" #include <dlfcn.h> -#include <pthread.h> -#include <cstdio> -#include <cerrno> namespace vespamalloc { @@ -34,7 +32,7 @@ typedef int (*pthread_create_function) (pthread_t *thread, int linuxthreads_pthread_getattr_np(pthread_t pid, pthread_attr_t *dst); static void * _G_mallocThreadProxyReturnAddress = NULL; -static volatile size_t _G_threadCount = 1; // You always have the main thread. +static std::atomic<size_t> _G_threadCount(1); // You always have the main thread. static void cleanupThread(void * arg) { @@ -42,7 +40,7 @@ static void cleanupThread(void * arg) delete ta; vespamalloc::_G_myMemP->quitThisThread(); vespamalloc::Mutex::subThread(); - vespalib::Atomic::postDec(&_G_threadCount); + _G_threadCount.fetch_sub(1); } void * mallocThreadProxy (void * arg) @@ -77,11 +75,11 @@ VESPA_DLL_EXPORT int local_pthread_create (pthread_t *thread, void * (*start_routine) (void *), void * arg) { - size_t numThreads; - for (numThreads = _G_threadCount - ;(numThreads < vespamalloc::_G_myMemP->getMaxNumThreads()) && ! vespalib::Atomic::cmpSwap(&_G_threadCount, numThreads+1, numThreads) - ; numThreads = _G_threadCount) { - } + size_t numThreads = _G_threadCount; + while ((numThreads < vespamalloc::_G_myMemP->getMaxNumThreads()) + && ! _G_threadCount.compare_exchange_strong(numThreads, numThreads+1)) + { } + if (numThreads >= vespamalloc::_G_myMemP->getMaxNumThreads()) { return EAGAIN; } diff --git a/vespamalloc/src/vespamalloc/malloc/threadproxy.h b/vespamalloc/src/vespamalloc/malloc/threadproxy.h index 4865e5fbd5f..0d86bef9e95 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadproxy.h +++ b/vespamalloc/src/vespamalloc/malloc/threadproxy.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespamalloc/malloc/common.h> +#include "common.h" namespace vespamalloc { diff --git a/vespamalloc/src/vespamalloc/util/index.h b/vespamalloc/src/vespamalloc/util/index.h index f7513114edc..b2c20f7bf02 100644 --- a/vespamalloc/src/vespamalloc/util/index.h +++ b/vespamalloc/src/vespamalloc/util/index.h @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/atomic.h> -#include <stdio.h> +#include <cstddef> namespace vespamalloc { @@ -20,19 +19,4 @@ private: index_t _index; }; -class AtomicIndex -{ -public: - typedef size_t index_t; - AtomicIndex(index_t index = 0) : _index(index) { } - operator index_t () const { return _index; } - index_t operator ++ (int) { return vespalib::Atomic::postInc(&_index); } - index_t operator -- (int) { return vespalib::Atomic::postDec(&_index); } - index_t operator += (index_t v) { return _index += v; } - index_t operator -= (index_t v) { return _index -= v; } -private: - index_t _index; -}; - } - diff --git a/vespamalloc/src/vespamalloc/util/osmem.h b/vespamalloc/src/vespamalloc/util/osmem.h index f5c51c2000d..b95f8ac72e8 100644 --- a/vespamalloc/src/vespamalloc/util/osmem.h +++ b/vespamalloc/src/vespamalloc/util/osmem.h @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <ctype.h> -#include <stdlib.h> +#include <cctype> +#include <cstdlib> #include <unistd.h> -#include <assert.h> -#include <string.h> +#include <cassert> +#include <cstring> #include <algorithm> namespace vespamalloc { |