summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--document/src/tests/documenttestcase.cpp2
-rw-r--r--fnet/src/tests/info/info.cpp4
-rw-r--r--memfilepersistence/src/tests/spi/buffer_test.cpp4
-rw-r--r--memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp2
-rw-r--r--searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp2
-rw-r--r--searchlib/src/tests/datastore/logdatastore_test.cpp2
-rw-r--r--vespalib/src/tests/alloc/alloc_test.cpp6
-rw-r--r--vespalib/src/tests/array/array_test.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/util/alloc.cpp158
-rw-r--r--vespalib/src/vespa/vespalib/util/alloc.h60
10 files changed, 133 insertions, 109 deletions
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index 1fc1b27ca4b..352c22bd3fd 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -81,7 +81,7 @@ void DocumentTest::testSizeOf()
CPPUNIT_ASSERT_EQUAL(120ul, sizeof(Document));
CPPUNIT_ASSERT_EQUAL(64ul, sizeof(StructFieldValue));
CPPUNIT_ASSERT_EQUAL(16ul, sizeof(StructuredFieldValue));
- CPPUNIT_ASSERT_EQUAL(112ul, sizeof(SerializableArray));
+ CPPUNIT_ASSERT_EQUAL(120ul, sizeof(SerializableArray));
}
void DocumentTest::testFieldPath()
diff --git a/fnet/src/tests/info/info.cpp b/fnet/src/tests/info/info.cpp
index 284be22db63..34b2e851ec7 100644
--- a/fnet/src/tests/info/info.cpp
+++ b/fnet/src/tests/info/info.cpp
@@ -73,9 +73,9 @@ TEST("size of important objects")
EXPECT_EQUAL(184u, sizeof(FNET_IOComponent));
EXPECT_EQUAL(32u, sizeof(FNET_Channel));
EXPECT_EQUAL(40u, sizeof(FNET_PacketQueue_NoLock));
- EXPECT_EQUAL(512u, sizeof(FNET_Connection));
+ EXPECT_EQUAL(536u, sizeof(FNET_Connection));
EXPECT_EQUAL(96u, sizeof(FNET_Cond));
- EXPECT_EQUAL(48u, sizeof(FNET_DataBuffer));
+ EXPECT_EQUAL(56u, sizeof(FNET_DataBuffer));
EXPECT_EQUAL(24u, sizeof(FastOS_Time));
EXPECT_EQUAL(8u, sizeof(FNET_Context));
EXPECT_EQUAL(8u, sizeof(fastos::TimeStamp));
diff --git a/memfilepersistence/src/tests/spi/buffer_test.cpp b/memfilepersistence/src/tests/spi/buffer_test.cpp
index fdbd7a32f17..0addb1032f5 100644
--- a/memfilepersistence/src/tests/spi/buffer_test.cpp
+++ b/memfilepersistence/src/tests/spi/buffer_test.cpp
@@ -36,8 +36,8 @@ BufferTest::getSizeReturnsInitiallyAllocatedSize()
void
BufferTest::getSizeReturnsUnAlignedSizeForMMappedAllocs()
{
- Buffer buf(vespalib::alloc::MMapAllocator::HUGEPAGE_SIZE + 1);
- CPPUNIT_ASSERT_EQUAL(size_t(vespalib::alloc::MMapAllocator::HUGEPAGE_SIZE + 1), buf.getSize());
+ Buffer buf(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1);
+ CPPUNIT_ASSERT_EQUAL(size_t(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1), buf.getSize());
}
void
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp
index 886c2a35c13..38c2e76929c 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp
@@ -16,7 +16,7 @@ namespace memfile {
// It is crucial that any backing buffer type returns an address that is
// 512-byte aligned, or direct IO will scream at us and fail everything.
Buffer::Buffer(size_t size)
- : _buffer(DefaultAlloc::create(size, MMapAllocator::HUGEPAGE_SIZE, 512)),
+ : _buffer(DefaultAlloc::create(size, MemoryAllocator::HUGEPAGE_SIZE, 512)),
_size(size)
{
}
diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
index e34613d24ce..1983ff7a318 100644
--- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
+++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
@@ -49,7 +49,7 @@ shafile(const string &baseDir,
string fullFile(prependBaseDir(baseDir, file));
FastOS_File f;
std::ostringstream os;
- Alloc buf = DefaultAlloc::create(65536, MMapAllocator::HUGEPAGE_SIZE, 0x1000);
+ Alloc buf = DefaultAlloc::create(65536, MemoryAllocator::HUGEPAGE_SIZE, 0x1000);
f.EnableDirectIO();
bool openres = f.OpenReadOnly(fullFile.c_str());
if (!openres) {
diff --git a/searchlib/src/tests/datastore/logdatastore_test.cpp b/searchlib/src/tests/datastore/logdatastore_test.cpp
index 12a7fc28a7b..e246fcee981 100644
--- a/searchlib/src/tests/datastore/logdatastore_test.cpp
+++ b/searchlib/src/tests/datastore/logdatastore_test.cpp
@@ -146,7 +146,7 @@ TEST("test that DirectIOPadding works accordng to spec") {
FastOS_File file("directio.test");
file.EnableDirectIO();
EXPECT_TRUE(file.OpenReadWrite());
- Alloc buf(DefaultAlloc::create(FILE_SIZE, MMapAllocator::HUGEPAGE_SIZE, 4096));
+ Alloc buf(DefaultAlloc::create(FILE_SIZE, MemoryAllocator::HUGEPAGE_SIZE, 4096));
memset(buf.get(), 'a', buf.size());
EXPECT_EQUAL(FILE_SIZE, file.Write2(buf.get(), FILE_SIZE));
size_t padBefore(0);
diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp
index c1ba982aa32..b16afbcc7a6 100644
--- a/vespalib/src/tests/alloc/alloc_test.cpp
+++ b/vespalib/src/tests/alloc/alloc_test.cpp
@@ -53,7 +53,7 @@ Test::testBasic()
EXPECT_TRUE(h.get() != NULL);
}
{
- EXPECT_EXCEPTION(AlignedHeapAllocFactory::create(100, 0), IllegalArgumentException, "posix_memalign(100, 0) failed with code 22");
+ EXPECT_EXCEPTION(AlignedHeapAllocFactory::create(100, 7), IllegalArgumentException, "AlignedHeapAllocFactory::create(100, 7) does not support 7 alignment");
Alloc h = AlignedHeapAllocFactory::create(100, 1024);
EXPECT_EQUAL(100u, h.size());
EXPECT_TRUE(h.get() != NULL);
@@ -86,13 +86,13 @@ void
Test::testAlignedAllocation()
{
{
- Alloc buf = AutoAllocFactory::create(10, 2048, 1024);
+ Alloc buf = AutoAllocFactory::create(10, MemoryAllocator::HUGEPAGE_SIZE, 1024);
EXPECT_TRUE(reinterpret_cast<ptrdiff_t>(buf.get()) % 1024 == 0);
}
{
// Mmapped pointers are page-aligned, but sanity test anyway.
- Alloc buf = AutoAllocFactory::create(3000, 1024, 512);
+ Alloc buf = AutoAllocFactory::create(3000000, MemoryAllocator::HUGEPAGE_SIZE, 512);
EXPECT_TRUE(reinterpret_cast<ptrdiff_t>(buf.get()) % 512 == 0);
}
}
diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp
index 70eb07131f5..231bc00611a 100644
--- a/vespalib/src/tests/array/array_test.cpp
+++ b/vespalib/src/tests/array/array_test.cpp
@@ -157,7 +157,7 @@ void Test::testArray(const T & a, const T & b)
{
Array<T> array;
- ASSERT_EQUAL(sizeof(array), 24u);
+ ASSERT_EQUAL(sizeof(array), 32u);
ASSERT_EQUAL(array.size(), 0u);
ASSERT_EQUAL(array.capacity(), 0u);
for(size_t i(0); i < 5; i++) {
diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp
index abe4c193860..b037535a635 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.cpp
+++ b/vespalib/src/vespa/vespalib/util/alloc.cpp
@@ -13,6 +13,7 @@
#include <vespa/log/log.h>
#include <map>
#include <atomic>
+#include <unordered_map>
LOG_SETUP(".vespalib.alloc");
@@ -84,20 +85,114 @@ size_t sum(const MMapStore & s)
return sum;
}
+class MMapLimitAndAlignment {
+public:
+ MMapLimitAndAlignment(size_t mmapLimit, size_t alignment);
+ uint32_t hash() const { return _key; }
+ bool operator == (MMapLimitAndAlignment rhs) const { return _key == rhs._key; }
+private:
+ uint32_t _key;
+};
+
+void verifyMMapLimitAndAlignment(size_t mmapLimit, size_t alignment) __attribute__((noinline));
+
+void verifyMMapLimitAndAlignment(size_t mmapLimit, size_t alignment) {
+ if ((0x01ul << Optimized::msbIdx(mmapLimit)) != mmapLimit) {
+ throw IllegalArgumentException(make_string("We only support mmaplimit(%0lx) to be a power of 2", mmapLimit));
+ }
+ if ((alignment != 0) && (0x01ul << Optimized::msbIdx(alignment)) != alignment) {
+ throw IllegalArgumentException(make_string("We only support alignment(%0lx) to be a power of 2", alignment));
+ }
+}
+
+MMapLimitAndAlignment::MMapLimitAndAlignment(size_t mmapLimit, size_t alignment) :
+ _key(Optimized::msbIdx(mmapLimit) | Optimized::msbIdx(alignment) << 6)
+{
+ verifyMMapLimitAndAlignment(mmapLimit, alignment);
+}
+}
+
+namespace alloc {
+
+class HeapAllocator : public MemoryAllocator {
+public:
+ void * alloc(size_t sz) const override;
+ void free(void * buf, size_t sz) const override;
+ static void * salloc(size_t sz);
+ static void sfree(void * buf, size_t sz);
+ static MemoryAllocator & getDefault();
+};
+
+class AlignedHeapAllocator : public HeapAllocator {
+public:
+ AlignedHeapAllocator(size_t alignment) : _alignment(alignment) { }
+ void * alloc(size_t sz) const override;
+ static MemoryAllocator & get4K();
+ static MemoryAllocator & get1K();
+ static MemoryAllocator & get512B();
+private:
+ size_t _alignment;
+};
+
+class MMapAllocator : public MemoryAllocator {
+public:
+ void * alloc(size_t sz) const override;
+ void free(void * buf, size_t sz) const override;
+ static void * salloc(size_t sz);
+ static void sfree(void * buf, size_t sz);
+ static MemoryAllocator & getDefault();
+};
+
+class AutoAllocator : public MemoryAllocator {
+public:
+ AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { }
+ void * alloc(size_t sz) const override;
+ void free(void * buf, size_t sz) const override;
+ static MemoryAllocator & getDefault();
+ static MemoryAllocator & getAllocator(size_t mmapLimit, size_t alignment);
+private:
+ size_t roundUpToHugePages(size_t sz) const {
+ return (_mmapLimit >= MemoryAllocator::HUGEPAGE_SIZE)
+ ? MMapAllocator::roundUpToHugePages(sz)
+ : sz;
+ }
+ bool useMMap(size_t sz) const { return (sz >= _mmapLimit); }
+ size_t _mmapLimit;
+ size_t _alignment;
+};
+
+
+namespace {
+
+struct MMapLimitAndAlignmentHash {
+ std::size_t operator ()(MMapLimitAndAlignment key) const { return key.hash(); }
+};
+
+using AutoAllocatorsMap = std::unordered_map<MMapLimitAndAlignment, AutoAllocator::UP, MMapLimitAndAlignmentHash>;
+
+AutoAllocatorsMap createAutoAllocators() {
+ AutoAllocatorsMap map;
+ map.reserve(15);
+ for (size_t alignment : {0,0x200, 0x400, 0x1000}) {
+ for (size_t pages : {1,2,4,8,16}) {
+ size_t mmapLimit = pages * MemoryAllocator::HUGEPAGE_SIZE;
+ MMapLimitAndAlignment key(mmapLimit, alignment);
+ auto result = map.emplace(key, AutoAllocator::UP(new AutoAllocator(mmapLimit, alignment)));
+ assert( result.second );
+ }
+ }
+ return map;
+}
+
+AutoAllocatorsMap _G_availableAutoAllocators = createAutoAllocators();
alloc::HeapAllocator _G_heapAllocatorDefault;
-alloc::AlignedHeapAllocator _G_4KalignedHeapAllocator(4096);
+alloc::AlignedHeapAllocator _G_4KalignedHeapAllocator(1024);
+alloc::AlignedHeapAllocator _G_1KalignedHeapAllocator(4096);
alloc::AlignedHeapAllocator _G_512BalignedHeapAllocator(512);
alloc::MMapAllocator _G_mmapAllocatorDefault;
-alloc::AutoAllocator _G_autoAllocatorDefault(alloc::MMapAllocator::HUGEPAGE_SIZE, 0);
-alloc::AutoAllocator _G_2PautoAllocatorDefault(2 * alloc::MMapAllocator::HUGEPAGE_SIZE, 0);
-alloc::AutoAllocator _G_4PautoAllocatorDefault(4 * alloc::MMapAllocator::HUGEPAGE_SIZE, 0);
-alloc::AutoAllocator _G_8PautoAllocatorDefault(8 * alloc::MMapAllocator::HUGEPAGE_SIZE, 0);
-alloc::AutoAllocator _G_16PautoAllocatorDefault(16 * alloc::MMapAllocator::HUGEPAGE_SIZE, 0);
}
-namespace alloc {
-
MemoryAllocator & HeapAllocator::getDefault() {
return _G_heapAllocatorDefault;
}
@@ -106,6 +201,10 @@ MemoryAllocator & AlignedHeapAllocator::get4K() {
return _G_4KalignedHeapAllocator;
}
+MemoryAllocator & AlignedHeapAllocator::get1K() {
+ return _G_1KalignedHeapAllocator;
+}
+
MemoryAllocator & AlignedHeapAllocator::get512B() {
return _G_512BalignedHeapAllocator;
}
@@ -115,23 +214,16 @@ MemoryAllocator & MMapAllocator::getDefault() {
}
MemoryAllocator & AutoAllocator::getDefault() {
- return _G_autoAllocatorDefault;
-}
-
-MemoryAllocator & AutoAllocator::get2P() {
- return _G_2PautoAllocatorDefault;
-}
-
-MemoryAllocator & AutoAllocator::get4P() {
- return _G_4PautoAllocatorDefault;
-}
-
-MemoryAllocator & AutoAllocator::get8P() {
- return _G_8PautoAllocatorDefault;
+ return getAllocator(1 * MemoryAllocator::HUGEPAGE_SIZE, 0);
}
-MemoryAllocator & AutoAllocator::get16P() {
- return _G_16PautoAllocatorDefault;
+MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment) {
+ MMapLimitAndAlignment key(mmapLimit, alignment);
+ auto found = _G_availableAutoAllocators.find(key);
+ if (found == _G_availableAutoAllocators.end()) {
+ throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment));
+ }
+ return *(found->second);
}
void * HeapAllocator::alloc(size_t sz) const {
@@ -268,10 +360,12 @@ AlignedHeapAllocFactory::create(size_t sz, size_t alignment)
return Alloc(&AlignedHeapAllocator::getDefault(), sz);
} else if (alignment == 0x200) {
return Alloc(&AlignedHeapAllocator::get512B(), sz);
+ } else if (alignment == 0x400) {
+ return Alloc(&AlignedHeapAllocator::get1K(), sz);
} else if (alignment == 0x1000) {
return Alloc(&AlignedHeapAllocator::get4K(), sz);
} else {
- abort();
+ throw IllegalArgumentException(make_string("AlignedHeapAllocFactory::create(%zu, %zu) does not support %zu alignment", sz, alignment, alignment));
}
}
@@ -284,21 +378,7 @@ MMapAllocFactory::create(size_t sz)
Alloc
AutoAllocFactory::create(size_t sz, size_t mmapLimit, size_t alignment)
{
- if (alignment == 0) {
- if (mmapLimit <= MMapAllocator::HUGEPAGE_SIZE) {
- return Alloc(&AutoAllocator::getDefault(), sz);
- } else if (mmapLimit <= 2*MMapAllocator::HUGEPAGE_SIZE) {
- return Alloc(&AutoAllocator::get2P(), sz);
- } else if (mmapLimit <= 4*MMapAllocator::HUGEPAGE_SIZE) {
- return Alloc(&AutoAllocator::get4P(), sz);
- } else if (mmapLimit <= 8*MMapAllocator::HUGEPAGE_SIZE) {
- return Alloc(&AutoAllocator::get8P(), sz);
- } else {
- return Alloc(&AutoAllocator::get16P(), sz);
- }
- } else {
- abort();
- }
+ return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz);
}
}
diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h
index 23f4923b3c5..4a02bd2977d 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.h
+++ b/vespalib/src/vespa/vespalib/util/alloc.h
@@ -12,6 +12,7 @@ namespace alloc {
class MemoryAllocator {
public:
+ enum {HUGEPAGE_SIZE=0x200000};
using UP = std::unique_ptr<MemoryAllocator>;
MemoryAllocator(const MemoryAllocator &) = delete;
MemoryAllocator & operator = (const MemoryAllocator &) = delete;
@@ -19,62 +20,11 @@ public:
virtual ~MemoryAllocator() { }
virtual void * alloc(size_t sz) const = 0;
virtual void free(void * buf, size_t sz) const = 0;
-};
-
-class HeapAllocator : public MemoryAllocator {
-public:
- void * alloc(size_t sz) const override;
- void free(void * buf, size_t sz) const override;
- static void * salloc(size_t sz);
- static void sfree(void * buf, size_t sz);
- static MemoryAllocator & getDefault();
-};
-
-class AlignedHeapAllocator : public HeapAllocator {
-public:
- AlignedHeapAllocator(size_t alignment) : _alignment(alignment) { }
- void * alloc(size_t sz) const override;
- static MemoryAllocator & get4K();
- static MemoryAllocator & get512B();
-private:
- size_t _alignment;
-};
-
-class MMapAllocator : public MemoryAllocator {
-public:
- enum {HUGEPAGE_SIZE=0x200000};
- void * alloc(size_t sz) const override;
- void free(void * buf, size_t sz) const override;
- static void * salloc(size_t sz);
- static void sfree(void * buf, size_t sz);
static size_t roundUpToHugePages(size_t sz) {
return (sz+(HUGEPAGE_SIZE-1)) & ~(HUGEPAGE_SIZE-1);
}
- static MemoryAllocator & getDefault();
-};
-
-class AutoAllocator : public MemoryAllocator {
-public:
- AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { }
- void * alloc(size_t sz) const override;
- void free(void * buf, size_t sz) const override;
- static MemoryAllocator & getDefault();
- static MemoryAllocator & get2P();
- static MemoryAllocator & get4P();
- static MemoryAllocator & get8P();
- static MemoryAllocator & get16P();
-private:
- size_t roundUpToHugePages(size_t sz) const {
- return (_mmapLimit >= MMapAllocator::HUGEPAGE_SIZE)
- ? MMapAllocator::roundUpToHugePages(sz)
- : sz;
- }
- bool useMMap(size_t sz) const { return (sz >= _mmapLimit); }
- size_t _mmapLimit;
- size_t _alignment;
};
-
/**
* This represents an allocation.
* It can be created, moved, swapped and resized.
@@ -123,12 +73,6 @@ public:
return Alloc(_allocator, sz);
}
protected:
- void alloc(const MemoryAllocator * allocator, size_t sz) {
- assert(_buf == nullptr);
- _allocator = allocator;
- _sz = sz;
- _buf = allocator->alloc(sz);
- }
void * _buf;
size_t _sz;
const MemoryAllocator * _allocator;
@@ -161,7 +105,7 @@ public:
class AutoAllocFactory
{
public:
- static Alloc create(size_t sz=0, size_t mmapLimit=MMapAllocator::HUGEPAGE_SIZE, size_t alignment=0);
+ static Alloc create(size_t sz=0, size_t mmapLimit=MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0);
};
}