diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-05-11 11:12:29 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-05-11 11:12:29 +0000 |
commit | efe84b0f2c3720e98a26a509fd70a9e2307e14db (patch) | |
tree | e42bc81d52bf766d8b669c2f13ba1626b1d72ea9 /vespalib | |
parent | 331679dc85953f6ab6db2706da92722b03277967 (diff) |
Use a smart allocator for allocating memory for large 'long' lived
vectors. Large vectors will be allocated directly with mmap.
This cancels the main reason for using vespalib::Array.
Diffstat (limited to 'vespalib')
10 files changed, 84 insertions, 22 deletions
diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index d23c2c6b68c..e86a9ad020a 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> +#include <vespa/vespalib/stllike/allocator.h> #include <cstddef> #include <algorithm> @@ -553,4 +554,17 @@ TEST("test that begin and end are identical with empty hashtables") { EXPECT_TRUE(empty_but_reserved.begin() == empty_but_reserved.end()); } +TEST ("test that large_allocator works fine with std::vector") { + using V = std::vector<uint64_t, allocator_large<uint64_t>>; + V a; + a.push_back(1); + a.reserve(14); + for (size_t i(0); i < 400000; i++) { + a.push_back(i); + } + V b = std::move(a); + V c = b; + ASSERT_EQUAL(b.size(), c.size()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index 6b9a4f7201e..2c495ca9a1e 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -44,9 +44,9 @@ public: virtual void move_entries(ICompactable& compactable) = 0; virtual uint32_t get_num_uniques() const = 0; virtual vespalib::MemoryUsage get_memory_usage() const = 0; - virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0; + virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0; virtual void build(vespalib::ConstArrayRef<EntryRef> refs) = 0; - virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0; + virtual void build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) = 0; virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h index f2be4e1237b..cbafdbafd4f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h @@ -3,6 +3,7 @@ #pragma once #include "unique_store_allocator.h" +#include <vespa/vespalib/stllike/allocator.h> namespace vespalib::datastore { @@ -21,8 +22,8 @@ class UniqueStoreBuilder { Allocator& _allocator; IUniqueStoreDictionary& _dict; - std::vector<EntryRef> _refs; - std::vector<uint32_t> _refCounts; + std::vector<EntryRef, allocator_large<EntryRef>> _refs; + std::vector<uint32_t, allocator_large<uint32_t>> _refCounts; public: UniqueStoreBuilder(Allocator& allocator, IUniqueStoreDictionary& dict, uint32_t uniqueValuesHint); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp index 0925702ddcb..d54b86558be 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp @@ -20,9 +20,7 @@ UniqueStoreBuilder<Allocator>::UniqueStoreBuilder(Allocator& allocator, IUniqueS } template <typename Allocator> -UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder() -{ -} +UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder() = default; template <typename Allocator> void diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 4ec98dc2100..8f7a344a99f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -46,9 +46,9 @@ public: void move_entries(ICompactable& compactable) override; uint32_t get_num_uniques() const override; vespalib::MemoryUsage get_memory_usage() const override; - void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override; + void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override; void build(vespalib::ConstArrayRef<EntryRef> refs) override; - void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override; + void build_with_payload(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> payloads) override; std::unique_ptr<ReadSnapshot> get_read_snapshot() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index bce11591970..8ecf71d08c7 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -158,8 +158,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::get_memory_usage() const template <typename DictionaryT, typename ParentT> void -UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> &refs, - const std::vector<uint32_t> &ref_counts, +UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<EntryRef> refs, + vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) { assert(refs.size() == ref_counts.size()); @@ -188,8 +188,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<Entry template <typename DictionaryT, typename ParentT> void -UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs, - const std::vector<uint32_t>& payloads) +UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, + vespalib::ConstArrayRef<uint32_t> payloads) { assert(refs.size() == payloads.size()); typename DictionaryType::Builder builder(_dict.getAllocator()); diff --git a/vespalib/src/vespa/vespalib/stllike/allocator.h b/vespalib/src/vespa/vespalib/stllike/allocator.h new file mode 100644 index 00000000000..3aaa05a7d8f --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/allocator.h @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib { + +/** + * std compliant allocator that will use a smart allocator + * that uses mmap prefering huge pages for large allocations. + * This is a good fit for use with std::vector and std::deque. + */ +template <typename T> +class allocator_large { + using PtrAndSize = alloc::MemoryAllocator::PtrAndSize; +public: + allocator_large() : _allocator(alloc::MemoryAllocator::select_allocator()) {} + using value_type = T; + T * allocate(std::size_t n) { + return static_cast<T *>(_allocator->alloc(n*sizeof(T)).first); + } + void deallocate(T * p, std::size_t n) { + _allocator->free(p, n*sizeof(T)); + } +private: + const alloc::MemoryAllocator * _allocator; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 4c0675198bb..c550964a19b 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -162,6 +162,7 @@ public: AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { } PtrAndSize alloc(size_t sz) const override; void free(PtrAndSize alloc) const override; + void free(void * ptr, size_t sz) const override; size_t resize_inplace(PtrAndSize current, size_t newSize) const override; static MemoryAllocator & getDefault(); static MemoryAllocator & getAllocator(size_t mmapLimit, size_t alignment); @@ -171,13 +172,11 @@ private: ? MMapAllocator::roundUpToHugePages(sz) : sz; } - bool isMMapped(size_t sz) const { return (sz >= _mmapLimit); } bool useMMap(size_t sz) const { - if (_mmapLimit >= HUGEPAGE_SIZE) { - return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit; - } else { - return (sz >= _mmapLimit); - } + return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit; + } + bool isMMapped(size_t sz) const { + return sz >= _mmapLimit; } size_t _mmapLimit; size_t _alignment; @@ -424,7 +423,7 @@ void MMapAllocator::sfree(PtrAndSize alloc) size_t AutoAllocator::resize_inplace(PtrAndSize current, size_t newSize) const { - if (isMMapped(current.second) && useMMap(newSize)) { + if (useMMap(current.second) && useMMap(newSize)) { newSize = roundUpToHugePages(newSize); return MMapAllocator::sresize_inplace(current, newSize); } else { @@ -455,8 +454,21 @@ AutoAllocator::free(PtrAndSize alloc) const { } } +void +AutoAllocator::free(void * ptr, size_t sz) const { + if (useMMap(sz)) { + return MMapAllocator::sfree(PtrAndSize(ptr, roundUpToHugePages(sz))); + } else { + return HeapAllocator::sfree(PtrAndSize(ptr, sz)); + } +} + } +const MemoryAllocator * +MemoryAllocator::select_allocator(size_t mmapLimit, size_t alignment) { + return & AutoAllocator::getAllocator(mmapLimit, alignment); +} Alloc Alloc::allocHeap(size_t sz) diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 03ebc2807f9..449cdde5fc7 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -17,6 +17,10 @@ public: virtual ~MemoryAllocator() { } virtual PtrAndSize alloc(size_t sz) const = 0; virtual void free(PtrAndSize alloc) const = 0; + // Allow for freeing memory there size is the size requested, and not the size allocated. + virtual void free(void * ptr, size_t sz) const { + free(PtrAndSize(ptr, sz)); + } /* * If possible the allocations will be resized. If it was possible it will return the real size, * if not it shall return 0. @@ -30,6 +34,7 @@ public: static size_t roundUpToHugePages(size_t sz) { return (sz+(HUGEPAGE_SIZE-1)) & ~(HUGEPAGE_SIZE-1); } + static const MemoryAllocator * select_allocator(size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); }; /** diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index 186316c10d3..749395ff574 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -15,11 +15,13 @@ class ArrayRef { public: ArrayRef() : _v(nullptr), _sz(0) { } ArrayRef(T * v, size_t sz) : _v(v), _sz(sz) { } - ArrayRef(std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + template<typename A=std::allocator<T>> + ArrayRef(std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { } ArrayRef(Array<T> &v) : _v(&v[0]), _sz(v.size()) { } T & operator [] (size_t i) { return _v[i]; } const T & operator [] (size_t i) const { return _v[i]; } size_t size() const { return _sz; } + bool empty() const { return _sz == 0; } T *begin() { return _v; } T *end() { return _v + _sz; } private: @@ -31,12 +33,14 @@ template <typename T> class ConstArrayRef { public: ConstArrayRef(const T *v, size_t sz) : _v(v), _sz(sz) { } - ConstArrayRef(const std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + template<typename A=std::allocator<T>> + ConstArrayRef(const std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const ArrayRef<T> & v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const Array<T> &v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef() : _v(nullptr), _sz(0) {} const T & operator [] (size_t i) const { return _v[i]; } size_t size() const { return _sz; } + bool empty() const { return _sz == 0; } const T *cbegin() const { return _v; } const T *cend() const { return _v + _sz; } const T *begin() const { return _v; } |