diff options
Diffstat (limited to 'vespalib/src/vespa')
7 files changed, 55 insertions, 140 deletions
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index dd6c767e9c6..5962e6cf2f7 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -232,7 +232,7 @@ DataStoreBase::transferElemHoldList(generation_t generation) void DataStoreBase::transferHoldLists(generation_t generation) { - _genHolder.transferHoldLists(generation); + _genHolder.assign_generation(generation); if (hasElemHold1()) { transferElemHoldList(generation); } @@ -250,7 +250,7 @@ void DataStoreBase::trimHoldLists(generation_t usedGen) { trimElemHoldList(usedGen); // Trim entries before trimming buffers - _genHolder.trimHoldLists(usedGen); + _genHolder.reclaim(usedGen); } void @@ -258,7 +258,7 @@ DataStoreBase::clearHoldLists() { transferElemHoldList(0); clearElemHoldList(); - _genHolder.clearHoldLists(); + _genHolder.reclaim_all(); } void @@ -268,7 +268,7 @@ DataStoreBase::dropBuffers() for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) { _states[bufferId].dropBuffer(bufferId, _buffers[bufferId].get_atomic_buffer()); } - _genHolder.clearHoldLists(); + _genHolder.reclaim_all(); } vespalib::MemoryUsage @@ -289,7 +289,7 @@ DataStoreBase::holdBuffer(uint32_t bufferId) _states[bufferId].onHold(bufferId); size_t holdBytes = 0u; // getMemStats() still accounts held buffers auto hold = std::make_unique<BufferHold>(holdBytes, *this, bufferId); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); } void @@ -356,7 +356,7 @@ DataStoreBase::getMemStats() const LOG_ABORT("should not be reached"); } } - size_t genHolderHeldBytes = _genHolder.getHeldBytes(); + size_t genHolderHeldBytes = _genHolder.get_held_bytes(); stats._holdBytes += genHolderHeldBytes; stats._allocBytes += genHolderHeldBytes; stats._usedBytes += genHolderHeldBytes; @@ -428,7 +428,7 @@ DataStoreBase::fallbackResize(uint32_t bufferId, size_t elemsNeeded) state.getTypeHandler(), state.getTypeId()); if (!_initializing) { - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); } } diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp index 2ae22084472..102aa1cefb3 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp @@ -32,7 +32,7 @@ ShardedHashMap::ShardedHashMap(std::unique_ptr<const EntryComparator> comp) ShardedHashMap::~ShardedHashMap() { - _gen_holder.clearHoldLists(); + _gen_holder.reclaim_all(); for (size_t i = 0; i < num_shards; ++i) { auto map = _maps[i].load(std::memory_order_relaxed); delete map; @@ -58,7 +58,7 @@ ShardedHashMap::hold_shard(std::unique_ptr<const FixedSizeHashMap> map) { auto usage = map->get_memory_usage(); auto hold = std::make_unique<ShardedHashMapShardHeld>(usage.allocatedBytes(), std::move(map)); - _gen_holder.hold(std::move(hold)); + _gen_holder.insert(std::move(hold)); } ShardedHashMap::KvType& @@ -115,7 +115,7 @@ ShardedHashMap::transfer_hold_lists(generation_t generation) map->transfer_hold_lists(generation); } } - _gen_holder.transferHoldLists(generation); + _gen_holder.assign_generation(generation); } void @@ -127,7 +127,7 @@ ShardedHashMap::trim_hold_lists(generation_t first_used) map->trim_hold_lists(first_used); } } - _gen_holder.trimHoldLists(first_used); + _gen_holder.reclaim(first_used); } size_t @@ -153,7 +153,7 @@ ShardedHashMap::get_memory_usage() const memory_usage.merge(map->get_memory_usage()); } } - size_t gen_holder_held_bytes = _gen_holder.getHeldBytes(); + size_t gen_holder_held_bytes = _gen_holder.get_held_bytes(); memory_usage.incAllocatedBytes(gen_holder_held_bytes); memory_usage.incAllocatedBytesOnHold(gen_holder_held_bytes); return memory_usage; @@ -222,7 +222,7 @@ ShardedHashMap::foreach_value(std::function<void(const std::vector<EntryRef>&)> bool ShardedHashMap::has_held_buffers() const { - return _gen_holder.getHeldBytes() != 0; + return _gen_holder.get_held_bytes() != 0; } void diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.h b/vespalib/src/vespa/vespalib/util/generation_hold_list.h index 5e12cb72bbd..b2f90934e84 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.h +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.h @@ -14,7 +14,7 @@ namespace vespalib { * * This class must be used in accordance with a GenerationHandler. */ -template <typename T, bool track_bytes_held> +template <typename T, bool track_bytes_held, bool use_deque> class GenerationHoldList { private: using generation_t = vespalib::GenerationHandler::generation_t; @@ -30,7 +30,9 @@ private: }; using ElemList = std::vector<T>; - using ElemWithGenList = std::deque<ElemWithGen>; + using ElemWithGenList = std::conditional_t<use_deque, + std::deque<ElemWithGen>, + std::vector<ElemWithGen>>; ElemList _phase_1_list; ElemWithGenList _phase_2_list; @@ -46,6 +48,7 @@ private: public: GenerationHoldList(); + ~GenerationHoldList(); /** * Insert the given data element on this hold list. diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp index e7eee2b0aef..ed4a99c4753 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + #include "generation_hold_list.h" +#include <cassert> namespace vespalib { -template <typename T, bool track_bytes_held> +template <typename T, bool track_bytes_held, bool use_deque> void -GenerationHoldList<T, track_bytes_held>::assign_generation_internal(generation_t current_gen) +GenerationHoldList<T, track_bytes_held, use_deque>::assign_generation_internal(generation_t current_gen) { for (auto& elem : _phase_1_list) { _phase_2_list.push_back(ElemWithGen(std::move(elem), current_gen)); @@ -14,9 +17,9 @@ GenerationHoldList<T, track_bytes_held>::assign_generation_internal(generation_t _phase_1_list.clear(); } -template <typename T, bool track_bytes_held> +template <typename T, bool track_bytes_held, bool use_deque> void -GenerationHoldList<T, track_bytes_held>::reclaim_internal(generation_t oldest_used_gen) +GenerationHoldList<T, track_bytes_held, use_deque>::reclaim_internal(generation_t oldest_used_gen) { auto itr = _phase_2_list.begin(); auto ite = _phase_2_list.end(); @@ -33,17 +36,25 @@ GenerationHoldList<T, track_bytes_held>::reclaim_internal(generation_t oldest_us } } -template <typename T, bool track_bytes_held> -GenerationHoldList<T, track_bytes_held>::GenerationHoldList() +template <typename T, bool track_bytes_held, bool use_deque> +GenerationHoldList<T, track_bytes_held, use_deque>::GenerationHoldList() : _phase_1_list(), _phase_2_list(), _held_bytes() { } -template <typename T, bool track_bytes_held> +template <typename T, bool track_bytes_held, bool use_deque> +GenerationHoldList<T, track_bytes_held, use_deque>::~GenerationHoldList() +{ + assert(_phase_1_list.empty()); + assert(_phase_2_list.empty()); + assert(get_held_bytes() == 0); +} + +template <typename T, bool track_bytes_held, bool use_deque> void -GenerationHoldList<T, track_bytes_held>::insert(T data) +GenerationHoldList<T, track_bytes_held, use_deque>::insert(T data) { _phase_1_list.push_back(std::move(data)); if (track_bytes_held) { @@ -51,9 +62,9 @@ GenerationHoldList<T, track_bytes_held>::insert(T data) } } -template <typename T, bool track_bytes_held> +template <typename T, bool track_bytes_held, bool use_deque> void -GenerationHoldList<T, track_bytes_held>::reclaim_all() +GenerationHoldList<T, track_bytes_held, use_deque>::reclaim_all() { _phase_1_list.clear(); _phase_2_list.clear(); diff --git a/vespalib/src/vespa/vespalib/util/generationholder.cpp b/vespalib/src/vespa/vespalib/util/generationholder.cpp index f2b10ff3af1..5f31f610c68 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.cpp +++ b/vespalib/src/vespa/vespalib/util/generationholder.cpp @@ -1,66 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "generationholder.h" -#include <cassert> +#include "generation_hold_list.hpp" namespace vespalib { GenerationHeldBase::~GenerationHeldBase() = default; GenerationHolder::GenerationHolder() - : _hold1List(), - _hold2List(), - _heldBytes(0) -{ } - -GenerationHolder::~GenerationHolder() -{ - assert(_hold1List.empty()); - assert(_hold2List.empty()); - assert(getHeldBytes() == 0); -} - -void -GenerationHolder::hold(GenerationHeldBase::UP data) -{ - _hold1List.push_back(std::move(data)); - _heldBytes.store(getHeldBytes() + _hold1List.back()->byte_size(), std::memory_order_relaxed); -} - -void -GenerationHolder::transferHoldListsSlow(generation_t generation) -{ - HoldList::iterator it(_hold1List.begin()); - HoldList::iterator ite(_hold1List.end()); - HoldList &hold2List = _hold2List; - for (; it != ite; ++it) { - assert((*it)->_generation == 0u); - (*it)->_generation = generation; - hold2List.push_back(std::move(*it)); - } - _hold1List.clear(); -} - -void -GenerationHolder::trimHoldListsSlow(generation_t usedGen) -{ - for (;;) { - if (_hold2List.empty()) - break; - GenerationHeldBase &first = *_hold2List.front(); - if (static_cast<sgeneration_t>(first._generation - usedGen) >= 0) - break; - _heldBytes.store(getHeldBytes() - first.byte_size(), std::memory_order_relaxed); - _hold2List.erase(_hold2List.begin()); - } -} - -void -GenerationHolder::clearHoldLists() + : GenerationHoldList<GenerationHeldBase::UP, true, false>() { - _hold1List.clear(); - _hold2List.clear(); - _heldBytes = 0; } } diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index df44f39ebff..ec2cded2e84 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -2,8 +2,8 @@ #pragma once +#include "generation_hold_list.h" #include "generationhandler.h" -#include <vector> #include <memory> namespace vespalib { @@ -11,79 +11,31 @@ namespace vespalib { class GenerationHeldBase { public: - typedef GenerationHandler::generation_t generation_t; - typedef std::unique_ptr<GenerationHeldBase> UP; - typedef std::shared_ptr<GenerationHeldBase> SP; + using generation_t = GenerationHandler::generation_t; + using UP = std::unique_ptr<GenerationHeldBase>; + using SP = std::shared_ptr<GenerationHeldBase>; - generation_t _generation; private: size_t _byte_size; public: GenerationHeldBase(size_t byte_size_in) - : _generation(0u), - _byte_size(byte_size_in) + : _byte_size(byte_size_in) { } virtual ~GenerationHeldBase(); size_t byte_size() const { return _byte_size; } }; +template class GenerationHoldList<GenerationHeldBase::UP, true, false>; + /* * GenerationHolder is meant to hold large elements until readers can * no longer access them. */ -class GenerationHolder -{ -private: - typedef GenerationHandler::generation_t generation_t; - typedef GenerationHandler::sgeneration_t sgeneration_t; - - typedef std::vector<GenerationHeldBase::UP> HoldList; - - HoldList _hold1List; - HoldList _hold2List; - std::atomic<size_t> _heldBytes; - - /** - * Transfer holds from hold1 to hold2 lists, assigning generation. - */ - void transferHoldListsSlow(generation_t generation); - - /** - * Remove all data elements from this holder where generation < usedGen. - **/ - void trimHoldListsSlow(generation_t usedGen); - +class GenerationHolder : public GenerationHoldList<GenerationHeldBase::UP, true, false> { public: GenerationHolder(); - ~GenerationHolder(); - - /** - * Add the given data pointer to this holder. - **/ - void hold(GenerationHeldBase::UP data); - - /** - * Transfer holds from hold1 to hold2 lists, assigning generation. - */ - void transferHoldLists(generation_t generation) { - if (!_hold1List.empty()) { - transferHoldListsSlow(generation); - } - } - - /** - * Remove all data elements from this holder where generation < usedGen. - **/ - void trimHoldLists(generation_t usedGen) { - if (!_hold2List.empty() && static_cast<sgeneration_t>(_hold2List.front()->_generation - usedGen) < 0) { - trimHoldListsSlow(usedGen); - } - } - - void clearHoldLists(); - size_t getHeldBytes() const { return _heldBytes.load(std::memory_order_relaxed); } }; } diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.hpp b/vespalib/src/vespa/vespalib/util/rcuvector.hpp index 97a73a73cc9..e551bb17db0 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.hpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.hpp @@ -80,7 +80,7 @@ RcuVectorBase<T>::replaceVector(ArrayType replacement) { replacement.swap(_data); // atomic switch of underlying data size_t holdSize = replacement.capacity() * sizeof(T); auto hold = std::make_unique<RcuVectorHeld<ArrayType>>(holdSize, std::move(replacement)); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); onReallocation(); } @@ -116,7 +116,7 @@ RcuVectorBase<T>::shrink(size_t newSize) tmpData.swap(_data); // atomic switch of underlying data size_t holdSize = tmpData.capacity() * sizeof(T); auto hold = std::make_unique<RcuVectorHeld<ArrayType>>(holdSize, std::move(tmpData)); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); onReallocation(); } } @@ -162,7 +162,7 @@ template <typename T> void RcuVector<T>::onReallocation() { RcuVectorBase<T>::onReallocation(); - _genHolderStore.transferHoldLists(_generation); + _genHolderStore.assign_generation(_generation); } template <typename T> @@ -182,14 +182,14 @@ RcuVector<T>::RcuVector(GrowStrategy growStrategy) template <typename T> RcuVector<T>::~RcuVector() { - _genHolderStore.clearHoldLists(); + _genHolderStore.reclaim_all(); } template <typename T> void RcuVector<T>::removeOldGenerations(generation_t firstUsed) { - _genHolderStore.trimHoldLists(firstUsed); + _genHolderStore.reclaim(firstUsed); } template <typename T> @@ -197,7 +197,7 @@ MemoryUsage RcuVector<T>::getMemoryUsage() const { MemoryUsage retval(RcuVectorBase<T>::getMemoryUsage()); - retval.mergeGenerationHeldBytes(_genHolderStore.getHeldBytes()); + retval.mergeGenerationHeldBytes(_genHolderStore.get_held_bytes()); return retval; } |