From ab10d27bd029f73d82dabd1f3fdb839edae29e61 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Mon, 10 Oct 2022 15:20:17 +0000 Subject: Implement GenerationHolder in terms of the generic generation hold list. --- .../lid_allocator/lid_allocator_test.cpp | 2 +- .../lid_state_vector/lid_state_vector_test.cpp | 2 +- .../proton/documentmetastore/documentmetastore.cpp | 10 ++-- .../src/tests/common/bitvector/bitvector_test.cpp | 8 +-- .../src/tests/predicate/simple_index_test.cpp | 2 +- .../vespa/searchlib/attribute/flagattribute.cpp | 4 +- .../src/vespa/searchlib/attribute/load_utils.hpp | 2 +- .../searchlib/attribute/predicate_attribute.cpp | 8 +-- .../searchlib/attribute/reference_attribute.cpp | 6 +- .../searchlib/attribute/singleboolattribute.cpp | 10 ++-- .../searchlib/attribute/singleenumattribute.hpp | 10 ++-- .../searchlib/attribute/singlenumericattribute.hpp | 10 ++-- .../attribute/singlesmallnumericattribute.cpp | 10 ++-- .../vespa/searchlib/common/growablebitvector.cpp | 2 +- .../searchlib/tensor/dense_tensor_attribute.cpp | 2 +- .../searchlib/tensor/direct_tensor_attribute.cpp | 2 +- .../tensor/serialized_fast_value_attribute.cpp | 2 +- .../vespa/searchlib/tensor/tensor_attribute.cpp | 8 +-- vespalib/CMakeLists.txt | 1 - .../fixed_size_hash_map_test.cpp | 4 +- .../generation_hold_list_test.cpp | 3 +- .../tests/util/generation_holder/CMakeLists.txt | 9 --- .../generation_holder/generation_holder_test.cpp | 38 ------------- .../src/tests/util/rcuvector/rcuvector_test.cpp | 44 +++++++-------- .../src/vespa/vespalib/datastore/datastorebase.cpp | 14 ++--- .../vespa/vespalib/datastore/sharded_hash_map.cpp | 12 ++-- .../src/vespa/vespalib/util/generation_hold_list.h | 7 ++- .../vespa/vespalib/util/generation_hold_list.hpp | 31 +++++++---- .../src/vespa/vespalib/util/generationholder.cpp | 55 +------------------ .../src/vespa/vespalib/util/generationholder.h | 64 +++------------------- vespalib/src/vespa/vespalib/util/rcuvector.hpp | 12 ++-- 31 files changed, 131 insertions(+), 263 deletions(-) delete mode 100644 vespalib/src/tests/util/generation_holder/CMakeLists.txt delete mode 100644 vespalib/src/tests/util/generation_holder/generation_holder_test.cpp diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp index 2d675e82db2..157d10e6652 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp @@ -28,7 +28,7 @@ protected: ~LidAllocatorTest() { - _gen_hold.clearHoldLists(); + _gen_hold.reclaim_all(); } uint32_t get_size() { return _allocator.getActiveLids().size(); } diff --git a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp index ab45cca0971..68958bbe3e4 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp @@ -22,7 +22,7 @@ protected: ~LidStateVectorTest() { - _gen_hold.clearHoldLists(); + _gen_hold.reclaim_all(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index b1b5f45a8fa..1853e939d42 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -224,7 +224,7 @@ DocumentMetaStore::onUpdateStat() { auto &compaction_strategy = getConfig().getCompactionStrategy(); vespalib::MemoryUsage usage = _metaDataStore.getMemoryUsage(); - usage.incAllocatedBytesOnHold(getGenerationHolder().getHeldBytes()); + usage.incAllocatedBytesOnHold(getGenerationHolder().get_held_bytes()); size_t bvSize = _lidAlloc.getUsedLidsSize(); usage.incAllocatedBytes(bvSize); usage.incUsedBytes(bvSize); @@ -245,7 +245,7 @@ DocumentMetaStore::onGenerationChange(generation_t generation) { _gidToLidMap.getAllocator().freeze(); _gidToLidMap.getAllocator().transferHoldLists(generation - 1); - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); updateStat(false); } @@ -254,7 +254,7 @@ DocumentMetaStore::removeOldGenerations(generation_t firstUsed) { _gidToLidMap.getAllocator().trimHoldLists(firstUsed); _lidAlloc.trimHoldLists(firstUsed); - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } std::unique_ptr @@ -442,7 +442,7 @@ DocumentMetaStore::~DocumentMetaStore() // TODO: Properly notify about modified buckets when using shared bucket db // between document types unload(); - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); assert(get_shrink_lid_space_blockers() == 0); } @@ -1009,7 +1009,7 @@ DocumentMetaStore::holdUnblockShrinkLidSpace() { assert(get_shrink_lid_space_blockers() > 0); auto hold = std::make_unique(*this); - getGenerationHolder().hold(std::move(hold)); + getGenerationHolder().insert(std::move(hold)); incGeneration(); } diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 79af28d20be..7a26202682b 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -654,8 +654,8 @@ TEST("requireThatGrowWorks") EXPECT_EQUAL(4095u, v.writer().capacity()); EXPECT_EQUAL(3u, v.writer().countTrueBits()); - g.transferHoldLists(1); - g.trimHoldLists(2); + g.assign_generation(1); + g.reclaim(2); } TEST("require that growable bit vectors keeps memory allocator") @@ -676,8 +676,8 @@ TEST("require that growable bit vectors keeps memory allocator") EXPECT_EQUAL(AllocStats(4, 1), stats); v.writer().resize(1); // DO NOT TRY THIS AT HOME EXPECT_EQUAL(AllocStats(5, 2), stats); - g.transferHoldLists(1); - g.trimHoldLists(2); + g.assign_generation(1); + g.reclaim(2); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index dfa8c12deec..7bf52680782 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -74,7 +74,7 @@ struct Fixture { Fixture() : _generation_holder(), _limit_provider(), _index(_generation_holder, _limit_provider, config) {} ~Fixture() { - _generation_holder.clearHoldLists(); + _generation_holder.reclaim_all(); } SimpleIndex &index() { return _index; diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index ef796d3f3d2..df75b0ab4e5 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -223,7 +223,7 @@ FlagAttributeT::resizeBitVectors(uint32_t neededSize) } } _bitVectorSize = newSize; - _bitVectorHolder.transferHoldLists(this->getCurrentGeneration()); + _bitVectorHolder.assign_generation(this->getCurrentGeneration()); } @@ -232,7 +232,7 @@ void FlagAttributeT::removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) { B::removeOldGenerations(firstUsed); - _bitVectorHolder.trimHoldLists(firstUsed); + _bitVectorHolder.reclaim(firstUsed); } template class FlagAttributeT; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 62d645326ce..463a62ab01a 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -68,7 +68,7 @@ loadFromEnumeratedSingleValue(Vector &vector, using ValueType = typename Vector::ValueType; using NonAtomicValueType = atomic_utils::NonAtomicValue_t; uint32_t numDocs = attrReader.getEnumCount(); - genHolder.clearHoldLists(); + genHolder.reclaim_all(); vector.reset(); vector.unsafe_reserve(numDocs); for (uint32_t doc = 0; doc < numDocs; ++doc) { diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index c1897c71366..f2e07bba853 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -89,7 +89,7 @@ PredicateAttribute::PredicateAttribute(const vespalib::string &base_file_name, c PredicateAttribute::~PredicateAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); } void PredicateAttribute::populateIfNeeded() { @@ -118,7 +118,7 @@ PredicateAttribute::onUpdateStat() combined.merge(_min_feature.getMemoryUsage()); combined.merge(_interval_range_vector.getMemoryUsage()); combined.merge(_index->getMemoryUsage()); - combined.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + combined.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); this->updateStatistics(_min_feature.size(), _min_feature.size(), combined.allocatedBytes(), combined.usedBytes(), combined.deadBytes(), combined.allocatedBytesOnHold()); @@ -127,14 +127,14 @@ PredicateAttribute::onUpdateStat() void PredicateAttribute::removeOldGenerations(generation_t firstUsed) { - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); _index->trimHoldLists(firstUsed); } void PredicateAttribute::onGenerationChange(generation_t generation) { - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); _index->transferHoldLists(generation - 1); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 0ebef4af8b0..36fb02f4c4b 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -165,7 +165,7 @@ ReferenceAttribute::removeOldGenerations(generation_t firstUsed) { _referenceMappings.trimHoldLists(firstUsed); _store.trimHoldLists(firstUsed); - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } void @@ -175,7 +175,7 @@ ReferenceAttribute::onGenerationChange(generation_t generation) _store.freeze(); _referenceMappings.transferHoldLists(generation - 1); _store.transferHoldLists(generation - 1); - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); } void @@ -203,7 +203,7 @@ ReferenceAttribute::onUpdateStat() _compaction_spec = ReferenceAttributeCompactionSpec(compaction_strategy.should_compact_memory(total), compaction_strategy.should_compact_memory(dictionary_memory_usage)); total.merge(dictionary_memory_usage); - total.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + total.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); total.merge(_indices.getMemoryUsage()); total.merge(_referenceMappings.getMemoryUsage()); updateStatistics(getTotalValueCount(), getUniqueValueCount(), diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 7574508517d..6e07a9e658e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -29,7 +29,7 @@ SingleBoolAttribute(const vespalib::string &baseFileName, const GrowStrategy & g SingleBoolAttribute::~SingleBoolAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); } void @@ -95,7 +95,7 @@ SingleBoolAttribute::onUpdateStat() { vespalib::MemoryUsage usage; usage.setAllocatedBytes(_bv.writer().extraByteSize()); usage.setUsedBytes(_bv.writer().sizeBytes()); - usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + usage.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); usage.merge(this->getChangeVectorMemoryUsage()); this->updateStatistics(_bv.writer().size(), _bv.writer().size(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); @@ -191,7 +191,7 @@ SingleBoolAttribute::onLoad(vespalib::Executor *) bool ok(attrReader.hasData()); if (ok) { setCreateSerialNum(attrReader.getCreateSerialNum()); - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _bv.writer().clear(); uint32_t numDocs = attrReader.getNextData(); _bv.extend(numDocs); @@ -258,12 +258,12 @@ SingleBoolAttribute::getEstimatedSaveByteSize() const void SingleBoolAttribute::removeOldGenerations(generation_t firstUsed) { - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } void SingleBoolAttribute::onGenerationChange(generation_t generation) { - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index c4abcfbc25a..11742bf8f48 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -25,7 +25,7 @@ SingleValueEnumAttribute(const vespalib::string &baseFileName, template SingleValueEnumAttribute::~SingleValueEnumAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); } template @@ -128,7 +128,7 @@ SingleValueEnumAttribute::onUpdateStat() // update statistics vespalib::MemoryUsage total = _enumIndices.getMemoryUsage(); auto& compaction_strategy = this->getConfig().getCompactionStrategy(); - total.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + total.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); total.merge(this->_enumStore.update_stat(compaction_strategy)); total.merge(this->getChangeVectorMemoryUsage()); mergeMemoryStats(total); @@ -218,7 +218,7 @@ SingleValueEnumAttribute::fillValues(LoadedVector & loaded) { if constexpr (!std::is_same_v) { uint32_t numDocs = this->getNumDocs(); - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _enumIndices.reset(); _enumIndices.unsafe_reserve(numDocs); for (DocId doc = 0; doc < numDocs; ++doc, loaded.next()) { @@ -267,7 +267,7 @@ void SingleValueEnumAttribute::removeOldGenerations(generation_t firstUsed) { this->_enumStore.trim_hold_lists(firstUsed); - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } template @@ -281,7 +281,7 @@ SingleValueEnumAttribute::onGenerationChange(generation_t generation) * sufficiently new frozen tree. */ freezeEnumDictionary(); - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); this->_enumStore.transfer_hold_lists(generation - 1); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index b9c1c3686de..bbacc10e79c 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -32,7 +32,7 @@ SingleValueNumericAttribute(const vespalib::string & baseFileName, const Attribu template SingleValueNumericAttribute::~SingleValueNumericAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); } template @@ -65,7 +65,7 @@ void SingleValueNumericAttribute::onUpdateStat() { vespalib::MemoryUsage usage = _data.getMemoryUsage(); - usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + usage.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); usage.merge(this->getChangeVectorMemoryUsage()); this->updateStatistics(_data.size(), _data.size(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); @@ -97,14 +97,14 @@ template void SingleValueNumericAttribute::removeOldGenerations(generation_t firstUsed) { - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } template void SingleValueNumericAttribute::onGenerationChange(generation_t generation) { - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); } template @@ -143,7 +143,7 @@ SingleValueNumericAttribute::onLoad(vespalib::Executor *) return onLoadEnumerated(attrReader); const size_t sz(attrReader.getDataCount()); - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _data.reset(); _data.unsafe_reserve(sz); for (uint32_t i = 0; i < sz; ++i) { diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index b69ed017b52..02ab5798f9f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -36,7 +36,7 @@ SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, SingleValueSmallNumericAttribute::~SingleValueSmallNumericAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); } void @@ -97,7 +97,7 @@ void SingleValueSmallNumericAttribute::onUpdateStat() { vespalib::MemoryUsage usage = _wordData.getMemoryUsage(); - usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + usage.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); uint32_t numDocs = B::getNumDocs(); updateStatistics(numDocs, numDocs, usage.allocatedBytes(), usage.usedBytes(), @@ -108,14 +108,14 @@ SingleValueSmallNumericAttribute::onUpdateStat() void SingleValueSmallNumericAttribute::removeOldGenerations(generation_t firstUsed) { - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } void SingleValueSmallNumericAttribute::onGenerationChange(generation_t generation) { - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); } @@ -127,7 +127,7 @@ SingleValueSmallNumericAttribute::onLoad(vespalib::Executor *) if (ok) { setCreateSerialNum(attrReader.getCreateSerialNum()); const size_t sz(attrReader.getDataCount()); - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _wordData.reset(); _wordData.unsafe_reserve(sz - 1); Word numDocs = attrReader.getNextData(); diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index e3334be3fd9..5f971e21cd3 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -72,7 +72,7 @@ bool GrowableBitVector::hold(GenerationHeldBase::UP v) { if (v) { - _generationHolder.hold(std::move(v)); + _generationHolder.insert(std::move(v)); return true; } return false; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 636c949be08..0c797fa7fe1 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -172,7 +172,7 @@ DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, con DenseTensorAttribute::~DenseTensorAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _tensorStore.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index d9fe025b4e5..a68f7fcc3da 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -15,7 +15,7 @@ DirectTensorAttribute::DirectTensorAttribute(stringref name, const Config &cfg) DirectTensorAttribute::~DirectTensorAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _tensorStore.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 52ccb4c91b1..d2153ac93ad 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -23,7 +23,7 @@ SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const SerializedFastValueAttribute::~SerializedFastValueAttribute() { - getGenerationHolder().clearHoldLists(); + getGenerationHolder().reclaim_all(); _tensorStore.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 99a30b59bd1..b233960e339 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -113,13 +113,13 @@ void TensorAttribute::removeOldGenerations(generation_t firstUsed) { _tensorStore.trimHoldLists(firstUsed); - getGenerationHolder().trimHoldLists(firstUsed); + getGenerationHolder().reclaim(firstUsed); } void TensorAttribute::onGenerationChange(generation_t generation) { - getGenerationHolder().transferHoldLists(generation - 1); + getGenerationHolder().assign_generation(generation - 1); _tensorStore.transferHoldLists(generation - 1); } @@ -169,7 +169,7 @@ TensorAttribute::update_stat() { vespalib::MemoryUsage result = _refVector.getMemoryUsage(); result.merge(_tensorStore.update_stat(getConfig().getCompactionStrategy())); - result.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + result.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); return result; } @@ -178,7 +178,7 @@ TensorAttribute::memory_usage() const { vespalib::MemoryUsage result = _refVector.getMemoryUsage(); result.merge(_tensorStore.getMemoryUsage()); - result.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + result.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes()); return result; } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 532ec870ce9..8edd7fb2c5d 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -182,7 +182,6 @@ vespa_define_module( src/tests/util/cgroup_resource_limits src/tests/util/file_area_freelist src/tests/util/generation_hold_list - src/tests/util/generation_holder src/tests/util/generationhandler src/tests/util/generationhandler_stress src/tests/util/hamming diff --git a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp index 599cb209e6c..ad10bc5c7e6 100644 --- a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp @@ -90,11 +90,11 @@ DataStoreFixedSizeHashTest::commit() { _store.transferHoldLists(_generation_handler.getCurrentGeneration()); _hash_map->transfer_hold_lists(_generation_handler.getCurrentGeneration()); - _generation_holder.transferHoldLists(_generation_handler.getCurrentGeneration()); + _generation_holder.assign_generation(_generation_handler.getCurrentGeneration()); _generation_handler.incGeneration(); _store.trimHoldLists(_generation_handler.getFirstUsedGeneration()); _hash_map->trim_hold_lists(_generation_handler.getFirstUsedGeneration()); - _generation_holder.trimHoldLists(_generation_handler.getFirstUsedGeneration()); + _generation_holder.reclaim(_generation_handler.getFirstUsedGeneration()); } size_t diff --git a/vespalib/src/tests/util/generation_hold_list/generation_hold_list_test.cpp b/vespalib/src/tests/util/generation_hold_list/generation_hold_list_test.cpp index 0490a99f1e0..8305b711d5f 100644 --- a/vespalib/src/tests/util/generation_hold_list/generation_hold_list_test.cpp +++ b/vespalib/src/tests/util/generation_hold_list/generation_hold_list_test.cpp @@ -3,12 +3,13 @@ #include #include #include +#include using vespalib::GenerationHeldBase; using vespalib::GenerationHoldList; using MyElem = GenerationHeldBase; -using MyHoldList = GenerationHoldList; +using MyHoldList = GenerationHoldList; TEST(GenerationHoldListTest, holding_of_unique_ptr_elements_with_tracking_of_held_bytes) { diff --git a/vespalib/src/tests/util/generation_holder/CMakeLists.txt b/vespalib/src/tests/util/generation_holder/CMakeLists.txt deleted file mode 100644 index 8acf9fadaff..00000000000 --- a/vespalib/src/tests/util/generation_holder/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_generation_holder_test_app TEST - SOURCES - generation_holder_test.cpp - DEPENDS - vespalib - GTest::GTest -) -vespa_add_test(NAME vespalib_generation_holder_test_app COMMAND vespalib_generation_holder_test_app) diff --git a/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp b/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp deleted file mode 100644 index 97c3330ac9e..00000000000 --- a/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -using vespalib::GenerationHolder; -using MyHeld = vespalib::GenerationHeldBase; - -TEST(GenerationHolderTest, basic_tracking) -{ - GenerationHolder gh; - gh.hold(std::make_unique(sizeof(int32_t))); - gh.transferHoldLists(0); - gh.hold(std::make_unique(sizeof(int32_t))); - gh.transferHoldLists(1); - gh.hold(std::make_unique(sizeof(int32_t))); - gh.transferHoldLists(2); - gh.hold(std::make_unique(sizeof(int32_t))); - gh.transferHoldLists(4); - EXPECT_EQ(4u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(0); - EXPECT_EQ(4u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(1); - EXPECT_EQ(3u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(2); - EXPECT_EQ(2u * sizeof(int32_t), gh.getHeldBytes()); - gh.hold(std::make_unique(sizeof(int32_t))); - gh.transferHoldLists(6); - EXPECT_EQ(3u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(6); - EXPECT_EQ(1u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(7); - EXPECT_EQ(0u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(7); - EXPECT_EQ(0u * sizeof(int32_t), gh.getHeldBytes()); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index eb2b00f9e20..c23065b7468 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -102,15 +102,15 @@ TEST(RcuVectorTest, resize) RcuVectorBase v(growStrategy(16, 1.0, 0), g); v.push_back(1); v.push_back(2); - g.transferHoldLists(0); - g.trimHoldLists(1); + g.assign_generation(0); + g.reclaim(1); const int8_t *old = &v[0]; EXPECT_EQ(16u, v.capacity()); EXPECT_EQ(2u, v.size()); v.ensure_size(32, 3); v[0] = 3; v[1] = 3; - g.transferHoldLists(1); + g.assign_generation(1); EXPECT_EQ(1, old[0]); EXPECT_EQ(2, old[1]); EXPECT_EQ(3, v[0]); @@ -119,7 +119,7 @@ TEST(RcuVectorTest, resize) EXPECT_EQ(3, v[31]); EXPECT_EQ(64u, v.capacity()); EXPECT_EQ(32u, v.size()); - g.trimHoldLists(2); + g.reclaim(2); } } @@ -197,11 +197,11 @@ void verify_shrink_with_buffer_copying(size_t initial_size, size_t absolute_mini v.push_back(2); v.push_back(3); v.push_back(4); - g.transferHoldLists(0); - g.trimHoldLists(1); + g.assign_generation(0); + g.reclaim(1); MemoryUsage mu; mu = v.getMemoryUsage(); - mu.incAllocatedBytesOnHold(g.getHeldBytes()); + mu.incAllocatedBytesOnHold(g.get_held_bytes()); EXPECT_TRUE(assertUsage(MemoryUsage(initial_capacity, 4, 0, 0), mu)); EXPECT_EQ(4u, v.size()); EXPECT_EQ(initial_capacity, v.capacity()); @@ -211,18 +211,18 @@ void verify_shrink_with_buffer_copying(size_t initial_size, size_t absolute_mini EXPECT_EQ(4, v[3]); const int8_t *old = &v[0]; v.shrink(2); - g.transferHoldLists(1); + g.assign_generation(1); EXPECT_EQ(2u, v.size()); EXPECT_EQ(minimal_capacity, v.capacity()); EXPECT_EQ(1, v[0]); EXPECT_EQ(2, v[1]); EXPECT_EQ(1, old[0]); EXPECT_EQ(2, old[1]); - g.trimHoldLists(2); + g.reclaim(2); EXPECT_EQ(1, v[0]); EXPECT_EQ(2, v[1]); mu = v.getMemoryUsage(); - mu.incAllocatedBytesOnHold(g.getHeldBytes()); + mu.incAllocatedBytesOnHold(g.get_held_bytes()); EXPECT_TRUE(assertUsage(MemoryUsage(minimal_capacity, 2, 0, 0), mu)); } @@ -256,7 +256,7 @@ struct ShrinkFixture { EXPECT_EQ(oldPtr, &vec[0]); } void assertEmptyHoldList() { - EXPECT_EQ(0u, g.getHeldBytes()); + EXPECT_EQ(0u, g.get_held_bytes()); } static size_t page_ints() { return round_up_to_page_size(1) / sizeof(int); } }; @@ -294,8 +294,8 @@ TEST(RcuVectorTest, small_expand) v.push_back(2); EXPECT_EQ(2u, v.capacity()); EXPECT_EQ(2u, v.size()); - g.transferHoldLists(1); - g.trimHoldLists(2); + g.assign_generation(1); + g.reclaim(2); } struct FixtureBase { @@ -325,10 +325,10 @@ struct Fixture : public FixtureBase { Fixture(); ~Fixture(); - void transfer_and_trim(generation_t transfer_gen, generation_t trim_gen) + void assign_and_reclaim(generation_t assign_gen, generation_t reclaim_gen) { - g.transferHoldLists(transfer_gen); - g.trimHoldLists(trim_gen); + g.assign_generation(assign_gen); + g.reclaim(reclaim_gen); } }; @@ -345,7 +345,7 @@ TEST(RcuVectorTest, memory_allocator_can_be_set) { Fixture f; EXPECT_EQ(AllocStats(2, 0), f.stats); - f.transfer_and_trim(1, 2); + f.assign_and_reclaim(1, 2); EXPECT_EQ(AllocStats(2, 1), f.stats); } @@ -355,7 +355,7 @@ TEST(RcuVectorTest, memory_allocator_is_preserved_across_reset) f.arr.reset(); f.arr.reserve(100); EXPECT_EQ(AllocStats(4, 1), f.stats); - f.transfer_and_trim(1, 2); + f.assign_and_reclaim(1, 2); EXPECT_EQ(AllocStats(4, 3), f.stats); } @@ -366,7 +366,7 @@ TEST(RcuVectorTest, created_replacement_vector_uses_same_memory_allocator) EXPECT_EQ(AllocStats(2, 0), f.stats); arr2.reserve(100); EXPECT_EQ(AllocStats(3, 0), f.stats); - f.transfer_and_trim(1, 2); + f.assign_and_reclaim(1, 2); EXPECT_EQ(AllocStats(3, 1), f.stats); } @@ -377,7 +377,7 @@ TEST(RcuVectorTest, ensure_size_and_shrink_use_same_memory_allocator) EXPECT_EQ(AllocStats(3, 0), f.stats); f.arr.shrink(1000); EXPECT_EQ(AllocStats(4, 0), f.stats); - f.transfer_and_trim(1, 2); + f.assign_and_reclaim(1, 2); EXPECT_EQ(AllocStats(4, 3), f.stats); } @@ -432,10 +432,10 @@ void StressFixture::commit() { auto current_gen = generation_handler.getCurrentGeneration(); - g.transferHoldLists(current_gen); + g.assign_generation(current_gen); generation_handler.incGeneration(); auto first_used_gen = generation_handler.getFirstUsedGeneration(); - g.trimHoldLists(first_used_gen); + g.reclaim(first_used_gen); } void diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index dd6c767e9c6..5962e6cf2f7 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -232,7 +232,7 @@ DataStoreBase::transferElemHoldList(generation_t generation) void DataStoreBase::transferHoldLists(generation_t generation) { - _genHolder.transferHoldLists(generation); + _genHolder.assign_generation(generation); if (hasElemHold1()) { transferElemHoldList(generation); } @@ -250,7 +250,7 @@ void DataStoreBase::trimHoldLists(generation_t usedGen) { trimElemHoldList(usedGen); // Trim entries before trimming buffers - _genHolder.trimHoldLists(usedGen); + _genHolder.reclaim(usedGen); } void @@ -258,7 +258,7 @@ DataStoreBase::clearHoldLists() { transferElemHoldList(0); clearElemHoldList(); - _genHolder.clearHoldLists(); + _genHolder.reclaim_all(); } void @@ -268,7 +268,7 @@ DataStoreBase::dropBuffers() for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) { _states[bufferId].dropBuffer(bufferId, _buffers[bufferId].get_atomic_buffer()); } - _genHolder.clearHoldLists(); + _genHolder.reclaim_all(); } vespalib::MemoryUsage @@ -289,7 +289,7 @@ DataStoreBase::holdBuffer(uint32_t bufferId) _states[bufferId].onHold(bufferId); size_t holdBytes = 0u; // getMemStats() still accounts held buffers auto hold = std::make_unique(holdBytes, *this, bufferId); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); } void @@ -356,7 +356,7 @@ DataStoreBase::getMemStats() const LOG_ABORT("should not be reached"); } } - size_t genHolderHeldBytes = _genHolder.getHeldBytes(); + size_t genHolderHeldBytes = _genHolder.get_held_bytes(); stats._holdBytes += genHolderHeldBytes; stats._allocBytes += genHolderHeldBytes; stats._usedBytes += genHolderHeldBytes; @@ -428,7 +428,7 @@ DataStoreBase::fallbackResize(uint32_t bufferId, size_t elemsNeeded) state.getTypeHandler(), state.getTypeId()); if (!_initializing) { - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); } } diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp index 2ae22084472..102aa1cefb3 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp @@ -32,7 +32,7 @@ ShardedHashMap::ShardedHashMap(std::unique_ptr comp) ShardedHashMap::~ShardedHashMap() { - _gen_holder.clearHoldLists(); + _gen_holder.reclaim_all(); for (size_t i = 0; i < num_shards; ++i) { auto map = _maps[i].load(std::memory_order_relaxed); delete map; @@ -58,7 +58,7 @@ ShardedHashMap::hold_shard(std::unique_ptr map) { auto usage = map->get_memory_usage(); auto hold = std::make_unique(usage.allocatedBytes(), std::move(map)); - _gen_holder.hold(std::move(hold)); + _gen_holder.insert(std::move(hold)); } ShardedHashMap::KvType& @@ -115,7 +115,7 @@ ShardedHashMap::transfer_hold_lists(generation_t generation) map->transfer_hold_lists(generation); } } - _gen_holder.transferHoldLists(generation); + _gen_holder.assign_generation(generation); } void @@ -127,7 +127,7 @@ ShardedHashMap::trim_hold_lists(generation_t first_used) map->trim_hold_lists(first_used); } } - _gen_holder.trimHoldLists(first_used); + _gen_holder.reclaim(first_used); } size_t @@ -153,7 +153,7 @@ ShardedHashMap::get_memory_usage() const memory_usage.merge(map->get_memory_usage()); } } - size_t gen_holder_held_bytes = _gen_holder.getHeldBytes(); + size_t gen_holder_held_bytes = _gen_holder.get_held_bytes(); memory_usage.incAllocatedBytes(gen_holder_held_bytes); memory_usage.incAllocatedBytesOnHold(gen_holder_held_bytes); return memory_usage; @@ -222,7 +222,7 @@ ShardedHashMap::foreach_value(std::function&)> bool ShardedHashMap::has_held_buffers() const { - return _gen_holder.getHeldBytes() != 0; + return _gen_holder.get_held_bytes() != 0; } void diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.h b/vespalib/src/vespa/vespalib/util/generation_hold_list.h index 5e12cb72bbd..b2f90934e84 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.h +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.h @@ -14,7 +14,7 @@ namespace vespalib { * * This class must be used in accordance with a GenerationHandler. */ -template +template class GenerationHoldList { private: using generation_t = vespalib::GenerationHandler::generation_t; @@ -30,7 +30,9 @@ private: }; using ElemList = std::vector; - using ElemWithGenList = std::deque; + using ElemWithGenList = std::conditional_t, + std::vector>; ElemList _phase_1_list; ElemWithGenList _phase_2_list; @@ -46,6 +48,7 @@ private: public: GenerationHoldList(); + ~GenerationHoldList(); /** * Insert the given data element on this hold list. diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp index e7eee2b0aef..ed4a99c4753 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + #include "generation_hold_list.h" +#include namespace vespalib { -template +template void -GenerationHoldList::assign_generation_internal(generation_t current_gen) +GenerationHoldList::assign_generation_internal(generation_t current_gen) { for (auto& elem : _phase_1_list) { _phase_2_list.push_back(ElemWithGen(std::move(elem), current_gen)); @@ -14,9 +17,9 @@ GenerationHoldList::assign_generation_internal(generation_t _phase_1_list.clear(); } -template +template void -GenerationHoldList::reclaim_internal(generation_t oldest_used_gen) +GenerationHoldList::reclaim_internal(generation_t oldest_used_gen) { auto itr = _phase_2_list.begin(); auto ite = _phase_2_list.end(); @@ -33,17 +36,25 @@ GenerationHoldList::reclaim_internal(generation_t oldest_us } } -template -GenerationHoldList::GenerationHoldList() +template +GenerationHoldList::GenerationHoldList() : _phase_1_list(), _phase_2_list(), _held_bytes() { } -template +template +GenerationHoldList::~GenerationHoldList() +{ + assert(_phase_1_list.empty()); + assert(_phase_2_list.empty()); + assert(get_held_bytes() == 0); +} + +template void -GenerationHoldList::insert(T data) +GenerationHoldList::insert(T data) { _phase_1_list.push_back(std::move(data)); if (track_bytes_held) { @@ -51,9 +62,9 @@ GenerationHoldList::insert(T data) } } -template +template void -GenerationHoldList::reclaim_all() +GenerationHoldList::reclaim_all() { _phase_1_list.clear(); _phase_2_list.clear(); diff --git a/vespalib/src/vespa/vespalib/util/generationholder.cpp b/vespalib/src/vespa/vespalib/util/generationholder.cpp index f2b10ff3af1..5f31f610c68 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.cpp +++ b/vespalib/src/vespa/vespalib/util/generationholder.cpp @@ -1,66 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "generationholder.h" -#include +#include "generation_hold_list.hpp" namespace vespalib { GenerationHeldBase::~GenerationHeldBase() = default; GenerationHolder::GenerationHolder() - : _hold1List(), - _hold2List(), - _heldBytes(0) -{ } - -GenerationHolder::~GenerationHolder() -{ - assert(_hold1List.empty()); - assert(_hold2List.empty()); - assert(getHeldBytes() == 0); -} - -void -GenerationHolder::hold(GenerationHeldBase::UP data) -{ - _hold1List.push_back(std::move(data)); - _heldBytes.store(getHeldBytes() + _hold1List.back()->byte_size(), std::memory_order_relaxed); -} - -void -GenerationHolder::transferHoldListsSlow(generation_t generation) -{ - HoldList::iterator it(_hold1List.begin()); - HoldList::iterator ite(_hold1List.end()); - HoldList &hold2List = _hold2List; - for (; it != ite; ++it) { - assert((*it)->_generation == 0u); - (*it)->_generation = generation; - hold2List.push_back(std::move(*it)); - } - _hold1List.clear(); -} - -void -GenerationHolder::trimHoldListsSlow(generation_t usedGen) -{ - for (;;) { - if (_hold2List.empty()) - break; - GenerationHeldBase &first = *_hold2List.front(); - if (static_cast(first._generation - usedGen) >= 0) - break; - _heldBytes.store(getHeldBytes() - first.byte_size(), std::memory_order_relaxed); - _hold2List.erase(_hold2List.begin()); - } -} - -void -GenerationHolder::clearHoldLists() + : GenerationHoldList() { - _hold1List.clear(); - _hold2List.clear(); - _heldBytes = 0; } } diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index df44f39ebff..ec2cded2e84 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -2,8 +2,8 @@ #pragma once +#include "generation_hold_list.h" #include "generationhandler.h" -#include #include namespace vespalib { @@ -11,79 +11,31 @@ namespace vespalib { class GenerationHeldBase { public: - typedef GenerationHandler::generation_t generation_t; - typedef std::unique_ptr UP; - typedef std::shared_ptr SP; + using generation_t = GenerationHandler::generation_t; + using UP = std::unique_ptr; + using SP = std::shared_ptr; - generation_t _generation; private: size_t _byte_size; public: GenerationHeldBase(size_t byte_size_in) - : _generation(0u), - _byte_size(byte_size_in) + : _byte_size(byte_size_in) { } virtual ~GenerationHeldBase(); size_t byte_size() const { return _byte_size; } }; +template class GenerationHoldList; + /* * GenerationHolder is meant to hold large elements until readers can * no longer access them. */ -class GenerationHolder -{ -private: - typedef GenerationHandler::generation_t generation_t; - typedef GenerationHandler::sgeneration_t sgeneration_t; - - typedef std::vector HoldList; - - HoldList _hold1List; - HoldList _hold2List; - std::atomic _heldBytes; - - /** - * Transfer holds from hold1 to hold2 lists, assigning generation. - */ - void transferHoldListsSlow(generation_t generation); - - /** - * Remove all data elements from this holder where generation < usedGen. - **/ - void trimHoldListsSlow(generation_t usedGen); - +class GenerationHolder : public GenerationHoldList { public: GenerationHolder(); - ~GenerationHolder(); - - /** - * Add the given data pointer to this holder. - **/ - void hold(GenerationHeldBase::UP data); - - /** - * Transfer holds from hold1 to hold2 lists, assigning generation. - */ - void transferHoldLists(generation_t generation) { - if (!_hold1List.empty()) { - transferHoldListsSlow(generation); - } - } - - /** - * Remove all data elements from this holder where generation < usedGen. - **/ - void trimHoldLists(generation_t usedGen) { - if (!_hold2List.empty() && static_cast(_hold2List.front()->_generation - usedGen) < 0) { - trimHoldListsSlow(usedGen); - } - } - - void clearHoldLists(); - size_t getHeldBytes() const { return _heldBytes.load(std::memory_order_relaxed); } }; } diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.hpp b/vespalib/src/vespa/vespalib/util/rcuvector.hpp index 97a73a73cc9..e551bb17db0 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.hpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.hpp @@ -80,7 +80,7 @@ RcuVectorBase::replaceVector(ArrayType replacement) { replacement.swap(_data); // atomic switch of underlying data size_t holdSize = replacement.capacity() * sizeof(T); auto hold = std::make_unique>(holdSize, std::move(replacement)); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); onReallocation(); } @@ -116,7 +116,7 @@ RcuVectorBase::shrink(size_t newSize) tmpData.swap(_data); // atomic switch of underlying data size_t holdSize = tmpData.capacity() * sizeof(T); auto hold = std::make_unique>(holdSize, std::move(tmpData)); - _genHolder.hold(std::move(hold)); + _genHolder.insert(std::move(hold)); onReallocation(); } } @@ -162,7 +162,7 @@ template void RcuVector::onReallocation() { RcuVectorBase::onReallocation(); - _genHolderStore.transferHoldLists(_generation); + _genHolderStore.assign_generation(_generation); } template @@ -182,14 +182,14 @@ RcuVector::RcuVector(GrowStrategy growStrategy) template RcuVector::~RcuVector() { - _genHolderStore.clearHoldLists(); + _genHolderStore.reclaim_all(); } template void RcuVector::removeOldGenerations(generation_t firstUsed) { - _genHolderStore.trimHoldLists(firstUsed); + _genHolderStore.reclaim(firstUsed); } template @@ -197,7 +197,7 @@ MemoryUsage RcuVector::getMemoryUsage() const { MemoryUsage retval(RcuVectorBase::getMemoryUsage()); - retval.mergeGenerationHeldBytes(_genHolderStore.getHeldBytes()); + retval.mergeGenerationHeldBytes(_genHolderStore.get_held_bytes()); return retval; } -- cgit v1.2.3