diff options
18 files changed, 234 insertions, 196 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 16b5ef78951..f336742ca4b 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -912,8 +912,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(887756u, ptr->getStatus().getAllocated()); - EXPECT_EQ(656444u, ptr->getStatus().getUsed()); + EXPECT_EQ(347084u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101852u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -934,8 +934,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); ptr->updateStat(true); - EXPECT_EQ(887756u, ptr->getStatus().getAllocated()); - EXPECT_EQ(656444u, ptr->getStatus().getUsed()); + EXPECT_EQ(347084u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101852u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -947,8 +947,8 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(403552u, ptr->getStatus().getAllocated()); - EXPECT_EQ(328576u, ptr->getStatus().getUsed()); + EXPECT_EQ(133216u, ptr->getStatus().getAllocated()); + EXPECT_EQ(53280u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -957,8 +957,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(902256u, ptr->getStatus().getAllocated()); - EXPECT_EQ(657088u, ptr->getStatus().getUsed()); + EXPECT_EQ(361584u, ptr->getStatus().getAllocated()); + EXPECT_EQ(105216u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1089,8 +1089,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(1474480u, ptr->getStatus().getAllocated()); - EXPECT_EQ(1462192u, ptr->getStatus().getUsed()); + EXPECT_EQ(528304u, ptr->getStatus().getAllocated()); + EXPECT_EQ(512576u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1099,8 +1099,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); ptr->updateStat(true); - EXPECT_EQ(1474480u, ptr->getStatus().getAllocated()); - EXPECT_EQ(1462192u, ptr->getStatus().getUsed()); + EXPECT_EQ(528304u, ptr->getStatus().getAllocated()); + EXPECT_EQ(512576u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1109,8 +1109,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(2371884u, ptr->getStatus().getAllocated()); - EXPECT_EQ(2118656u, ptr->getStatus().getUsed()); + EXPECT_EQ(885036u, ptr->getStatus().getAllocated()); + EXPECT_EQ(614448u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1128,8 +1128,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); ptr->updateStat(true); - EXPECT_EQ(2371884u, ptr->getStatus().getAllocated()); - EXPECT_EQ(2118656u, ptr->getStatus().getUsed()); + EXPECT_EQ(885036u, ptr->getStatus().getAllocated()); + EXPECT_EQ(614448u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1140,8 +1140,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(1865744u, ptr->getStatus().getAllocated()); - EXPECT_EQ(1790768u, ptr->getStatus().getUsed()); + EXPECT_EQ(649232u, ptr->getStatus().getAllocated()); + EXPECT_EQ(565856u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1150,8 +1150,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(2386384u, ptr->getStatus().getAllocated()); - EXPECT_EQ(2119300u, ptr->getStatus().getUsed()); + EXPECT_EQ(899536u, ptr->getStatus().getAllocated()); + EXPECT_EQ(617812u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 7d2a89b6e5b..39e31b23498 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -581,21 +581,15 @@ AttributePostingListTest::doCompactEnumStore(Tree &tree, TreeManager &treeMgr, ValueHandle &valueHandle) { - LOG(info, - "doCompactEnumStore start"); + LOG(info,"doCompactEnumStore start"); Tree::Iterator i = tree.begin(treeMgr); - uint32_t numBuffers = valueHandle.getNumBuffers(); std::vector<uint32_t> toHold; + valueHandle.for_each_active_buffer([&toHold](uint32_t buffer_id, const vespalib::datastore::BufferState &) { + toHold.push_back(buffer_id); + }); - for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) { - vespalib::datastore::BufferState &state = valueHandle.getBufferState(bufferId); - if (state.isActive()) { - toHold.push_back(bufferId); - // Freelists already disabled due to variable sized data - } - } valueHandle.switch_primary_buffer(0, 0u); for (; i.valid(); ++i) diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 1b1c1181666..8073fb8d232 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned) TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) { - constexpr size_t BASE_ALLOCATED = 1172040u; - constexpr size_t BASE_USED = 984116u; + constexpr size_t BASE_ALLOCATED = 361032u; + constexpr size_t BASE_USED = 151188u; { MySetup setup; Index index(setup); diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index c37fe2739ca..4ac4bdc32f0 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") { TEST("require that both features and ranges counts towards memory usage") { DocumentFeaturesStore features_store(10); - EXPECT_EQUAL(328152u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50136u, features_store.getMemoryUsage().usedBytes()); PredicateTreeAnnotations annotations; annotations.features.push_back(PredicateHash::hash64("foo=100-199")); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(328160u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50144u, features_store.getMemoryUsage().usedBytes()); annotations.features.clear(); annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(328256u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50240u, features_store.getMemoryUsage().usedBytes()); } TEST("require that DocumentFeaturesStore can be serialized") { @@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") { PredicateTreeAnnotations annotations; annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(328248u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50232u, features_store.getMemoryUsage().usedBytes()); annotations.range_features.push_back({"bar", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(328636u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50620u, features_store.getMemoryUsage().usedBytes()); features_store.remove(doc_id + 1); - EXPECT_EQUAL(328588u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50572u, features_store.getMemoryUsage().usedBytes()); vespalib::DataBuffer buffer; features_store.serialize(buffer); DocumentFeaturesStore features_store2(buffer); - EXPECT_EQUAL(328248u, features_store2.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50232u, features_store2.getMemoryUsage().usedBytes()); } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 2aa05e7fa9f..94720212faf 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -727,13 +727,12 @@ template <typename DataT> void PostingStore<DataT>::compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto compacting_buffers = this->start_compact_worst_buffers(compaction_spec, compaction_strategy); bool compact_btree_roots = false; auto filter = compacting_buffers->make_entry_ref_filter(); // Start with looking at buffers being compacted for (uint32_t buffer_id : compacting_buffers->get_buffer_ids()) { - if (isBTree(_store.getBufferState(buffer_id).getTypeId())) { + if (isBTree(_store.getBufferMeta(buffer_id).getTypeId())) { compact_btree_roots = true; } } diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index ef64549e16a..b8da9ea6042 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -1064,7 +1064,8 @@ adjustAllocatedBytes(size_t nodeCount, size_t nodeSize) TEST_F(BTreeTest, require_that_memory_usage_is_calculated) { - constexpr size_t BASE = 163912; + constexpr size_t BASE_ALLOCATED = 28744u; + constexpr size_t BASE_USED = 24984; typedef BTreeNodeAllocator<int32_t, int8_t, btree::NoAggregated, MyTraits::INTERNAL_SLOTS, MyTraits::LEAF_SLOTS> NodeAllocator; @@ -1083,8 +1084,8 @@ TEST_F(BTreeTest, require_that_memory_usage_is_calculated) const uint32_t initialLeafNodes = 128u; mu.incAllocatedBytes(adjustAllocatedBytes(initialInternalNodes, sizeof(INode))); mu.incAllocatedBytes(adjustAllocatedBytes(initialLeafNodes, sizeof(LNode))); - mu.incAllocatedBytes(BASE); - mu.incUsedBytes(BASE); + mu.incAllocatedBytes(BASE_ALLOCATED); + mu.incUsedBytes(BASE_USED); mu.incUsedBytes(sizeof(INode)); mu.incDeadBytes(sizeof(INode)); EXPECT_TRUE(assertMemoryUsage(mu, tm.getMemoryUsage())); @@ -1115,8 +1116,8 @@ TEST_F(BTreeTest, require_that_memory_usage_is_calculated) mu = vespalib::MemoryUsage(); mu.incAllocatedBytes(adjustAllocatedBytes(initialInternalNodes, sizeof(INode))); mu.incAllocatedBytes(adjustAllocatedBytes(initialLeafNodes, sizeof(LNode))); - mu.incAllocatedBytes(BASE); - mu.incUsedBytes(BASE); + mu.incAllocatedBytes(BASE_ALLOCATED); + mu.incUsedBytes(BASE_USED); mu.incUsedBytes(sizeof(INode) * 2); mu.incDeadBytes(sizeof(INode) * 2); mu.incUsedBytes(sizeof(LNode)); diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 97e1ddb985d..c32554232ef 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -210,16 +210,16 @@ INSTANTIATE_TEST_SUITE_P(NumberStoreFreeListsDisabledMultiTest, TEST_P(NumberStoreTest, control_static_sizes) { #ifdef _LIBCPP_VERSION - EXPECT_EQ(472u, sizeof(store)); - EXPECT_EQ(296u, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); + EXPECT_EQ(464u, sizeof(store)); + EXPECT_EQ(280u, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); #else - EXPECT_EQ(504u, sizeof(store)); - EXPECT_EQ(328u, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); + EXPECT_EQ(496u, sizeof(store)); + EXPECT_EQ(320u, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); #endif EXPECT_EQ(112u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); MemoryUsage usage = store.getMemoryUsage(); - EXPECT_EQ(1312160u, usage.allocatedBytes()); - EXPECT_EQ(1311232u, usage.usedBytes()); + EXPECT_EQ(202144u, usage.allocatedBytes()); + EXPECT_EQ(197776u, usage.usedBytes()); } TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 794be39ae9b..7121bf5ce11 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -474,7 +474,8 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) TEST(DataStoreTest, require_that_memory_usage_is_calculated) { - constexpr size_t BASE = 676; + constexpr size_t BASE_ALLOCATED = 4228; + constexpr size_t BASE_USED = 308; MyStore s; MyRef r = s.addEntry(10); s.addEntry(20); @@ -483,8 +484,8 @@ TEST(DataStoreTest, require_that_memory_usage_is_calculated) s.holdBuffer(r.bufferId()); s.assign_generation(100); vespalib::MemoryUsage m = s.getMemoryUsage(); - EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE, m.allocatedBytes()); - EXPECT_EQ(5 * sizeof(int) + BASE, m.usedBytes()); + EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); + EXPECT_EQ(5 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(0 * sizeof(int), m.deadBytes()); EXPECT_EQ(5 * sizeof(int), m.allocatedBytesOnHold()); s.reclaim_memory(101); @@ -492,28 +493,29 @@ TEST(DataStoreTest, require_that_memory_usage_is_calculated) TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list) { - constexpr size_t BASE = 676; + constexpr size_t BASE_ALLOCATED = 4228; + constexpr size_t BASE_USED = 308; MyStore s; MyRef r1 = s.addEntry(10); MyRef r2 = s.addEntry(20); MyRef r3 = s.addEntry(30); (void) r3; vespalib::MemoryUsage m = s.getMemoryUsage(); - EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE, m.allocatedBytes()); - EXPECT_EQ(4 * sizeof(int) + BASE, m.usedBytes()); + EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); + EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(0 * sizeof(int), m.allocatedBytesOnHold()); s.holdElem(r1, 1); m = s.getMemoryUsage(); - EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE, m.allocatedBytes()); - EXPECT_EQ(4 * sizeof(int) + BASE, m.usedBytes()); + EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); + EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(1 * sizeof(int), m.allocatedBytesOnHold()); s.disableElemHoldList(); s.holdElem(r2, 1); m = s.getMemoryUsage(); - EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE, m.allocatedBytes()); - EXPECT_EQ(4 * sizeof(int) + BASE, m.usedBytes()); + EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); + EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(2 * sizeof(int), m.deadBytes()); EXPECT_EQ(1 * sizeof(int), m.allocatedBytesOnHold()); s.assign_generation(100); @@ -538,7 +540,7 @@ void assertGrowStats(GrowthStats expSizes, TEST(DataStoreTest, require_that_buffer_growth_works) { - constexpr size_t BASE = 41032u; + constexpr size_t BASE = 10312; // Always switch to new buffer, min size 4 assertGrowStats({ 4, 4, 4, 4, 8, 16, 16, 32, 64, 64 }, { 4 }, 20 + BASE, 4, 0); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index cf62d238d53..1781a680bac 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -470,14 +470,14 @@ TEST_F(DoubleTest, nan_is_handled) } TEST_F(DoubleTest, control_memory_usage) { - EXPECT_EQ(464, sizeof(store)); + EXPECT_EQ(456u, sizeof(store)); EXPECT_EQ(144u, sizeof(BufferState)); - EXPECT_EQ(163908u, store.get_values_memory_usage().allocatedBytes()); - EXPECT_EQ(163892u, store.get_values_memory_usage().usedBytes()); - EXPECT_EQ(262120u, store.get_dictionary_memory_usage().allocatedBytes()); - EXPECT_EQ(164176u, store.get_dictionary_memory_usage().usedBytes()); - EXPECT_EQ(426028u, store.getMemoryUsage().allocatedBytes()); - EXPECT_EQ(328068u, store.getMemoryUsage().usedBytes()); + EXPECT_EQ(28740u, store.get_values_memory_usage().allocatedBytes()); + EXPECT_EQ(24804u, store.get_values_memory_usage().usedBytes()); + EXPECT_EQ(126952u, store.get_dictionary_memory_usage().allocatedBytes()); + EXPECT_EQ(25248u, store.get_dictionary_memory_usage().usedBytes()); + EXPECT_EQ(155692u, store.getMemoryUsage().allocatedBytes()); + EXPECT_EQ(50052, store.getMemoryUsage().usedBytes()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index dd786e5f2e2..c2b65d72f03 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -76,10 +76,9 @@ public: return ConstArrayRef(); } RefT internalRef(ref); - uint32_t typeId = _store.getTypeId(internalRef.bufferId()); - if (typeId != _largeArrayTypeId) [[likely]] { - size_t arraySize = _mapper.get_array_size(typeId); - return getSmallArray(internalRef, arraySize); + const BufferAndMeta & bufferAndMeta = _store.getBufferMeta(internalRef.bufferId()); + if (bufferAndMeta.getTypeId() != _largeArrayTypeId) [[likely]] { + return getSmallArray(internalRef, bufferAndMeta.getArraySize()); } else { return getLargeArray(internalRef); } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp index 0a440e3e867..224ed4b0c8f 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp @@ -52,5 +52,23 @@ BufferFreeList::disable() _free_list = nullptr; } +void +BufferFreeList::push_entry(EntryRef ref) { + if (empty()) { + attach(); + } + _free_refs.push_back(ref); +} +EntryRef +BufferFreeList::pop_entry() { + EntryRef ret = _free_refs.back(); + _free_refs.pop_back(); + if (empty()) { + detach(); + } + _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed); + return ret; +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h index 74fd47056de..148ddd8db88 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h @@ -41,21 +41,8 @@ public: bool enabled() const { return _free_list != nullptr; } bool empty() const { return _free_refs.empty(); } uint32_t array_size() const { return _array_size; } - void push_entry(EntryRef ref) { - if (empty()) { - attach(); - } - _free_refs.push_back(ref); - } - EntryRef pop_entry() { - EntryRef ret = _free_refs.back(); - _free_refs.pop_back(); - if (empty()) { - detach(); - } - _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed); - return ret; - } + void push_entry(EntryRef ref); + EntryRef pop_entry(); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp index 45a94693eeb..47fba1ef697 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp @@ -171,7 +171,7 @@ BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer) } void -BufferState::disableElemHoldList() +BufferState::disable_elem_hold_list() { _disableElemHoldList = true; } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index aa7f6dfdfa4..c3e6110cc52 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -86,7 +86,7 @@ public: * Disable hold of elements, just mark elements as dead without cleanup. * Typically used when tearing down data structure in a controlled manner. */ - void disableElemHoldList(); + void disable_elem_hold_list(); /** * Update stats to reflect that the given elements are put on hold. @@ -134,23 +134,28 @@ public: class BufferAndMeta { public: - BufferAndMeta() : BufferAndMeta(nullptr, 0, 0) { } - BufferAndMeta(void* buffer, uint32_t typeId, uint32_t arraySize) - : _buffer(buffer), - _typeId(typeId), - _arraySize(arraySize) - { } + BufferAndMeta() : BufferAndMeta(nullptr, nullptr, 0, 0) { } std::atomic<void*>& get_atomic_buffer() noexcept { return _buffer; } void* get_buffer_relaxed() noexcept { return _buffer.load(std::memory_order_relaxed); } const void* get_buffer_acquire() const noexcept { return _buffer.load(std::memory_order_acquire); } uint32_t getTypeId() const { return _typeId; } uint32_t getArraySize() const { return _arraySize; } + BufferState * get_state_relaxed() { return _state.load(std::memory_order_relaxed); } + const BufferState * get_state_acquire() const { return _state.load(std::memory_order_acquire); } void setTypeId(uint32_t typeId) { _typeId = typeId; } void setArraySize(uint32_t arraySize) { _arraySize = arraySize; } + void set_state(BufferState * state) { _state.store(state, std::memory_order_release); } private: - std::atomic<void*> _buffer; - uint32_t _typeId; - uint32_t _arraySize; + BufferAndMeta(void* buffer, BufferState * state, uint32_t typeId, uint32_t arraySize) + : _buffer(buffer), + _state(state), + _typeId(typeId), + _arraySize(arraySize) + { } + std::atomic<void*> _buffer; + std::atomic<BufferState*> _state; + uint32_t _typeId; + uint32_t _arraySize; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index 99bdb19576f..a40aa713bca 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -84,16 +84,17 @@ DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t m : _entry_ref_hold_list(), _buffers(numBuffers), _primary_buffer_ids(), - _states(numBuffers), + _stash(), _typeHandlers(), _free_lists(), _compaction_count(0u), _genHolder(), _maxArrays(maxArrays), - _numBuffers(numBuffers), - _offset_bits(offset_bits), + _bufferIdLimit(0u), _hold_buffer_count(0u), + _offset_bits(offset_bits), _freeListsEnabled(false), + _disableElemHoldList(false), _initializing(false) { } @@ -107,7 +108,7 @@ void DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t elemsNeeded) { size_t buffer_id = getFirstFreeBufferId(); - if ((buffer_id < _states.size()) && !getBufferState(buffer_id).isFree()) { + if (buffer_id >= getMaxNumBuffers()) { LOG_ABORT(vespalib::make_string("switch_primary_buffer(%u, %zu): did not find a free buffer", typeId, elemsNeeded).c_str()); } @@ -159,19 +160,24 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed uint32_t DataStoreBase::getFirstFreeBufferId() { - for (uint32_t buffer_id = 0; buffer_id < _states.size(); buffer_id++) { - if (getBufferState(buffer_id).isFree()) { + uint32_t buffer_id = 0; + for (auto & buffer : _buffers) { + BufferState * state = buffer.get_state_relaxed(); + if (state == nullptr || state->isFree()) { return buffer_id; } + buffer_id++; } - // Need next(new) buffer - return _states.size(); + // No free buffer, return out of bounds + return buffer_id; } BufferState & DataStoreBase::getBufferState(uint32_t buffer_id) noexcept { - assert(buffer_id < _states.size()); - return _states[buffer_id]; + assert(buffer_id < get_bufferid_limit_relaxed()); + BufferState * state = _buffers[buffer_id].get_state_relaxed(); + assert(state != nullptr); + return *state; } void @@ -202,7 +208,7 @@ DataStoreBase::init_primary_buffers() uint32_t numTypes = _primary_buffer_ids.size(); for (uint32_t typeId = 0; typeId < numTypes; ++typeId) { size_t buffer_id = getFirstFreeBufferId(); - assert((buffer_id == _states.size()) || getBufferState(buffer_id).isFree()); + assert(buffer_id <= get_bufferid_limit_relaxed()); onActive(buffer_id, typeId, 0u); _primary_buffer_ids[typeId] = buffer_id; } @@ -253,9 +259,12 @@ DataStoreBase::reclaim_all_memory() void DataStoreBase::dropBuffers() { - uint32_t numBuffers = _buffers.size(); - for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) { - getBufferState(bufferId).dropBuffer(bufferId, _buffers[bufferId].get_atomic_buffer()); + uint32_t buffer_id_limit = get_bufferid_limit_relaxed(); + for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { + BufferAndMeta & buffer = _buffers[bufferId]; + BufferState * state = buffer.get_state_relaxed(); + assert(state != nullptr); + state->dropBuffer(bufferId, buffer.get_atomic_buffer()); } _genHolder.reclaim_all(); } @@ -278,18 +287,17 @@ DataStoreBase::getMemoryUsage() const { size_t extra_allocated = 0; extra_allocated += _buffers.capacity() * sizeof(BufferAndMeta); extra_allocated += _primary_buffer_ids.capacity() * sizeof(uint32_t); - extra_allocated += _states.capacity() * sizeof(BufferState); extra_allocated += _typeHandlers.capacity() * sizeof(BufferTypeBase *); extra_allocated += _free_lists.capacity() * sizeof(FreeList); size_t extra_used = 0; extra_used += _buffers.size() * sizeof(BufferAndMeta); extra_used += _primary_buffer_ids.size() * sizeof(uint32_t); - extra_used += _states.size() * sizeof(BufferState); extra_used += _typeHandlers.size() * sizeof(BufferTypeBase *); extra_used += _free_lists.size() * sizeof(FreeList); usage.incAllocatedBytes(extra_allocated); usage.incUsedBytes(extra_used); + usage.merge(_stash.get_memory_usage()); return usage; } @@ -305,41 +313,27 @@ DataStoreBase::holdBuffer(uint32_t bufferId) void DataStoreBase::enableFreeLists() { - for (auto& bState : _states) { - if (!bState.isActive() || bState.getCompacting()) { - continue; - } - bState.enable_free_list(_free_lists[bState.getTypeId()]); - } + for_each_buffer([this](BufferState & state) { + if (!state.isActive() || state.getCompacting()) return; + state.enable_free_list(_free_lists[state.getTypeId()]); + }); _freeListsEnabled = true; } void DataStoreBase::disableFreeLists() { - for (auto& bState : _states) { - bState.disable_free_list(); - } + for_each_buffer([](BufferState & state) { state.disable_free_list(); }); _freeListsEnabled = false; } void -DataStoreBase::enableFreeList(uint32_t bufferId) -{ - BufferState &state = getBufferState(bufferId); - if (_freeListsEnabled && state.isActive() && !state.getCompacting()) { - state.enable_free_list(_free_lists[state.getTypeId()]); - } -} - -void DataStoreBase::disableElemHoldList() { - for (auto &state : _states) { - if (!state.isFree()) { - state.disableElemHoldList(); - } - } + for_each_buffer([](BufferState & state) { + if (!state.isFree()) state.disable_elem_hold_list(); + }); + _disableElemHoldList = true; } MemoryStats @@ -347,19 +341,23 @@ DataStoreBase::getMemStats() const { MemoryStats stats; - for (const auto& bState: _states) { - auto typeHandler = bState.getTypeHandler(); - auto state = bState.getState(); + uint32_t buffer_id_limit = get_bufferid_limit_acquire(); + stats._freeBuffers = (getMaxNumBuffers() - buffer_id_limit); + for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { + const BufferState * bState = _buffers[bufferId].get_state_acquire(); + assert(bState != nullptr); + auto typeHandler = bState->getTypeHandler(); + auto state = bState->getState(); if ((state == BufferState::State::FREE) || (typeHandler == nullptr)) { ++stats._freeBuffers; } else if (state == BufferState::State::ACTIVE) { size_t elementSize = typeHandler->elementSize(); ++stats._activeBuffers; - bState.stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(elementSize, stats); } else if (state == BufferState::State::HOLD) { size_t elementSize = typeHandler->elementSize(); ++stats._holdBuffers; - bState.stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(elementSize, stats); } else { LOG_ABORT("should not be reached"); } @@ -374,21 +372,24 @@ DataStoreBase::getMemStats() const vespalib::AddressSpace DataStoreBase::getAddressSpaceUsage() const { + uint32_t buffer_id_limit = get_bufferid_limit_acquire(); size_t usedArrays = 0; size_t deadArrays = 0; - size_t limitArrays = 0; - for (const auto& bState: _states) { - if (bState.isActive()) { - uint32_t arraySize = bState.getArraySize(); - usedArrays += bState.size() / arraySize; - deadArrays += bState.stats().dead_elems() / arraySize; - limitArrays += bState.capacity() / arraySize; - } else if (bState.isOnHold()) { - uint32_t arraySize = bState.getArraySize(); - usedArrays += bState.size() / arraySize; - limitArrays += bState.capacity() / arraySize; - } else if (bState.isFree()) { + size_t limitArrays = size_t(_maxArrays) * (getMaxNumBuffers() - buffer_id_limit); + for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { + const BufferState * bState = _buffers[bufferId].get_state_acquire(); + assert(bState != nullptr); + if (bState->isFree()) { limitArrays += _maxArrays; + } else if (bState->isActive()) { + uint32_t arraySize = bState->getArraySize(); + usedArrays += bState->size() / arraySize; + deadArrays += bState->stats().dead_elems() / arraySize; + limitArrays += bState->capacity() / arraySize; + } else if (bState->isOnHold()) { + uint32_t arraySize = bState->getArraySize(); + usedArrays += bState->size() / arraySize; + limitArrays += bState->capacity() / arraySize; } else { LOG_ABORT("should not be reached"); } @@ -400,13 +401,29 @@ void DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) { assert(typeId < _typeHandlers.size()); - assert(bufferId < _numBuffers); - BufferState &state = getBufferState(bufferId); + assert(bufferId <= _bufferIdLimit); + BufferAndMeta & bufferMeta = _buffers[bufferId]; - state.onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer()); + BufferState *state = bufferMeta.get_state_relaxed(); + if (state == nullptr) { + BufferState & newState = _stash.create<BufferState>(); + if (_disableElemHoldList) { + newState.disable_elem_hold_list(); + } + if ( ! _freeListsEnabled) { + newState.disable_free_list(); + } + state = & newState; + bufferMeta.set_state(state); + _bufferIdLimit.store(bufferId + 1, std::memory_order_release); + } + assert(state->isFree()); + state->onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer()); bufferMeta.setTypeId(typeId); - bufferMeta.setArraySize(state.getArraySize()); - enableFreeList(bufferId); + bufferMeta.setArraySize(state->getArraySize()); + if (_freeListsEnabled && state->isActive() && !state->getCompacting()) { + state->enable_free_list(_free_lists[state->getTypeId()]); + } } void @@ -448,7 +465,7 @@ DataStoreBase::markCompacting(uint32_t bufferId) } assert(!state.getCompacting()); state.setCompacting(); - state.disableElemHoldList(); + state.disable_elem_hold_list(); state.disable_free_list(); inc_compaction_count(); } @@ -456,39 +473,41 @@ DataStoreBase::markCompacting(uint32_t bufferId) std::unique_ptr<CompactingBuffers> DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { + uint32_t buffer_id_limit = get_bufferid_limit_relaxed(); // compact memory usage - CompactBufferCandidates elem_buffers(_numBuffers, compaction_strategy.get_max_buffers(), + CompactBufferCandidates elem_buffers(buffer_id_limit, compaction_strategy.get_max_buffers(), compaction_strategy.get_active_buffers_ratio(), compaction_strategy.getMaxDeadBytesRatio() / 2, CompactionStrategy::DEAD_BYTES_SLACK); // compact address space - CompactBufferCandidates array_buffers(_numBuffers, compaction_strategy.get_max_buffers(), + CompactBufferCandidates array_buffers(buffer_id_limit, compaction_strategy.get_max_buffers(), compaction_strategy.get_active_buffers_ratio(), compaction_strategy.getMaxDeadAddressSpaceRatio() / 2, CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK); - uint32_t free_buffers = _buffers.size() - _states.size(); - for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) { - const auto &state = getBufferState(bufferId); - if (state.isActive()) { - auto typeHandler = state.getTypeHandler(); + uint32_t free_buffers = getMaxNumBuffers() - buffer_id_limit; + for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { + BufferState * state = _buffers[bufferId].get_state_relaxed(); + assert(state != nullptr); + if (state->isFree()) { + free_buffers++; + } else if (state->isActive()) { + auto typeHandler = state->getTypeHandler(); uint32_t arraySize = typeHandler->getArraySize(); uint32_t reservedElements = typeHandler->getReservedElements(bufferId); - size_t used_elems = state.size(); - size_t deadElems = state.stats().dead_elems() - reservedElements; + size_t used_elems = state->size(); + size_t deadElems = state->stats().dead_elems() - reservedElements; if (compaction_spec.compact_memory()) { elem_buffers.add(bufferId, used_elems, deadElems); } if (compaction_spec.compact_address_space()) { - array_buffers.add(bufferId, used_elems / arraySize, deadElems / arraySize); + array_buffers.add(bufferId, used_elems / arraySize, deadElems / arraySize); } - } else if (state.isFree()) { - ++free_buffers; } } elem_buffers.set_free_buffers(free_buffers); array_buffers.set_free_buffers(free_buffers); std::vector<uint32_t> result; - result.reserve(std::min(_numBuffers, 2 * compaction_strategy.get_max_buffers())); + result.reserve(std::min(buffer_id_limit, 2 * compaction_strategy.get_max_buffers())); elem_buffers.select(result); array_buffers.select(result); std::sort(result.begin(), result.end()); @@ -497,7 +516,7 @@ DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const for (auto buffer_id : result) { markCompacting(buffer_id); } - return std::make_unique<CompactingBuffers>(*this, _numBuffers, _offset_bits, std::move(result)); + return std::make_unique<CompactingBuffers>(*this, buffer_id_limit, _offset_bits, std::move(result)); } void diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 8749f2a27e6..ecbac451c5a 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -8,9 +8,8 @@ #include <vespa/vespalib/util/address_space.h> #include <vespa/vespalib/util/generationholder.h> #include <vespa/vespalib/util/generation_hold_list.h> -#include <vespa/vespalib/util/memoryusage.h> +#include <vespa/vespalib/util/stash.h> #include <atomic> -#include <deque> #include <vector> namespace vespalib::datastore { @@ -75,7 +74,20 @@ public: uint32_t primary_buffer_id(uint32_t typeId) const { return _primary_buffer_ids[typeId]; } BufferState &getBufferState(uint32_t buffer_id) noexcept; const BufferAndMeta & getBufferMeta(uint32_t buffer_id) const { return _buffers[buffer_id]; } - uint32_t getNumBuffers() const { return _numBuffers; } + uint32_t getMaxNumBuffers() const noexcept { return _buffers.size(); } + uint32_t get_bufferid_limit_acquire() const noexcept { return _bufferIdLimit.load(std::memory_order_acquire); } + uint32_t get_bufferid_limit_relaxed() noexcept { return _bufferIdLimit.load(std::memory_order_relaxed); } + + template<typename FuncType> + void for_each_active_buffer(FuncType func) { + uint32_t buffer_id_limit = get_bufferid_limit_relaxed(); + for (uint32_t i = 0; i < buffer_id_limit; i++) { + const BufferState * state = _buffers[i].get_state_relaxed(); + if (state && state->isActive()) { + func(i, *state); + } + } + } /** * Assign generation on data elements on hold lists added since the last time this function was called. @@ -218,11 +230,7 @@ private: * Hold of buffer has ended. */ void doneHoldBuffer(uint32_t bufferId); - /** - * Enable free list management. - * This only works for fixed size elements. - */ - void enableFreeList(uint32_t bufferId); + /** * Switch buffer state to active for the given buffer. * @@ -236,24 +244,33 @@ private: void fallbackResize(uint32_t bufferId, size_t elementsNeeded); uint32_t getFirstFreeBufferId(); + template<typename FuncType> + void for_each_buffer(FuncType func) { + uint32_t buffer_id_limit = get_bufferid_limit_relaxed(); + for (uint32_t i = 0; i < buffer_id_limit; i++) { + func(*(_buffers[i].get_state_relaxed())); + } + } + virtual void reclaim_all_entry_refs() = 0; - std::vector<BufferAndMeta> _buffers; // For fast mapping with known types + std::vector<BufferAndMeta> _buffers; // For fast mapping with known types // Provides a mapping from typeId -> primary buffer for that type. // The primary buffer is used for allocations of new element(s) if no available slots are found in free lists. std::vector<uint32_t> _primary_buffer_ids; - std::vector<BufferState> _states; + Stash _stash; std::vector<BufferTypeBase *> _typeHandlers; // TypeId -> handler std::vector<FreeList> _free_lists; mutable std::atomic<uint64_t> _compaction_count; vespalib::GenerationHolder _genHolder; const uint32_t _maxArrays; - const uint32_t _numBuffers; - const uint32_t _offset_bits; + std::atomic<uint32_t> _bufferIdLimit; uint32_t _hold_buffer_count; + const uint8_t _offset_bits; bool _freeListsEnabled; + bool _disableElemHoldList; bool _initializing; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index aef6ea07290..52b0798543f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -102,8 +102,8 @@ private: std::unique_ptr<CompactingBuffers> _compacting_buffers; void allocMapping() { - _mapping.resize(RefT::numBuffers()); auto& data_store = _compacting_buffers->get_store(); + _mapping.resize(data_store.get_bufferid_limit_relaxed()); for (const auto bufferId : _compacting_buffers->get_buffer_ids()) { BufferState &state = data_store.getBufferState(bufferId); _mapping[bufferId].resize(state.get_used_arrays()); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index 6d08a027bf1..32513d09c72 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -42,13 +42,10 @@ template <typename RefT> void UniqueStoreEnumerator<RefT>::allocate_enum_values(DataStoreBase & store) { - _enumValues.resize(RefType::numBuffers()); - for (uint32_t bufferId = 0; bufferId < RefType::numBuffers(); ++bufferId) { - const BufferState &state = store.getBufferState(bufferId); - if (state.isActive()) { - _enumValues[bufferId].resize(state.get_used_arrays()); - } - } + _enumValues.resize(store.get_bufferid_limit_relaxed()); + store.for_each_active_buffer([this](uint32_t buffer_id, const BufferState & state) { + _enumValues[buffer_id].resize(state.get_used_arrays()); + }); } template <typename RefT> |