diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-04-20 14:15:21 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-04-20 14:15:21 +0200 |
commit | 81af61c96759a3ad3197b3ef56187697302dbec3 (patch) | |
tree | 1f0b2bdf5c6baa7f43ea4e8a0b6b0b782c638a18 | |
parent | f0b38157c7884f4b4c499cda6ff1b9f8eb19602b (diff) | |
parent | 91234c978ac30ab2421b7e1153f079bd0a218ad3 (diff) |
Merge pull request #17497 from vespa-engine/toregge/compact-posting-store-pass-3
Add consider_compact methods to posting store.
7 files changed, 86 insertions, 11 deletions
diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 7e49f8498da..ebaab3b4085 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -128,10 +128,13 @@ PostingStoreTest::populate(uint32_t sequence_length) { auto& store = _store; auto& dictionary = _value_store.get_dictionary(); + std::vector<EntryRef> refs; + for (int i = 0; i < 9000; ++i) { + refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + } dictionary.update_posting_list(_value_store.insert(1), _value_store.make_comparator(), [this, sequence_length](EntryRef) { return add_sequence(4, 4 + sequence_length); }); dictionary.update_posting_list(_value_store.insert(2), _value_store.make_comparator(), [this, sequence_length](EntryRef) { return add_sequence(5, 5 + sequence_length); }); - std::vector<EntryRef> refs; - for (int i = 0; i < 1000; ++i) { + for (int i = 9000; i < 11000; ++i) { refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); } for (auto& ref : refs) { @@ -156,10 +159,21 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length) EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); auto usage_before = store.getMemoryUsage(); - for (uint32_t pass = 0; pass < 15; ++pass) { - store.compact_worst_buffers(); + bool compaction_done = false; + search::CompactionStrategy compaction_strategy(0.05, 0.2); + for (uint32_t pass = 0; pass < 45; ++pass) { + store.update_stat(); + auto guard = _gen_handler.takeGuard(); + if (!store.consider_compact_worst_buffers(compaction_strategy)) { + compaction_done = true; + break; + } + inc_generation(); + EXPECT_FALSE(store.consider_compact_worst_buffers(compaction_strategy)); + guard = GenerationHandler::Guard(); inc_generation(); } + EXPECT_TRUE(compaction_done); EntryRef ref1 = get_posting_ref(1); EntryRef ref2 = get_posting_ref(2); EXPECT_NE(old_ref1, ref1); @@ -178,10 +192,21 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); auto usage_before = store.getMemoryUsage(); - for (uint32_t pass = 0; pass < 15; ++pass) { - store.compact_worst_btree_nodes(); + bool compaction_done = false; + search::CompactionStrategy compaction_strategy(0.05, 0.2); + for (uint32_t pass = 0; pass < 55; ++pass) { + store.update_stat(); + auto guard = _gen_handler.takeGuard(); + if (!store.consider_compact_worst_btree_nodes(compaction_strategy)) { + compaction_done = true; + break; + } + inc_generation(); + EXPECT_FALSE(store.consider_compact_worst_btree_nodes(compaction_strategy)); + guard = GenerationHandler::Guard(); inc_generation(); } + EXPECT_TRUE(compaction_done); EntryRef ref1 = get_posting_ref(1); EntryRef ref2 = get_posting_ref(2); EXPECT_EQ(old_ref1, ref1); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index 697fa33a060..eb54bb07753 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -18,7 +18,7 @@ template <typename B, typename M> void MultiValueNumericPostingAttribute<B, M>::mergeMemoryStats(vespalib::MemoryUsage & total) { - total.merge(this->getPostingList().getMemoryUsage()); + total.merge(this->getPostingList().update_stat()); } template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index d427df69903..a464a4d91d6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -63,7 +63,7 @@ template <typename B, typename T> void MultiValueStringPostingAttributeT<B, T>::mergeMemoryStats(vespalib::MemoryUsage &total) { - total.merge(this->_postingList.getMemoryUsage()); + total.merge(this->_postingList.update_stat()); } template <typename B, typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 336f53fb66a..6c62e650345 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -32,7 +32,9 @@ PostingStoreBase2::PostingStoreBase2(IEnumStoreDictionary& dictionary, Status &s _bvs(), _dictionary(dictionary), _status(status), - _bvExtraBytes(0) + _bvExtraBytes(0), + _cached_allocator_memory_usage(), + _cached_store_memory_usage() { } @@ -629,6 +631,21 @@ PostingStore<DataT>::getMemoryUsage() const } template <typename DataT> +vespalib::MemoryUsage +PostingStore<DataT>::update_stat() +{ + vespalib::MemoryUsage usage; + _cached_allocator_memory_usage = _allocator.getMemoryUsage(); + _cached_store_memory_usage = _store.getMemoryUsage(); + usage.merge(_cached_allocator_memory_usage); + usage.merge(_cached_store_memory_usage); + uint64_t bvExtraBytes = _bvExtraBytes; + usage.incUsedBytes(bvExtraBytes); + usage.incAllocatedBytes(bvExtraBytes); + return usage; +} + +template <typename DataT> void PostingStore<DataT>::move_btree_nodes(EntryRef ref) { @@ -720,6 +737,34 @@ PostingStore<DataT>::compact_worst_buffers() this->finishCompact(to_hold); } +template <typename DataT> +bool +PostingStore<DataT>::consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) +{ + if (_allocator.getNodeStore().has_held_buffers()) { + return false; + } + if (compaction_strategy.should_compact_memory(_cached_allocator_memory_usage.usedBytes(), _cached_allocator_memory_usage.deadBytes())) { + compact_worst_btree_nodes(); + return true; + } + return false; +} + +template <typename DataT> +bool +PostingStore<DataT>::consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy) +{ + if (_store.has_held_buffers()) { + return false; + } + if (compaction_strategy.should_compact_memory(_cached_store_memory_usage.usedBytes(), _cached_store_memory_usage.deadBytes())) { + compact_worst_buffers(); + return true; + } + return false; +} + template class PostingStore<BTreeNoLeafData>; template class PostingStore<int32_t>; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 1abe9c839c0..d1d86dc9f91 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -47,6 +47,8 @@ protected: IEnumStoreDictionary& _dictionary; Status &_status; uint64_t _bvExtraBytes; + vespalib::MemoryUsage _cached_allocator_memory_usage; + vespalib::MemoryUsage _cached_store_memory_usage; static constexpr uint32_t BUFFERTYPE_BITVECTOR = 9u; @@ -184,12 +186,15 @@ public: static inline DataT bitVectorWeight(); vespalib::MemoryUsage getMemoryUsage() const; + vespalib::MemoryUsage update_stat(); void move_btree_nodes(EntryRef ref); EntryRef move(EntryRef ref); void compact_worst_btree_nodes(); void compact_worst_buffers(); + bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy); + bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy); private: size_t internalSize(uint32_t typeId, const RefType & iRef) const; size_t internalFrozenSize(uint32_t typeId, const RefType & iRef) const; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 325874091ef..2bd501eebf2 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -36,7 +36,7 @@ template <typename B> void SingleValueNumericPostingAttribute<B>::mergeMemoryStats(vespalib::MemoryUsage & total) { - total.merge(this->_postingList.getMemoryUsage()); + total.merge(this->_postingList.update_stat()); } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index 50ed621b13e..402f66d18d1 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -34,7 +34,7 @@ template <typename B> void SingleValueStringPostingAttributeT<B>::mergeMemoryStats(vespalib::MemoryUsage & total) { - total.merge(this->_postingList.getMemoryUsage()); + total.merge(this->_postingList.update_stat()); } template <typename B> |