diff options
author | Tor Egge <Tor.Egge@online.no> | 2022-04-28 00:23:15 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2022-04-28 00:23:15 +0200 |
commit | de6042314e39dd9efcba561a4e4bdfc3c90a2642 (patch) | |
tree | 43873f696178d5cbda35c1dcf6512fcfd3bdc9ea /searchlib | |
parent | 3845451fd2e55db7e6979ab12b8704a2520de205 (diff) |
Only sample hash map memory usage once per commit.
Diffstat (limited to 'searchlib')
4 files changed, 12 insertions, 2 deletions
diff --git a/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp index a82c7ea396d..273936c9e76 100644 --- a/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp +++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp @@ -106,6 +106,7 @@ TEST(CompactWordStoreTest, multiple_words_can_be_inserted_retrieved_and_removed) b.insert(EntryRef(wordRef)); } store.insert(b); + store.commit(); MemoryUsage usage = store.getMemoryUsage(); std::cout << "memory usage (insert): docId=" << docId << ", alloc=" << usage.allocatedBytes() << ", used=" << usage.usedBytes() << std::endl; } @@ -117,6 +118,7 @@ TEST(CompactWordStoreTest, multiple_words_can_be_inserted_retrieved_and_removed) EXPECT_EQ(wordRef++, word.ref()); } store.remove(docId); + store.commit(); MemoryUsage usage = store.getMemoryUsage(); std::cout << "memory usage (remove): docId=" << docId << ", alloc=" << usage.allocatedBytes() << ", used=" << usage.usedBytes() << std::endl; } @@ -145,6 +147,7 @@ TEST(CompactWordStoreTest, memory_usage_is_updated_after_insert) MemoryUsage init = store.getMemoryUsage(); store.insert(Builder(d1).insert(w1)); + store.commit(); MemoryUsage after = store.getMemoryUsage(); EXPECT_GE(after.allocatedBytes(), init.allocatedBytes()); EXPECT_GT(after.usedBytes(), init.usedBytes()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp index d640506765a..59df4a731d2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp @@ -147,14 +147,12 @@ CompactWordsStore::insert(const Builder &builder) builder.docId()); LOG_ABORT("should not be reached"); } - update_docs_memory_usage(); } void CompactWordsStore::remove(uint32_t docId) { _docs.erase(docId); - update_docs_memory_usage(); } CompactWordsStore::Iterator @@ -174,6 +172,12 @@ CompactWordsStore::update_docs_memory_usage() _docs_allocated_bytes.store(_docs.getMemoryConsumption(), std::memory_order_relaxed); } +void +CompactWordsStore::commit() +{ + update_docs_memory_usage(); +} + vespalib::MemoryUsage CompactWordsStore::getMemoryUsage() const { diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h index b8c98129b9f..d90c04dc5b6 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h @@ -95,6 +95,7 @@ public: void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; + void commit(); vespalib::MemoryUsage getMemoryUsage() const; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp index 1861544939f..3cdf26d09ff 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp @@ -43,6 +43,7 @@ void FieldIndexRemover::flush() { if (_wordFieldDocTuples.empty()) { + _store.commit(); return; } ShiftBasedRadixSorter<WordFieldDocTuple, WordFieldDocTuple::Radix, std::less<WordFieldDocTuple>, 24, true>:: @@ -56,6 +57,7 @@ FieldIndexRemover::flush() builder->insert(tuple._wordRef); } _store.insert(*builder); + _store.commit(); _wordFieldDocTuples.clear(); } |