From de6042314e39dd9efcba561a4e4bdfc3c90a2642 Mon Sep 17 00:00:00 2001
From: Tor Egge <Tor.Egge@online.no>
Date: Thu, 28 Apr 2022 00:23:15 +0200
Subject: Only sample hash map memory usage once per commit.

---
 .../memoryindex/compact_words_store/compact_words_store_test.cpp  | 3 +++
 searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp | 8 ++++++--
 searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h   | 1 +
 searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp | 2 ++
 4 files changed, 12 insertions(+), 2 deletions(-)

(limited to 'searchlib')

diff --git a/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp
index a82c7ea396d..273936c9e76 100644
--- a/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp
+++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp
@@ -106,6 +106,7 @@ TEST(CompactWordStoreTest, multiple_words_can_be_inserted_retrieved_and_removed)
             b.insert(EntryRef(wordRef));
         }
         store.insert(b);
+        store.commit();
         MemoryUsage usage = store.getMemoryUsage();
         std::cout << "memory usage (insert): docId=" << docId << ", alloc=" << usage.allocatedBytes() << ", used=" << usage.usedBytes() << std::endl;
     }
@@ -117,6 +118,7 @@ TEST(CompactWordStoreTest, multiple_words_can_be_inserted_retrieved_and_removed)
             EXPECT_EQ(wordRef++, word.ref());
         }
         store.remove(docId);
+        store.commit();
         MemoryUsage usage = store.getMemoryUsage();
         std::cout << "memory usage (remove): docId=" << docId << ", alloc=" << usage.allocatedBytes() << ", used=" << usage.usedBytes() << std::endl;
     }
@@ -145,6 +147,7 @@ TEST(CompactWordStoreTest, memory_usage_is_updated_after_insert)
     MemoryUsage init = store.getMemoryUsage();
 
     store.insert(Builder(d1).insert(w1));
+    store.commit();
     MemoryUsage after = store.getMemoryUsage();
     EXPECT_GE(after.allocatedBytes(), init.allocatedBytes());
     EXPECT_GT(after.usedBytes(), init.usedBytes());
diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp
index d640506765a..59df4a731d2 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp
@@ -147,14 +147,12 @@ CompactWordsStore::insert(const Builder &builder)
             builder.docId());
         LOG_ABORT("should not be reached");
     }
-    update_docs_memory_usage();
 }
 
 void 
 CompactWordsStore::remove(uint32_t docId)
 {
     _docs.erase(docId);
-    update_docs_memory_usage();
 }
 
 CompactWordsStore::Iterator
@@ -174,6 +172,12 @@ CompactWordsStore::update_docs_memory_usage()
     _docs_allocated_bytes.store(_docs.getMemoryConsumption(), std::memory_order_relaxed);
 }
 
+void
+CompactWordsStore::commit()
+{
+    update_docs_memory_usage();
+}
+
 vespalib::MemoryUsage
 CompactWordsStore::getMemoryUsage() const
 {
diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h
index b8c98129b9f..d90c04dc5b6 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h
@@ -95,6 +95,7 @@ public:
     void insert(const Builder &builder);
     void remove(uint32_t docId);
     Iterator get(uint32_t docId) const;
+    void commit();
     vespalib::MemoryUsage getMemoryUsage() const;
 };
 
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp
index 1861544939f..3cdf26d09ff 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp
@@ -43,6 +43,7 @@ void
 FieldIndexRemover::flush()
 {
     if (_wordFieldDocTuples.empty()) {
+        _store.commit();
         return;
     }
     ShiftBasedRadixSorter<WordFieldDocTuple, WordFieldDocTuple::Radix, std::less<WordFieldDocTuple>, 24, true>::
@@ -56,6 +57,7 @@ FieldIndexRemover::flush()
         builder->insert(tuple._wordRef);
     }
     _store.insert(*builder);
+    _store.commit();
     _wordFieldDocTuples.clear();
 }
 
-- 
cgit v1.2.3