Merge pull request #29309 from vespa-engine/geirst/posting-store-memory-details

More detailed posting store memory usage in attribute explorer.

7 files changed, 51 insertions, 16 deletions

diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
index a1c78265e00..61d94e120f5 100644
--- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
+++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
@@ -177,7 +177,7 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length)
     EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1));
     EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2));
     auto usage_after = store.getMemoryUsage();
-    EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes());
+    EXPECT_GT(usage_before.total.deadBytes(), usage_after.total.deadBytes());
 }
 
 void
@@ -211,9 +211,9 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length)
     EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2));
     auto usage_after = store.getMemoryUsage();
     if ((sequence_length < huge_sequence_length) || !_config.getIsFilter()) {
-        EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes());
+        EXPECT_GT(usage_before.total.deadBytes(), usage_after.total.deadBytes());
     } else {
-        EXPECT_EQ(usage_before.deadBytes(), usage_after.deadBytes());
+        EXPECT_EQ(usage_before.total.deadBytes(), usage_after.total.deadBytes());
     }
 }
 
diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h
index bb376c59ca0..88eac32fc76 100644
--- a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h
+++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include "posting_store_memory_usage.h"
 #include <vespa/searchcommon/attribute/iattributevector.h>
 
 namespace vespalib::datastore { class CompactionStrategy; }
@@ -17,10 +18,10 @@ public:
     virtual ~IPostingListAttributeBase() = default;
     virtual void clearPostings(IAttributeVector::EnumHandle eidx, uint32_t fromLid, uint32_t toLid) = 0;
     virtual void forwardedShrinkLidSpace(uint32_t newSize) = 0;
-    virtual vespalib::MemoryUsage getMemoryUsage() const = 0;
+    virtual PostingStoreMemoryUsage getMemoryUsage() const = 0;
     virtual bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) = 0;
     virtual bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy) = 0;
 };
 
-} // namespace search::attribute
+}
 
diff --git a/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h b/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h
new file mode 100644
index 00000000000..59a537d5a65
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h
@@ -0,0 +1,33 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/memoryusage.h>
+
+namespace search::attribute {
+
+/**
+ * Memory usage for a posting store with details for each type of posting list.
+ */
+struct PostingStoreMemoryUsage {
+    vespalib::MemoryUsage btrees;
+    vespalib::MemoryUsage short_arrays;
+    vespalib::MemoryUsage bitvectors;
+    vespalib::MemoryUsage total;
+    PostingStoreMemoryUsage(vespalib::MemoryUsage btrees_in,
+                            vespalib::MemoryUsage short_arrays_in,
+                            vespalib::MemoryUsage bitvectors_in)
+        : btrees(btrees_in),
+          short_arrays(short_arrays_in),
+          bitvectors(bitvectors_in),
+          total()
+    {
+        total.merge(btrees);
+        total.merge(short_arrays);
+        total.merge(bitvectors);
+    }
+
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp
index 10df06c3181..4e88fb96c7e 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp
@@ -173,7 +173,7 @@ PostingListAttributeBase<P>::forwardedShrinkLidSpace(uint32_t newSize)
 }
 
 template <typename P>
-vespalib::MemoryUsage
+attribute::PostingStoreMemoryUsage
 PostingListAttributeBase<P>::getMemoryUsage() const
 {
     return _postingList.getMemoryUsage();
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
index 33f86889624..3987d661d26 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
@@ -66,7 +66,7 @@ protected:
                        uint32_t toLid, const vespalib::datastore::EntryComparator &cmp);
 
     void forwardedShrinkLidSpace(uint32_t newSize) override;
-    vespalib::MemoryUsage getMemoryUsage() const override;
+    attribute::PostingStoreMemoryUsage getMemoryUsage() const override;
     bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) override;
     bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy) override;
 
diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
index d63828f11fe..8bee8b3f7f7 100644
--- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
@@ -594,16 +594,16 @@ PostingStore<DataT>::clear(const EntryRef ref)
 
 
 template <typename DataT>
-vespalib::MemoryUsage
+PostingStoreMemoryUsage
 PostingStore<DataT>::getMemoryUsage() const
 {
-    vespalib::MemoryUsage usage;
-    usage.merge(_allocator.getMemoryUsage());
-    usage.merge(_store.getMemoryUsage());
+    auto btrees = _allocator.getMemoryUsage();
+    auto short_arrays = _store.getMemoryUsage();
+    vespalib::MemoryUsage bitvectors;
     uint64_t bvExtraBytes = _bvExtraBytes;
-    usage.incUsedBytes(bvExtraBytes);
-    usage.incAllocatedBytes(bvExtraBytes);
-    return usage;
+    bitvectors.incUsedBytes(bvExtraBytes);
+    bitvectors.incAllocatedBytes(bvExtraBytes);
+    return {btrees, short_arrays, bitvectors};
 }
 
 template <typename DataT>
diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h
index 033deb4e280..e3fb88c81f1 100644
--- a/searchlib/src/vespa/searchlib/attribute/postingstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h
@@ -3,8 +3,9 @@
 #pragma once
 
 #include "enum_store_dictionary.h"
-#include "postinglisttraits.h"
 #include "posting_store_compaction_spec.h"
+#include "posting_store_memory_usage.h"
+#include "postinglisttraits.h"
 #include <set>
 
 namespace search {
@@ -190,7 +191,7 @@ public:
     std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(RefType ref, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const;
 
     static inline DataT bitVectorWeight();
-    vespalib::MemoryUsage getMemoryUsage() const;
+    PostingStoreMemoryUsage getMemoryUsage() const;
     vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy);
 
     void move_btree_nodes(const std::vector<EntryRef> &refs);