diff options
8 files changed, 58 insertions, 18 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index e801e6f3706..8f75ddd1315 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -100,7 +100,12 @@ convertChangeVectorToSlime(const AttributeVector &v, Cursor &object) void convertPostingBaseToSlime(const IPostingListAttributeBase &postingBase, Cursor &object) { - convertMemoryUsageToSlime(postingBase.getMemoryUsage(), object.setObject("memoryUsage")); + auto& cursor = object.setObject("memory_usage"); + auto memory_usage = postingBase.getMemoryUsage(); + convertMemoryUsageToSlime(memory_usage.total, cursor.setObject("total")); + convertMemoryUsageToSlime(memory_usage.btrees, cursor.setObject("btrees")); + convertMemoryUsageToSlime(memory_usage.short_arrays, cursor.setObject("short_arrays")); + convertMemoryUsageToSlime(memory_usage.bitvectors, cursor.setObject("bitvectors")); } vespalib::string @@ -173,7 +178,7 @@ AttributeVectorExplorer::get_state_helper(const AttributeVector& attr, const ves } const IPostingListAttributeBase *postingBase = attr.getIPostingListAttributeBase(); if (postingBase) { - convertPostingBaseToSlime(*postingBase, object.setObject("postingList")); + convertPostingBaseToSlime(*postingBase, object.setObject("posting_store")); } const auto* tensor_attr = attr.asTensorAttribute(); if (tensor_attr) { diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index a1c78265e00..61d94e120f5 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -177,7 +177,7 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length) EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1)); EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2)); auto usage_after = store.getMemoryUsage(); - EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); + EXPECT_GT(usage_before.total.deadBytes(), usage_after.total.deadBytes()); } void @@ -211,9 +211,9 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2)); auto usage_after = store.getMemoryUsage(); if ((sequence_length < huge_sequence_length) || !_config.getIsFilter()) { - EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); + EXPECT_GT(usage_before.total.deadBytes(), usage_after.total.deadBytes()); } else { - EXPECT_EQ(usage_before.deadBytes(), usage_after.deadBytes()); + EXPECT_EQ(usage_before.total.deadBytes(), usage_after.total.deadBytes()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h index bb376c59ca0..88eac32fc76 100644 --- a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h +++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h @@ -2,6 +2,7 @@ #pragma once +#include "posting_store_memory_usage.h" #include <vespa/searchcommon/attribute/iattributevector.h> namespace vespalib::datastore { class CompactionStrategy; } @@ -17,10 +18,10 @@ public: virtual ~IPostingListAttributeBase() = default; virtual void clearPostings(IAttributeVector::EnumHandle eidx, uint32_t fromLid, uint32_t toLid) = 0; virtual void forwardedShrinkLidSpace(uint32_t newSize) = 0; - virtual vespalib::MemoryUsage getMemoryUsage() const = 0; + virtual PostingStoreMemoryUsage getMemoryUsage() const = 0; virtual bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) = 0; virtual bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy) = 0; }; -} // namespace search::attribute +} diff --git a/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h b/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h new file mode 100644 index 00000000000..59a537d5a65 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/posting_store_memory_usage.h @@ -0,0 +1,33 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/memoryusage.h> + +namespace search::attribute { + +/** + * Memory usage for a posting store with details for each type of posting list. + */ +struct PostingStoreMemoryUsage { + vespalib::MemoryUsage btrees; + vespalib::MemoryUsage short_arrays; + vespalib::MemoryUsage bitvectors; + vespalib::MemoryUsage total; + PostingStoreMemoryUsage(vespalib::MemoryUsage btrees_in, + vespalib::MemoryUsage short_arrays_in, + vespalib::MemoryUsage bitvectors_in) + : btrees(btrees_in), + short_arrays(short_arrays_in), + bitvectors(bitvectors_in), + total() + { + total.merge(btrees); + total.merge(short_arrays); + total.merge(bitvectors); + } + +}; + +} + diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index 10df06c3181..4e88fb96c7e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -173,7 +173,7 @@ PostingListAttributeBase<P>::forwardedShrinkLidSpace(uint32_t newSize) } template <typename P> -vespalib::MemoryUsage +attribute::PostingStoreMemoryUsage PostingListAttributeBase<P>::getMemoryUsage() const { return _postingList.getMemoryUsage(); diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index 33f86889624..3987d661d26 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -66,7 +66,7 @@ protected: uint32_t toLid, const vespalib::datastore::EntryComparator &cmp); void forwardedShrinkLidSpace(uint32_t newSize) override; - vespalib::MemoryUsage getMemoryUsage() const override; + attribute::PostingStoreMemoryUsage getMemoryUsage() const override; bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) override; bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy) override; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index d63828f11fe..8bee8b3f7f7 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -594,16 +594,16 @@ PostingStore<DataT>::clear(const EntryRef ref) template <typename DataT> -vespalib::MemoryUsage +PostingStoreMemoryUsage PostingStore<DataT>::getMemoryUsage() const { - vespalib::MemoryUsage usage; - usage.merge(_allocator.getMemoryUsage()); - usage.merge(_store.getMemoryUsage()); + auto btrees = _allocator.getMemoryUsage(); + auto short_arrays = _store.getMemoryUsage(); + vespalib::MemoryUsage bitvectors; uint64_t bvExtraBytes = _bvExtraBytes; - usage.incUsedBytes(bvExtraBytes); - usage.incAllocatedBytes(bvExtraBytes); - return usage; + bitvectors.incUsedBytes(bvExtraBytes); + bitvectors.incAllocatedBytes(bvExtraBytes); + return {btrees, short_arrays, bitvectors}; } template <typename DataT> diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 033deb4e280..e3fb88c81f1 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -3,8 +3,9 @@ #pragma once #include "enum_store_dictionary.h" -#include "postinglisttraits.h" #include "posting_store_compaction_spec.h" +#include "posting_store_memory_usage.h" +#include "postinglisttraits.h" #include <set> namespace search { @@ -190,7 +191,7 @@ public: std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(RefType ref, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const; static inline DataT bitVectorWeight(); - vespalib::MemoryUsage getMemoryUsage() const; + PostingStoreMemoryUsage getMemoryUsage() const; vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy); void move_btree_nodes(const std::vector<EntryRef> &refs); |