diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-03-09 14:10:14 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-03-09 14:10:58 +0000 |
commit | 414f81bdd8fa59944f79d86b9ca7c03d1e4080f4 (patch) | |
tree | df92895f93a0446ff6ed1ee410af434916e5e449 /searchlib | |
parent | 435af93308d985ec4c503d3789a6d48f923dd60a (diff) |
Report more of the static memory usage for datastores.
Diffstat (limited to 'searchlib')
6 files changed, 37 insertions, 32 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 5a41d738acb..16b5ef78951 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -912,8 +912,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(231640u, ptr->getStatus().getAllocated()); - EXPECT_EQ(544u, ptr->getStatus().getUsed()); + EXPECT_EQ(887756u, ptr->getStatus().getAllocated()); + EXPECT_EQ(656444u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -934,8 +934,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); ptr->updateStat(true); - EXPECT_EQ(231640u, ptr->getStatus().getAllocated()); - EXPECT_EQ(544u, ptr->getStatus().getUsed()); + EXPECT_EQ(887756u, ptr->getStatus().getAllocated()); + EXPECT_EQ(656444u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -947,8 +947,8 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(74648u, ptr->getStatus().getAllocated()); - EXPECT_EQ(176u, ptr->getStatus().getUsed()); + EXPECT_EQ(403552u, ptr->getStatus().getAllocated()); + EXPECT_EQ(328576u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -957,8 +957,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(245024u, ptr->getStatus().getAllocated()); - EXPECT_EQ(576, ptr->getStatus().getUsed()); + EXPECT_EQ(902256u, ptr->getStatus().getAllocated()); + EXPECT_EQ(657088u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1089,8 +1089,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(12320u, ptr->getStatus().getAllocated()); - EXPECT_EQ(32u, ptr->getStatus().getUsed()); + EXPECT_EQ(1474480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(1462192u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1099,8 +1099,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); ptr->updateStat(true); - EXPECT_EQ(12320u, ptr->getStatus().getAllocated()); - EXPECT_EQ(32u, ptr->getStatus().getUsed()); + EXPECT_EQ(1474480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(1462192u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1109,8 +1109,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(253608u, ptr->getStatus().getAllocated()); - EXPECT_EQ(596u, ptr->getStatus().getUsed()); + EXPECT_EQ(2371884u, ptr->getStatus().getAllocated()); + EXPECT_EQ(2118656u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1128,8 +1128,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); ptr->updateStat(true); - EXPECT_EQ(253608u, ptr->getStatus().getAllocated()); - EXPECT_EQ(596u, ptr->getStatus().getUsed()); + EXPECT_EQ(2371884u, ptr->getStatus().getAllocated()); + EXPECT_EQ(2118656u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1140,8 +1140,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(74680u, ptr->getStatus().getAllocated()); - EXPECT_EQ(208u, ptr->getStatus().getUsed()); + EXPECT_EQ(1865744u, ptr->getStatus().getAllocated()); + EXPECT_EQ(1790768u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1150,8 +1150,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(266992u, ptr->getStatus().getAllocated()); - EXPECT_EQ(628u, ptr->getStatus().getUsed()); + EXPECT_EQ(2386384u, ptr->getStatus().getAllocated()); + EXPECT_EQ(2119300u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 3b233298659..1b1c1181666 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned) TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) { - constexpr size_t BASE_ALLOCATED = 188172u; - constexpr size_t BASE_USED = 500u; + constexpr size_t BASE_ALLOCATED = 1172040u; + constexpr size_t BASE_USED = 984116u; { MySetup setup; Index index(setup); diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index fbd4629cc4a..c37fe2739ca 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") { TEST("require that both features and ranges counts towards memory usage") { DocumentFeaturesStore features_store(10); - EXPECT_EQUAL(364u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328152u, features_store.getMemoryUsage().usedBytes()); PredicateTreeAnnotations annotations; annotations.features.push_back(PredicateHash::hash64("foo=100-199")); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(372u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328160u, features_store.getMemoryUsage().usedBytes()); annotations.features.clear(); annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(468u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328256u, features_store.getMemoryUsage().usedBytes()); } TEST("require that DocumentFeaturesStore can be serialized") { @@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") { PredicateTreeAnnotations annotations; annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(460u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328248u, features_store.getMemoryUsage().usedBytes()); annotations.range_features.push_back({"bar", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(848u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328636u, features_store.getMemoryUsage().usedBytes()); features_store.remove(doc_id + 1); - EXPECT_EQUAL(800u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328588u, features_store.getMemoryUsage().usedBytes()); vespalib::DataBuffer buffer; features_store.serialize(buffer); DocumentFeaturesStore features_store2(buffer); - EXPECT_EQUAL(460u, features_store2.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(328248u, features_store2.getMemoryUsage().usedBytes()); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 7f8f3f92f9e..e6b584d29b2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -590,7 +590,6 @@ AttributeVector::getEstimatedSaveByteSize() const uint64_t idxFileSize = 0; uint64_t udatFileSize = 0; size_t fixedWidth = getFixedWidth(); - vespalib::MemoryUsage values_mem_usage = getEnumStoreValuesMemoryUsage(); if (hasMultiValue()) { idxFileSize = headerSize + sizeof(uint32_t) * (docIdLimit + 1); @@ -603,6 +602,7 @@ AttributeVector::getEstimatedSaveByteSize() const if (fixedWidth != 0) { udatFileSize = headerSize + fixedWidth * uniqueValueCount; } else { + vespalib::MemoryUsage values_mem_usage = getEnumStoreValuesMemoryUsage(); size_t unique_values_bytes = values_mem_usage.usedBytes() - (values_mem_usage.deadBytes() + values_mem_usage.allocatedBytesOnHold()); size_t ref_count_mem_usage = sizeof(uint32_t) * uniqueValueCount; diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 6d6022d59dd..c5188b89129 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -74,7 +74,7 @@ template <typename B> vespalib::MemoryUsage EnumAttribute<B>::getEnumStoreValuesMemoryUsage() const { - return _enumStore.get_values_memory_usage(); + return _enumStore.get_dynamic_values_memory_usage(); } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 0a0b2040b2a..df3a595ae34 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -91,7 +91,12 @@ public: uint32_t get_num_uniques() const override { return _dict->get_num_uniques(); } bool is_folded() const { return _is_folded;} - vespalib::MemoryUsage get_values_memory_usage() const override { return _store.get_allocator().get_data_store().getMemoryUsage(); } + vespalib::MemoryUsage get_values_memory_usage() const override { + return _store.get_allocator().get_data_store().getMemoryUsage(); + } + vespalib::MemoryUsage get_dynamic_values_memory_usage() const { + return _store.get_allocator().get_data_store().getDynamicMemoryUsage(); + } vespalib::MemoryUsage get_dictionary_memory_usage() const override { return _dict->get_memory_usage(); } vespalib::AddressSpace get_values_address_space_usage() const override; |