Report more of the static memory usage for datastores.

6 files changed, 37 insertions, 32 deletions

diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp
index 5a41d738acb..16b5ef78951 100644
--- a/searchlib/src/tests/attribute/attribute_test.cpp
+++ b/searchlib/src/tests/attribute/attribute_test.cpp
@@ -912,8 +912,8 @@ AttributeTest::testSingle()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-post-int32", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(231640u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(544u, ptr->getStatus().getUsed());
+            EXPECT_EQ(887756u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(656444u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values);
         }
@@ -934,8 +934,8 @@ AttributeTest::testSingle()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-post-float", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(231640u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(544u, ptr->getStatus().getUsed());
+            EXPECT_EQ(887756u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(656444u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testSingle<FloatingPointAttribute, double, float>(ptr, values);
         }
@@ -947,8 +947,8 @@ AttributeTest::testSingle()
         {
             AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE));
             ptr->updateStat(true);
-            EXPECT_EQ(74648u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(176u, ptr->getStatus().getUsed());
+            EXPECT_EQ(403552u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(328576u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testSingle<StringAttribute, string, string>(ptr, values);
         }
@@ -957,8 +957,8 @@ AttributeTest::testSingle()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-fs-string", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(245024u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(576, ptr->getStatus().getUsed());
+            EXPECT_EQ(902256u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(657088u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testSingle<StringAttribute, string, string>(ptr, values);
         }
@@ -1089,8 +1089,8 @@ AttributeTest::testArray()
         {
             AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY));
             ptr->updateStat(true);
-            EXPECT_EQ(12320u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(32u, ptr->getStatus().getUsed());
+            EXPECT_EQ(1474480u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(1462192u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1099,8 +1099,8 @@ AttributeTest::testArray()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("flags", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(12320u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(32u, ptr->getStatus().getUsed());
+            EXPECT_EQ(1474480u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(1462192u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1109,8 +1109,8 @@ AttributeTest::testArray()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("a-fs-int32", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(253608u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(596u, ptr->getStatus().getUsed());
+            EXPECT_EQ(2371884u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(2118656u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1128,8 +1128,8 @@ AttributeTest::testArray()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("a-fs-float", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(253608u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(596u, ptr->getStatus().getUsed());
+            EXPECT_EQ(2371884u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(2118656u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<FloatingPointAttribute, double>(ptr, values);
         }
@@ -1140,8 +1140,8 @@ AttributeTest::testArray()
         {
             AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY));
             ptr->updateStat(true);
-            EXPECT_EQ(74680u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(208u, ptr->getStatus().getUsed());
+            EXPECT_EQ(1865744u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(1790768u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<StringAttribute, string>(ptr, values);
         }
@@ -1150,8 +1150,8 @@ AttributeTest::testArray()
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("afs-string", cfg);
             ptr->updateStat(true);
-            EXPECT_EQ(266992u, ptr->getStatus().getAllocated());
-            EXPECT_EQ(628u, ptr->getStatus().getUsed());
+            EXPECT_EQ(2386384u, ptr->getStatus().getAllocated());
+            EXPECT_EQ(2119300u, ptr->getStatus().getUsed());
             addDocs(ptr, numDocs);
             testArray<StringAttribute, string>(ptr, values);
         }
diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
index 3b233298659..1b1c1181666 100644
--- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
@@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned)
 
 TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint)
 {
-    constexpr size_t BASE_ALLOCATED = 188172u;
-    constexpr size_t BASE_USED = 500u;
+    constexpr size_t BASE_ALLOCATED = 1172040u;
+    constexpr size_t BASE_USED = 984116u;
     {
         MySetup setup;
         Index index(setup);
diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp
index fbd4629cc4a..c37fe2739ca 100644
--- a/searchlib/src/tests/predicate/document_features_store_test.cpp
+++ b/searchlib/src/tests/predicate/document_features_store_test.cpp
@@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") {
 
 TEST("require that both features and ranges counts towards memory usage") {
     DocumentFeaturesStore features_store(10);
-    EXPECT_EQUAL(364u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328152u, features_store.getMemoryUsage().usedBytes());
 
     PredicateTreeAnnotations annotations;
     annotations.features.push_back(PredicateHash::hash64("foo=100-199"));
     features_store.insert(annotations, doc_id);
-    EXPECT_EQUAL(372u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328160u, features_store.getMemoryUsage().usedBytes());
 
     annotations.features.clear();
     annotations.range_features.push_back({"foo", 100, 199});
     features_store.insert(annotations, doc_id + 1);
-    EXPECT_EQUAL(468u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328256u, features_store.getMemoryUsage().usedBytes());
 }
 
 TEST("require that DocumentFeaturesStore can be serialized") {
@@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") {
     PredicateTreeAnnotations annotations;
     annotations.range_features.push_back({"foo", 100, 199});
     features_store.insert(annotations, doc_id);
-    EXPECT_EQUAL(460u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328248u, features_store.getMemoryUsage().usedBytes());
     annotations.range_features.push_back({"bar", 100, 199});
     features_store.insert(annotations, doc_id + 1);
-    EXPECT_EQUAL(848u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328636u, features_store.getMemoryUsage().usedBytes());
     features_store.remove(doc_id + 1);
-    EXPECT_EQUAL(800u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328588u, features_store.getMemoryUsage().usedBytes());
 
     vespalib::DataBuffer buffer;
     features_store.serialize(buffer);
     DocumentFeaturesStore features_store2(buffer);
-    EXPECT_EQUAL(460u, features_store2.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(328248u, features_store2.getMemoryUsage().usedBytes());
 }
 
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
index 7f8f3f92f9e..e6b584d29b2 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
@@ -590,7 +590,6 @@ AttributeVector::getEstimatedSaveByteSize() const
     uint64_t idxFileSize = 0;
     uint64_t udatFileSize = 0;
     size_t fixedWidth = getFixedWidth();
-    vespalib::MemoryUsage values_mem_usage = getEnumStoreValuesMemoryUsage();
 
     if (hasMultiValue()) {
         idxFileSize = headerSize + sizeof(uint32_t) * (docIdLimit + 1);
@@ -603,6 +602,7 @@ AttributeVector::getEstimatedSaveByteSize() const
         if (fixedWidth != 0) {
             udatFileSize = headerSize + fixedWidth * uniqueValueCount;
         } else {
+            vespalib::MemoryUsage values_mem_usage = getEnumStoreValuesMemoryUsage();
             size_t unique_values_bytes = values_mem_usage.usedBytes() -
                     (values_mem_usage.deadBytes() + values_mem_usage.allocatedBytesOnHold());
             size_t ref_count_mem_usage = sizeof(uint32_t) * uniqueValueCount;
diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp
index 6d6022d59dd..c5188b89129 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp
@@ -74,7 +74,7 @@ template <typename B>
 vespalib::MemoryUsage
 EnumAttribute<B>::getEnumStoreValuesMemoryUsage() const
 {
-    return _enumStore.get_values_memory_usage();
+    return _enumStore.get_dynamic_values_memory_usage();
 }
 
 template <typename B>
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h
index 0a0b2040b2a..df3a595ae34 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h
@@ -91,7 +91,12 @@ public:
     uint32_t get_num_uniques() const override { return _dict->get_num_uniques(); }
     bool is_folded() const { return _is_folded;}
 
-    vespalib::MemoryUsage get_values_memory_usage() const override { return _store.get_allocator().get_data_store().getMemoryUsage(); }
+    vespalib::MemoryUsage get_values_memory_usage() const override {
+        return _store.get_allocator().get_data_store().getMemoryUsage();
+    }
+    vespalib::MemoryUsage get_dynamic_values_memory_usage() const {
+        return _store.get_allocator().get_data_store().getDynamicMemoryUsage();
+    }
     vespalib::MemoryUsage get_dictionary_memory_usage() const override { return _dict->get_memory_usage(); }
 
     vespalib::AddressSpace get_values_address_space_usage() const override;