Add test showing accounted memory usage for empty attributes.

5 files changed, 53 insertions, 3 deletions

diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp
index 5c0e2c26bd2..18896ca24c1 100644
--- a/searchlib/src/tests/attribute/attribute_test.cpp
+++ b/searchlib/src/tests/attribute/attribute_test.cpp
@@ -896,6 +896,8 @@ AttributeTest::testSingle()
         fillNumeric(nibbleValues, numUniqueNibbles);
         {
             AttributePtr ptr = createAttribute("sv-int32", Config(BasicType::INT32, CollectionType::SINGLE));
+            ptr->updateStat(true);
+            EXPECT_EQ(12288u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values);
         }
@@ -908,6 +910,8 @@ AttributeTest::testSingle()
             Config cfg(BasicType::INT32, CollectionType::SINGLE);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-post-int32", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(231640u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values);
         }
@@ -917,6 +921,8 @@ AttributeTest::testSingle()
         fillNumeric(values, numUniques);
         {
             AttributePtr ptr = createAttribute("sv-float", Config(BasicType::FLOAT, CollectionType::SINGLE));
+            ptr->updateStat(true);
+            EXPECT_EQ(12288u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<FloatingPointAttribute, double, float>(ptr, values);
         }
@@ -924,6 +930,8 @@ AttributeTest::testSingle()
             Config cfg(BasicType::FLOAT, CollectionType::SINGLE);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-post-float", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(231640u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<FloatingPointAttribute, double, float>(ptr, values);
         }
@@ -934,6 +942,8 @@ AttributeTest::testSingle()
         fillString(values, numUniques);
         {
             AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE));
+            ptr->updateStat(true);
+            EXPECT_EQ(74648u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<StringAttribute, string, string>(ptr, values);
         }
@@ -941,6 +951,8 @@ AttributeTest::testSingle()
             Config cfg(Config(BasicType::STRING, CollectionType::SINGLE));
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("sv-fs-string", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(245024u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testSingle<StringAttribute, string, string>(ptr, values);
         }
@@ -1070,6 +1082,8 @@ AttributeTest::testArray()
         fillNumeric(values, numUniques);
         {
             AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY));
+            ptr->updateStat(true);
+            EXPECT_EQ(12320u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1077,6 +1091,8 @@ AttributeTest::testArray()
             Config cfg(BasicType::INT8, CollectionType::ARRAY);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("flags", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(12320u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1084,6 +1100,8 @@ AttributeTest::testArray()
             Config cfg(BasicType::INT32, CollectionType::ARRAY);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("a-fs-int32", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(253608u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
         }
@@ -1100,6 +1118,8 @@ AttributeTest::testArray()
             Config cfg(BasicType::FLOAT, CollectionType::ARRAY);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("a-fs-float", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(253608u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<FloatingPointAttribute, double>(ptr, values);
         }
@@ -1109,6 +1129,8 @@ AttributeTest::testArray()
         fillString(values, numUniques);
         {
             AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY));
+            ptr->updateStat(true);
+            EXPECT_EQ(74680u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<StringAttribute, string>(ptr, values);
         }
@@ -1116,6 +1138,8 @@ AttributeTest::testArray()
             Config cfg(BasicType::STRING, CollectionType::ARRAY);
             cfg.setFastSearch(true);
             AttributePtr ptr = createAttribute("afs-string", cfg);
+            ptr->updateStat(true);
+            EXPECT_EQ(266992u, ptr->getStatus().getAllocated());
             addDocs(ptr, numDocs);
             testArray<StringAttribute, string>(ptr, values);
         }
diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
index 176792962a6..b9f9becc5a4 100644
--- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
@@ -467,6 +467,14 @@ TEST_F(DoubleTest, nan_is_handled)
     }
     std::vector<uint32_t> exp_enumerated = { 0, 1, 4, 2, 3, 1, 4, 2 };
     EXPECT_EQ(exp_enumerated, enumerated);
-}    
+}
+
+TEST_F(DoubleTest, control_memory_usage) {
+    EXPECT_EQ(472u, sizeof(store));
+    EXPECT_EQ(32u, store.get_values_memory_usage().allocatedBytes());
+    EXPECT_EQ(98208u, store.get_dictionary_memory_usage().allocatedBytes());
+    EXPECT_EQ(98240u, store.getMemoryUsage().allocatedBytes());
+    EXPECT_EQ(144u, sizeof(BufferState));
+}
                 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h
index 2a193172959..d0ac6a216db 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.h
@@ -112,7 +112,7 @@ public:
     // Use this if references to array store is not an array of AtomicEntryRef
     std::unique_ptr<CompactingBuffers> start_compact_worst_buffers(const CompactionStrategy &compaction_strategy);
 
-    vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); }
+    vespalib::MemoryUsage getMemoryUsage() const;
     vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy);
     bool consider_compact() const noexcept { return _compaction_spec.compact() && !_store.has_held_buffers(); }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
index 53b149fda80..a0fdf3f6563 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
@@ -57,6 +57,15 @@ ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, s
 }
 
 template <typename EntryT, typename RefT, typename TypeMapperT>
+vespalib::MemoryUsage
+ArrayStore<EntryT, RefT, TypeMapperT>::getMemoryUsage() const {
+    vespalib::MemoryUsage usage =  _store.getMemoryUsage();
+    //TODO Must be accounted
+    // usage.incAllocatedBytes(_smallArrayTypes.capacity() * sizeof(SmallBufferType));
+    return usage;
+}
+
+template <typename EntryT, typename RefT, typename TypeMapperT>
 ArrayStore<EntryT, RefT, TypeMapperT>::~ArrayStore()
 {
     _store.reclaim_all_memory();
@@ -181,7 +190,7 @@ vespalib::MemoryUsage
 ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy)
 {
     auto address_space_usage = _store.getAddressSpaceUsage();
-    auto memory_usage = _store.getMemoryUsage();
+    auto memory_usage = getMemoryUsage();
     _compaction_spec = compaction_strategy.should_compact(memory_usage, address_space_usage);
     return memory_usage;
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
index a14082e2d5c..6c8b12b8be5 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
@@ -264,6 +264,15 @@ vespalib::MemoryUsage
 DataStoreBase::getMemoryUsage() const
 {
     auto stats = getMemStats();
+
+    size_t extra = 0;
+    extra += _buffers.capacity() * sizeof(BufferAndTypeId);
+    extra += _primary_buffer_ids.capacity() * sizeof(uint32_t);
+    extra += _states.capacity() * sizeof(BufferState);
+    extra += _typeHandlers.capacity() * sizeof(BufferTypeBase *);
+    extra += _free_lists.capacity() * sizeof(FreeList);
+    (void) extra; //TODO Must be accounted as static cost
+
     vespalib::MemoryUsage usage;
     usage.setAllocatedBytes(stats._allocBytes);
     usage.setUsedBytes(stats._usedBytes);