diff options
5 files changed, 53 insertions, 3 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 5c0e2c26bd2..18896ca24c1 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -896,6 +896,8 @@ AttributeTest::testSingle() fillNumeric(nibbleValues, numUniqueNibbles); { AttributePtr ptr = createAttribute("sv-int32", Config(BasicType::INT32, CollectionType::SINGLE)); + ptr->updateStat(true); + EXPECT_EQ(12288u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -908,6 +910,8 @@ AttributeTest::testSingle() Config cfg(BasicType::INT32, CollectionType::SINGLE); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); + ptr->updateStat(true); + EXPECT_EQ(231640u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -917,6 +921,8 @@ AttributeTest::testSingle() fillNumeric(values, numUniques); { AttributePtr ptr = createAttribute("sv-float", Config(BasicType::FLOAT, CollectionType::SINGLE)); + ptr->updateStat(true); + EXPECT_EQ(12288u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -924,6 +930,8 @@ AttributeTest::testSingle() Config cfg(BasicType::FLOAT, CollectionType::SINGLE); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); + ptr->updateStat(true); + EXPECT_EQ(231640u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -934,6 +942,8 @@ AttributeTest::testSingle() fillString(values, numUniques); { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); + ptr->updateStat(true); + EXPECT_EQ(74648u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -941,6 +951,8 @@ AttributeTest::testSingle() Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); + ptr->updateStat(true); + EXPECT_EQ(245024u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1070,6 +1082,8 @@ AttributeTest::testArray() fillNumeric(values, numUniques); { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); + ptr->updateStat(true); + EXPECT_EQ(12320u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1077,6 +1091,8 @@ AttributeTest::testArray() Config cfg(BasicType::INT8, CollectionType::ARRAY); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); + ptr->updateStat(true); + EXPECT_EQ(12320u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1084,6 +1100,8 @@ AttributeTest::testArray() Config cfg(BasicType::INT32, CollectionType::ARRAY); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); + ptr->updateStat(true); + EXPECT_EQ(253608u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1100,6 +1118,8 @@ AttributeTest::testArray() Config cfg(BasicType::FLOAT, CollectionType::ARRAY); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); + ptr->updateStat(true); + EXPECT_EQ(253608u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1109,6 +1129,8 @@ AttributeTest::testArray() fillString(values, numUniques); { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); + ptr->updateStat(true); + EXPECT_EQ(74680u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1116,6 +1138,8 @@ AttributeTest::testArray() Config cfg(BasicType::STRING, CollectionType::ARRAY); cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); + ptr->updateStat(true); + EXPECT_EQ(266992u, ptr->getStatus().getAllocated()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 176792962a6..b9f9becc5a4 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -467,6 +467,14 @@ TEST_F(DoubleTest, nan_is_handled) } std::vector<uint32_t> exp_enumerated = { 0, 1, 4, 2, 3, 1, 4, 2 }; EXPECT_EQ(exp_enumerated, enumerated); -} +} + +TEST_F(DoubleTest, control_memory_usage) { + EXPECT_EQ(472u, sizeof(store)); + EXPECT_EQ(32u, store.get_values_memory_usage().allocatedBytes()); + EXPECT_EQ(98208u, store.get_dictionary_memory_usage().allocatedBytes()); + EXPECT_EQ(98240u, store.getMemoryUsage().allocatedBytes()); + EXPECT_EQ(144u, sizeof(BufferState)); +} GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 2a193172959..d0ac6a216db 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -112,7 +112,7 @@ public: // Use this if references to array store is not an array of AtomicEntryRef std::unique_ptr<CompactingBuffers> start_compact_worst_buffers(const CompactionStrategy &compaction_strategy); - vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } + vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy); bool consider_compact() const noexcept { return _compaction_spec.compact() && !_store.has_held_buffers(); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 53b149fda80..a0fdf3f6563 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -57,6 +57,15 @@ ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, s } template <typename EntryT, typename RefT, typename TypeMapperT> +vespalib::MemoryUsage +ArrayStore<EntryT, RefT, TypeMapperT>::getMemoryUsage() const { + vespalib::MemoryUsage usage = _store.getMemoryUsage(); + //TODO Must be accounted + // usage.incAllocatedBytes(_smallArrayTypes.capacity() * sizeof(SmallBufferType)); + return usage; +} + +template <typename EntryT, typename RefT, typename TypeMapperT> ArrayStore<EntryT, RefT, TypeMapperT>::~ArrayStore() { _store.reclaim_all_memory(); @@ -181,7 +190,7 @@ vespalib::MemoryUsage ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy) { auto address_space_usage = _store.getAddressSpaceUsage(); - auto memory_usage = _store.getMemoryUsage(); + auto memory_usage = getMemoryUsage(); _compaction_spec = compaction_strategy.should_compact(memory_usage, address_space_usage); return memory_usage; } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index a14082e2d5c..6c8b12b8be5 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -264,6 +264,15 @@ vespalib::MemoryUsage DataStoreBase::getMemoryUsage() const { auto stats = getMemStats(); + + size_t extra = 0; + extra += _buffers.capacity() * sizeof(BufferAndTypeId); + extra += _primary_buffer_ids.capacity() * sizeof(uint32_t); + extra += _states.capacity() * sizeof(BufferState); + extra += _typeHandlers.capacity() * sizeof(BufferTypeBase *); + extra += _free_lists.capacity() * sizeof(FreeList); + (void) extra; //TODO Must be accounted as static cost + vespalib::MemoryUsage usage; usage.setAllocatedBytes(stats._allocBytes); usage.setUsedBytes(stats._usedBytes); |