diff options
author | Tor Egge <Tor.Egge@yahoo-inc.com> | 2017-01-05 13:27:40 +0000 |
---|---|---|
committer | Tor Egge <Tor.Egge@yahoo-inc.com> | 2017-01-05 13:27:40 +0000 |
commit | 8183accd4463a6934e400dfef1005ee89d78957b (patch) | |
tree | ee3cba13b758c932ed6968dbd865f4230d82131a | |
parent | cbe2b9f63a29e784e6f46e5d2063c88529796fe2 (diff) |
Expose change vector memory usage.
16 files changed, 63 insertions, 6 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index 6cc94abdd10..5ca3ee75270 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -90,6 +90,14 @@ convertMultiValueToSlime(const MultiValueMappingBase &multiValue, Cursor &object convertMemoryUsageToSlime(multiValue.getMemoryUsage(), object.setObject("memoryUsage")); } + +void +convertChangeVectorToSlime(const AttributeVector &v, Cursor &object) +{ + MemoryUsage usage = v.getChangeVectorMemoryUsage(); + convertMemoryUsageToSlime(usage, object); +} + } AttributeVectorExplorer::AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute) @@ -116,6 +124,7 @@ AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bo if (multiValue) { convertMultiValueToSlime(*multiValue, object.setObject("multiValue")); } + convertChangeVectorToSlime(attr, object.setObject("changeVector")); object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit()); object.setLong("createSerialNum", attr.getCreateSerialNum()); } else { diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 9069c0c2d3e..ce6202f202f 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1850,11 +1850,13 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) EXPECT_EQUAL(1u, ia.getCurrentGeneration()); uint64_t lastAllocated; uint64_t lastOnHold; + MemoryUsage changeVectorMemoryUsage(attr->getChangeVectorMemoryUsage()); + size_t changeVectorAllocated = changeVectorMemoryUsage.allocatedBytes(); if (exactStatus) { - EXPECT_EQUAL(2u, ia.getStatus().getAllocated()); + EXPECT_EQUAL(2u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); } else { - EXPECT_LESS(0u, ia.getStatus().getAllocated()); + EXPECT_LESS(0u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); lastAllocated = ia.getStatus().getAllocated(); lastOnHold = ia.getStatus().getOnHold(); @@ -1866,7 +1868,7 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) ia.commit(true); EXPECT_EQUAL(3u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(6u, ia.getStatus().getAllocated()); + EXPECT_EQUAL(6u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(2u, ia.getStatus().getOnHold()); // no cleanup due to guard } else { EXPECT_LESS(lastAllocated, ia.getStatus().getAllocated()); @@ -1882,7 +1884,7 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) ia.commit(true); EXPECT_EQUAL(4u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(4u, ia.getStatus().getAllocated()); + EXPECT_EQUAL(4u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); // cleanup at end of addDoc() } else { EXPECT_GREATER(lastAllocated, ia.getStatus().getAllocated()); @@ -1898,7 +1900,7 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) ia.commit(); EXPECT_EQUAL(6u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(10u, ia.getStatus().getAllocated()); + EXPECT_EQUAL(10u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(4u, ia.getStatus().getOnHold()); // no cleanup due to guard } else { EXPECT_LESS(lastAllocated, ia.getStatus().getAllocated()); @@ -1910,7 +1912,7 @@ AttributeTest::testGeneration(const AttributePtr & attr, bool exactStatus) ia.commit(true); EXPECT_EQUAL(7u, ia.getCurrentGeneration()); if (exactStatus) { - EXPECT_EQUAL(6u, ia.getStatus().getAllocated()); + EXPECT_EQUAL(6u + changeVectorAllocated, ia.getStatus().getAllocated()); EXPECT_EQUAL(0u, ia.getStatus().getOnHold()); // cleanup at end of commit() } else { EXPECT_GREATER(lastAllocated, ia.getStatus().getAllocated()); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 2432851c481..25d42604890 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -843,6 +843,11 @@ AttributeVector::getEstimatedSaveByteSize() const return datFileSize + weightFileSize + idxFileSize + udatFileSize; } +MemoryUsage +AttributeVector::getChangeVectorMemoryUsage() const +{ + return MemoryUsage(0, 0, 0, 0); +} template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool); template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 7328d733818..c3d0e36184e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -710,6 +710,8 @@ public: virtual uint64_t getEstimatedSaveByteSize() const; static bool isEnumerated(const vespalib::GenericHeader &header); + + virtual MemoryUsage getChangeVectorMemoryUsage() const; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index f042064f8a4..e7831491257 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -7,6 +7,8 @@ namespace search { +class MemoryUsage; + struct ChangeBase { enum Type { NOOP, @@ -168,6 +170,7 @@ public: void clear(); const_iterator begin() const { return const_iterator(_v, 0); } const_iterator end() const { return const_iterator(_v, size()); } + MemoryUsage getMemoryUsage() const; private: void linkIn(uint32_t doc, size_t index, size_t last); Vector _v; diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.hpp b/searchlib/src/vespa/searchlib/attribute/changevector.hpp index c96829b931c..2de0d03d73d 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/changevector.hpp @@ -4,6 +4,7 @@ #include "changevector.h" #include <vespa/vespalib/util/array.hpp> +#include <vespa/searchlib/util/memoryusage.h> namespace search { @@ -71,5 +72,14 @@ ChangeVectorT<T>::linkIn(uint32_t doc, size_t first, size_t last) _v[_tail].setNext(size()); } +template <typename T> +MemoryUsage +ChangeVectorT<T>::getMemoryUsage() const +{ + size_t usedBytes = _v.size() * sizeof(T) + _docs.getMemoryUsed(); + size_t allocBytes = _v.capacity() * sizeof(T) + _docs.getMemoryConsumption(); + return MemoryUsage(allocBytes, usedBytes, 0, 0); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp index dd1206b500e..6eb9065335c 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp @@ -90,6 +90,12 @@ FloatingPointAttribute::getString(DocId doc, char * s, size_t sz) const { return s; } +MemoryUsage +FloatingPointAttribute::getChangeVectorMemoryUsage() const +{ + return _changes.getMemoryUsage(); +} + template class FloatingPointAttributeTemplate<float>; template class FloatingPointAttributeTemplate<double>; } diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index e34701a47cb..66b4125f832 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -39,6 +39,7 @@ protected: typedef ChangeVectorT< Change > ChangeVector; ChangeVector _changes; + virtual MemoryUsage getChangeVectorMemoryUsage() const override; private: uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; uint32_t get(DocId doc, const char ** v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index 558023c54e8..34c73737825 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -88,6 +88,12 @@ bool IntegerAttribute::apply(DocId doc, const ArithmeticValueUpdate & op) return retval; } +MemoryUsage +IntegerAttribute::getChangeVectorMemoryUsage() const +{ + return _changes.getMemoryUsage(); +} + template class IntegerAttributeTemplate<int8_t>; template class IntegerAttributeTemplate<int16_t>; template class IntegerAttributeTemplate<int32_t>; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index cf21c5920dd..aef139fe8a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -38,6 +38,7 @@ protected: typedef ChangeVectorT< Change > ChangeVector; ChangeVector _changes; + virtual MemoryUsage getChangeVectorMemoryUsage() const override; private: const char * getString(DocId doc, char * s, size_t sz) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 6a29f391790..712b51f5ea7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -163,6 +163,7 @@ MultiValueEnumAttribute<B, M>::onUpdateStat() total.merge(this->_enumStore.getMemoryUsage()); total.merge(this->_enumStore.getTreeMemoryUsage()); total.merge(this->_mvMapping.updateStat()); + total.merge(this->getChangeVectorMemoryUsage()); mergeMemoryStats(total); this->updateStatistics(this->_mvMapping.getTotalValueCnt(), this->_enumStore.getNumUniques(), total.allocatedBytes(), total.usedBytes(), total.deadBytes(), total.allocatedBytesOnHold()); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 71e1c367755..2cbebea242a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -75,6 +75,7 @@ template <typename B, typename M> void MultiValueNumericAttribute<B, M>::onUpdateStat() { MemoryUsage usage = this->_mvMapping.updateStat(); + usage.merge(this->getChangeVectorMemoryUsage()); this->updateStatistics(this->_mvMapping.getTotalValueCnt(), this->_mvMapping.getTotalValueCnt(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 4f131f3f94c..7d396451e6b 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -87,6 +87,7 @@ SingleValueEnumAttribute<B>::onUpdateStat() total.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); total.merge(this->_enumStore.getMemoryUsage()); total.merge(this->_enumStore.getTreeMemoryUsage()); + total.merge(this->getChangeVectorMemoryUsage()); mergeMemoryStats(total); this->updateStatistics(_enumIndices.size(), this->_enumStore.getNumUniques(), total.allocatedBytes(), total.usedBytes(), total.deadBytes(), total.allocatedBytesOnHold()); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 251b2d3acd1..99ec4d1e00f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -61,6 +61,7 @@ SingleValueNumericAttribute<B>::onUpdateStat() { MemoryUsage usage = _data.getMemoryUsage(); usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); + usage.merge(this->getChangeVectorMemoryUsage()); this->updateStatistics(_data.size(), _data.size(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); } diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 05e41957ca8..06a033654b6 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -537,4 +537,10 @@ StringAttribute::fixupEnumRefCounts(const EnumVector &enumHist) fprintf(stderr, "StringAttribute::fixupEnumRefCounts\n"); } +MemoryUsage +StringAttribute::getChangeVectorMemoryUsage() const +{ + return _changes.getMemoryUsage(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index 6be2d0ec082..0d287f2e671 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -70,6 +70,8 @@ protected: virtual bool onAddDoc(DocId doc); + + virtual MemoryUsage getChangeVectorMemoryUsage() const override; private: typedef attribute::LoadedStringVectorReal LoadedVectorR; virtual void fillPostings(LoadedVector & loaded); |