diff options
-rw-r--r-- | metrics/src/vespa/metrics/CMakeLists.txt | 1 | ||||
-rw-r--r-- | metrics/src/vespa/metrics/memoryconsumption.cpp | 24 | ||||
-rw-r--r-- | metrics/src/vespa/metrics/memoryconsumption.h | 14 | ||||
-rw-r--r-- | metrics/src/vespa/metrics/namehash.cpp | 43 | ||||
-rw-r--r-- | metrics/src/vespa/metrics/namehash.h | 32 | ||||
-rw-r--r-- | metrics/src/vespa/metrics/summetric.hpp | 1 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/memoryindex/memoryindex.h | 3 | ||||
-rw-r--r-- | searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h | 1 |
8 files changed, 75 insertions, 44 deletions
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index eeb90553309..cd5a5ba4629 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(metrics metricsnapshot.cpp metrictimer.cpp metricvalueset.cpp + namehash.cpp state_api_adapter.cpp summetric.cpp textwriter.cpp diff --git a/metrics/src/vespa/metrics/memoryconsumption.cpp b/metrics/src/vespa/metrics/memoryconsumption.cpp index d0a75fe186d..31748b836c4 100644 --- a/metrics/src/vespa/metrics/memoryconsumption.cpp +++ b/metrics/src/vespa/metrics/memoryconsumption.cpp @@ -1,12 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memoryconsumption.h" +#include <vespa/vespalib/stllike/hash_set.h> #include <sstream> namespace metrics { -MemoryConsumption::MemoryConsumption() { +struct SeenStrings : public vespalib::hash_set<const void*> { }; +struct SnapShotUsage : public std::vector<std::pair<std::string, uint32_t> > { }; + +MemoryConsumption::MemoryConsumption() + : _seenStrings(std::make_unique<SeenStrings>()), + _snapShotUsage(std::make_unique<SnapShotUsage>()) +{ memset(&_consumerCount, 0, reinterpret_cast<size_t>(&_seenStrings) - reinterpret_cast<size_t>(&_consumerCount)); - _seenStrings.resize(1000); + _seenStrings->resize(1000); } MemoryConsumption::~MemoryConsumption() { } @@ -15,17 +22,17 @@ uint32_t MemoryConsumption::getStringMemoryUsage(const std::string& s, uint32_t& uniqueCount) { ++_totalStringCount; const char* internalString = s.c_str(); - if (_seenStrings.find(internalString) != _seenStrings.end()) { + if (_seenStrings->find(internalString) != _seenStrings->end()) { return 0; } ++uniqueCount; - _seenStrings.insert(internalString); + _seenStrings->insert(internalString); return s.capacity(); } void MemoryConsumption::addSnapShotUsage(const std::string& name, uint32_t usage) { - _snapShotUsage.push_back(std::pair<std::string, uint32_t>(name, usage)); + _snapShotUsage->push_back(std::pair<std::string, uint32_t>(name, usage)); } uint32_t @@ -80,7 +87,7 @@ MemoryConsumption::print(std::ostream& out, bool verbose, << newl << "Sum metric parent path: " << bval(_sumMetricParentPath) << newl << "Load metric count: " << _loadMetricCount << newl << "Load metric meta: " << bval(_loadMetricMeta) - << newl << "Unique string count: " << _seenStrings.size() + << newl << "Unique string count: " << _seenStrings->size() << newl << "Strings stored: " << _totalStringCount << newl << "Unique consumer ids: " << _consumerIdUnique << newl << "Unique cons metric ids: " << _consumerMetricIdsUnique @@ -91,9 +98,8 @@ MemoryConsumption::print(std::ostream& out, bool verbose, << newl << "Unique metric tags: " << _metricTagsUnique << newl << "Unique sum metric paths: " << _sumMetricParentPathUnique << newl << "Unique name hash strings: " << _nameHashUnique; - for (uint32_t i=0; i<_snapShotUsage.size(); ++i) { - out << newl << "Snapshot " << _snapShotUsage[i].first << ": " - << bval(_snapShotUsage[i].second); + for (const auto & entry : *_snapShotUsage) { + out << newl << "Snapshot " << entry.first << ": " << bval(entry.second); } out << "\n" << indent << ")"; } diff --git a/metrics/src/vespa/metrics/memoryconsumption.h b/metrics/src/vespa/metrics/memoryconsumption.h index 5a8bd56e2c6..e964a7c0ca2 100644 --- a/metrics/src/vespa/metrics/memoryconsumption.h +++ b/metrics/src/vespa/metrics/memoryconsumption.h @@ -18,12 +18,14 @@ #pragma once #include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/stllike/hash_set.h> -#include <sstream> namespace metrics { -struct MemoryConsumption : public vespalib::Printable { +class SeenStrings; +class SnapShotUsage; + +class MemoryConsumption : public vespalib::Printable { +public: typedef std::unique_ptr<MemoryConsumption> UP; uint32_t _consumerCount; @@ -80,9 +82,6 @@ struct MemoryConsumption : public vespalib::Printable { uint32_t _totalStringCount; - vespalib::hash_set<const void*> _seenStrings; - std::vector<std::pair<std::string, uint32_t> > _snapShotUsage; - MemoryConsumption(); ~MemoryConsumption(); @@ -94,6 +93,9 @@ struct MemoryConsumption : public vespalib::Printable { virtual void print(std::ostream& out, bool verbose, const std::string& indent) const; static std::string bval(uint32_t bytes); +private: + std::unique_ptr<SeenStrings> _seenStrings; + std::unique_ptr<SnapShotUsage> _snapShotUsage; }; } // metrics diff --git a/metrics/src/vespa/metrics/namehash.cpp b/metrics/src/vespa/metrics/namehash.cpp new file mode 100644 index 00000000000..29a6f55257b --- /dev/null +++ b/metrics/src/vespa/metrics/namehash.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "namehash.h" +#include <vespa/metrics/memoryconsumption.h> +#include <vespa/vespalib/stllike/hash_set.h> + +namespace metrics { + +struct NameSet : public vespalib::hash_set<std::string> { }; + +NameHash::NameHash() + : _hash(std::make_unique<NameSet>()), + _unifiedCounter(0), + _checkedCounter(0) +{ } + +NameHash::~NameHash() { } + +void +NameHash::updateName(std::string& name) { + ++_checkedCounter; + NameSet::const_iterator it(_hash->find(name)); + if (it != _hash->end()) { + if (name.c_str() != it->c_str()) { + name = *it; + ++_unifiedCounter; + } + } else { + _hash->insert(name); + } +} + +void +NameHash::addMemoryUsage(MemoryConsumption& mc) const { + mc._nameHash += sizeof(NameHash) + + _hash->getMemoryConsumption() + - sizeof(NameSet); + for (const std::string & name : *_hash) { + mc._nameHashStrings += mc.getStringMemoryUsage(name, mc._nameHashUnique); + } +} + +} // metrics diff --git a/metrics/src/vespa/metrics/namehash.h b/metrics/src/vespa/metrics/namehash.h index 52c07b0f75f..a6e73ac47d6 100644 --- a/metrics/src/vespa/metrics/namehash.h +++ b/metrics/src/vespa/metrics/namehash.h @@ -21,46 +21,28 @@ #pragma once #include <vespa/metrics/memoryconsumption.h> -#include <vespa/vespalib/stllike/hash_set.h> namespace metrics { +class NameSet; + class NameHash { - vespalib::hash_set<std::string> _hash; + std::unique_ptr<NameSet> _hash; uint32_t _unifiedCounter; uint32_t _checkedCounter; public: NameHash(const NameHash &) = delete; NameHash & operator = (const NameHash &) = delete; - NameHash() : _unifiedCounter(0), _checkedCounter(0) {} + NameHash(); + ~NameHash(); - void updateName(std::string& name) { - ++_checkedCounter; - vespalib::hash_set<std::string>::const_iterator it(_hash.find(name)); - if (it != _hash.end()) { - if (name.c_str() != it->c_str()) { - name = *it; - ++_unifiedCounter; - } - } else { - _hash.insert(name); - } - } + void updateName(std::string& name); uint32_t getUnifiedStringCount() const { return _unifiedCounter; } uint32_t getCheckedStringCount() const { return _checkedCounter; } void resetCounts() { _unifiedCounter = 0; _checkedCounter = 0; } - void addMemoryUsage(MemoryConsumption& mc) const { - mc._nameHash += sizeof(NameHash) - + _hash.getMemoryConsumption() - - sizeof(vespalib::hash_set<std::string>); - for (const std::string & name : _hash) { - mc._nameHashStrings += mc.getStringMemoryUsage(name, mc._nameHashUnique); - } - } + void addMemoryUsage(MemoryConsumption& mc) const; }; - } // metrics - diff --git a/metrics/src/vespa/metrics/summetric.hpp b/metrics/src/vespa/metrics/summetric.hpp index 84769fffd13..e2421f322d3 100644 --- a/metrics/src/vespa/metrics/summetric.hpp +++ b/metrics/src/vespa/metrics/summetric.hpp @@ -5,6 +5,7 @@ #include "metricset.h" #include "memoryconsumption.h" #include <vespa/vespalib/util/exceptions.h> +#include <ostream> namespace metrics { diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h b/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h index 1a8a993275e..9d7c9ff3b3f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h @@ -6,10 +6,7 @@ #include "documentinverter.h" #include <vespa/document/document.h> #include <vespa/searchlib/queryeval/searchable.h> -#include <string> #include <vector> -#include <map> -#include <set> #include <vespa/vespalib/stllike/hash_set.h> namespace search { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index 8297bc89ae8..a276b5bf38a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -62,7 +62,6 @@ private: IndexPrefix *_legalPrefixes; Set _legalIndexes; - bool IsLegalIndexPrefix(const char *idxName) const { for (const IndexPrefix *pt = _legalPrefixes; |