diff options
author | Geir Storli <geirst@oath.com> | 2018-09-21 09:22:15 +0000 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-09-21 12:02:53 +0000 |
commit | 2657bced445b2144d29bc5872955ef663b10166f (patch) | |
tree | b805104deabab57f7d85d83ebea01d8c221cf495 /searchcore | |
parent | 3c418bab1471ca36c9295309cd671106e5b7f52a (diff) |
Move code that updates document db metrics into separate class.
Diffstat (limited to 'searchcore')
11 files changed, 470 insertions, 360 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 2df34312b52..d47e87e9e03 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -28,6 +28,7 @@ vespa_add_library(searchcore_server STATIC documentbucketmover.cpp documentdb.cpp documentdb_commit_job.cpp + documentdb_metrics_updater.cpp documentdbconfig.cpp documentdbconfigscout.cpp documentdbconfigmanager.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp index e51e968e550..0c3772cab5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp @@ -32,10 +32,7 @@ DocumentDBExplorer::get_state(const Inserter &inserter, bool full) const StateReporterUtils::convertToSlime(*_docDb->reportStatus(), ObjectInserter(object, "status")); } { - // TODO(geirst): Avoid const cast by adding const interface to - // IDocumentMetaStoreContext as seen from IDocumentSubDB. - DocumentMetaStoreReadGuards dmss - (const_cast<DocumentSubDBCollection &>(_docDb->getDocumentSubDBs())); + DocumentMetaStoreReadGuards dmss(_docDb->getDocumentSubDBs()); Cursor &documents = object.setObject("documents"); documents.setLong("active", dmss.numActiveDocs()); documents.setLong("ready", dmss.numReadyDocs()); diff --git a/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.cpp b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.cpp index 2335cd01ec8..952910970ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.cpp @@ -6,7 +6,7 @@ namespace proton { -DocumentMetaStoreReadGuards::DocumentMetaStoreReadGuards(DocumentSubDBCollection &subDBs) +DocumentMetaStoreReadGuards::DocumentMetaStoreReadGuards(const DocumentSubDBCollection &subDBs) : readydms(subDBs.getReadySubDB()->getDocumentMetaStoreContext().getReadGuard()), notreadydms(subDBs.getNotReadySubDB()->getDocumentMetaStoreContext().getReadGuard()), remdms(subDBs.getRemSubDB()->getDocumentMetaStoreContext().getReadGuard()) diff --git a/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h index e58c99ffbec..7baa5576b3b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h @@ -18,7 +18,7 @@ struct DocumentMetaStoreReadGuards IDocumentMetaStoreContext::IReadGuard::UP notreadydms; IDocumentMetaStoreContext::IReadGuard::UP remdms; - DocumentMetaStoreReadGuards(DocumentSubDBCollection &subDBs); + DocumentMetaStoreReadGuards(const DocumentSubDBCollection &subDBs); ~DocumentMetaStoreReadGuards(); uint32_t numActiveDocs() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index e9e974463b6..94ff5b43a3a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -136,8 +136,13 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _visibility(_feedHandler, _writeService, _feedView), _lidSpaceCompactionHandlers(), _jobTrackers(), - _lastDocStoreCacheStats(), - _calc() + _calc(), + _metricsUpdater(_subDBs, + _writeService, + _jobTrackers, + *_sessionManager, + _writeFilter, + _state) { assert(configSnapshot); @@ -1010,356 +1015,13 @@ DocumentDB::notifyAllBucketsChanged() _clusterStateHandler, "notready"); } -namespace { - -void -updateMemoryUsageMetrics(MemoryUsageMetrics &metrics, const MemoryUsage &memoryUsage, MemoryUsage &totalMemoryUsage) -{ - metrics.update(memoryUsage); - totalMemoryUsage.merge(memoryUsage); -} - -void -updateIndexMetrics(DocumentDBMetricsCollection &metrics, const search::SearchableStats &stats, MemoryUsage &totalMemoryUsage) -{ - DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.getTaggedMetrics().index; - indexMetrics.diskUsage.set(stats.sizeOnDisk()); - updateMemoryUsageMetrics(indexMetrics.memoryUsage, stats.memoryUsage(), totalMemoryUsage); - indexMetrics.docsInMemory.set(stats.docsInMemory()); - - LegacyDocumentDBMetrics::IndexMetrics &legacyIndexMetrics = metrics.getLegacyMetrics().index; - legacyIndexMetrics.memoryUsage.set(stats.memoryUsage().allocatedBytes()); - legacyIndexMetrics.docsInMemory.set(stats.docsInMemory()); - legacyIndexMetrics.diskUsage.set(stats.sizeOnDisk()); -} - -struct TempAttributeMetric -{ - MemoryUsage _memoryUsage; - uint64_t _bitVectors; - - TempAttributeMetric() - : _memoryUsage(), - _bitVectors(0) - {} -}; - -struct TempAttributeMetrics -{ - typedef std::map<vespalib::string, TempAttributeMetric> AttrMap; - TempAttributeMetric _total; - AttrMap _attrs; -}; - -bool -isReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &subDbs) -{ - return subDb == subDbs.getReadySubDB(); -} - -bool -isNotReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &subDbs) -{ - return subDb == subDbs.getNotReadySubDB(); -} - -void -fillTempAttributeMetrics(TempAttributeMetrics &metrics, const vespalib::string &attrName, - const MemoryUsage &memoryUsage, uint32_t bitVectors) -{ - metrics._total._memoryUsage.merge(memoryUsage); - metrics._total._bitVectors += bitVectors; - TempAttributeMetric &m = metrics._attrs[attrName]; - m._memoryUsage.merge(memoryUsage); - m._bitVectors += bitVectors; -} - -void -fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics, - TempAttributeMetrics &readyMetrics, - TempAttributeMetrics ¬ReadyMetrics, - const DocumentSubDBCollection &subDbs) -{ - for (const auto subDb : subDbs) { - proton::IAttributeManager::SP attrMgr(subDb->getAttributeManager()); - if (attrMgr) { - TempAttributeMetrics *subMetrics = - (isReadySubDB(subDb, subDbs) ? &readyMetrics : - (isNotReadySubDB(subDb, subDbs) ? ¬ReadyMetrics : nullptr)); - std::vector<search::AttributeGuard> list; - attrMgr->getAttributeListAll(list); - for (const auto &attr : list) { - const search::attribute::Status &status = attr->getStatus(); - MemoryUsage memoryUsage(status.getAllocated(), status.getUsed(), status.getDead(), status.getOnHold()); - uint32_t bitVectors = status.getBitVectors(); - fillTempAttributeMetrics(totalMetrics, attr->getName(), memoryUsage, bitVectors); - if (subMetrics != nullptr) { - fillTempAttributeMetrics(*subMetrics, attr->getName(), memoryUsage, bitVectors); - } - } - } - } -} - -void -updateLegacyAttributeMetrics(LegacyAttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) -{ - for (const auto &attr : tmpMetrics._attrs) { - LegacyAttributeMetrics::List::Entry *entry = metrics.list.get(attr.first); - if (entry) { - entry->memoryUsage.set(attr.second._memoryUsage.allocatedBytes()); - entry->bitVectors.set(attr.second._bitVectors); - } else { - LOG(debug, "Could not update metrics for attribute: '%s'", attr.first.c_str()); - } - } - metrics.memoryUsage.set(tmpMetrics._total._memoryUsage.allocatedBytes()); - metrics.bitVectors.set(tmpMetrics._total._bitVectors); -} - -void -updateAttributeMetrics(AttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) -{ - for (const auto &attr : tmpMetrics._attrs) { - auto entry = metrics.get(attr.first); - if (entry) { - entry->memoryUsage.update(attr.second._memoryUsage); - } - } -} - -void -updateAttributeMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDBCollection &subDbs, MemoryUsage &totalMemoryUsage) -{ - TempAttributeMetrics totalMetrics; - TempAttributeMetrics readyMetrics; - TempAttributeMetrics notReadyMetrics; - fillTempAttributeMetrics(totalMetrics, readyMetrics, notReadyMetrics, subDbs); - - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().attributes, totalMetrics); - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().ready.attributes, readyMetrics); - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().notReady.attributes, notReadyMetrics); - - updateAttributeMetrics(metrics.getTaggedMetrics().ready.attributes, readyMetrics); - updateAttributeMetrics(metrics.getTaggedMetrics().notReady.attributes, notReadyMetrics); - updateMemoryUsageMetrics(metrics.getTaggedMetrics().attribute.totalMemoryUsage, totalMetrics._total._memoryUsage, totalMemoryUsage); -} - -namespace { - -void -updateLegacyRankProfileMetrics(LegacyDocumentDBMetrics::MatchingMetrics &matchingMetrics, - const vespalib::string &rankProfileName, - const MatchingStats &stats) -{ - auto itr = matchingMetrics.rank_profiles.find(rankProfileName); - assert(itr != matchingMetrics.rank_profiles.end()); - itr->second->update(stats); -} - -} - -void -updateMatchingMetrics(DocumentDBMetricsCollection &metrics, const IDocumentSubDB &ready) -{ - MatchingStats totalStats; - for (const auto &rankProfile : metrics.getTaggedMetrics().matching.rank_profiles) { - MatchingStats matchingStats = ready.getMatcherStats(rankProfile.first); - rankProfile.second->update(matchingStats); - updateLegacyRankProfileMetrics(metrics.getLegacyMetrics().matching, rankProfile.first, matchingStats); - - totalStats.add(matchingStats); - } - metrics.getTaggedMetrics().matching.update(totalStats); - metrics.getLegacyMetrics().matching.update(totalStats); -} - -void -updateSessionCacheMetrics(DocumentDBMetricsCollection &metrics, proton::matching::SessionManager &sessionManager) -{ - auto searchStats = sessionManager.getSearchStats(); - metrics.getTaggedMetrics().sessionCache.search.update(searchStats); - - auto groupingStats = sessionManager.getGroupingStats(); - metrics.getTaggedMetrics().sessionCache.grouping.update(groupingStats); - metrics.getLegacyMetrics().sessionManager.update(groupingStats); -} - -void -updateDocumentsMetrics(DocumentDBMetricsCollection &metrics, DocumentSubDBCollection &subDbs) -{ - DocumentMetaStoreReadGuards dms(subDbs); - uint32_t active = dms.numActiveDocs(); - uint32_t ready = dms.numReadyDocs(); - uint32_t total = dms.numTotalDocs(); - uint32_t removed = dms.numRemovedDocs(); - - auto &docsMetrics = metrics.getTaggedMetrics().documents; - docsMetrics.active.set(active); - docsMetrics.ready.set(ready); - docsMetrics.total.set(total); - docsMetrics.removed.set(removed); - - auto &legacyMetrics = metrics.getLegacyMetrics(); - legacyMetrics.numActiveDocs.set(active); - legacyMetrics.numIndexedDocs.set(ready); - legacyMetrics.numStoredDocs.set(total); - legacyMetrics.numRemovedDocs.set(removed); -} - -void -updateDocumentStoreCacheHitRate(const CacheStats ¤t, const CacheStats &last, - metrics::LongAverageMetric &cacheHitRate) -{ - if (current.lookups() < last.lookups() || current.hits < last.hits) { - LOG(warning, "Not adding document store cache hit rate metrics as values calculated " - "are corrupt. current.lookups=%" PRIu64 ", last.lookups=%" PRIu64 ", current.hits=%" PRIu64 ", last.hits=%" PRIu64 ".", - current.lookups(), last.lookups(), current.hits, last.hits); - } else { - if ((current.lookups() - last.lookups()) > 0xffffffffull - || (current.hits - last.hits) > 0xffffffffull) - { - LOG(warning, "Document store cache hit rate metrics to add are suspiciously high." - " lookups diff=%" PRIu64 ", hits diff=%" PRIu64 ".", - current.lookups() - last.lookups(), current.hits - last.hits); - } - cacheHitRate.addTotalValueWithCount(current.hits - last.hits, current.lookups() - last.lookups()); - } -} - -void -updateCountMetric(uint64_t currVal, uint64_t lastVal, metrics::LongCountMetric &metric) -{ - uint64_t delta = (currVal >= lastVal) ? (currVal - lastVal) : 0; - metric.inc(delta); -} - -void -updateDocstoreMetrics(LegacyDocumentDBMetrics::DocstoreMetrics &metrics, - const DocumentSubDBCollection &sub_dbs, - CacheStats &lastCacheStats) -{ - size_t memoryUsage = 0; - CacheStats cache_stats; - for (const auto subDb : sub_dbs) { - const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager(); - if (summaryMgr) { - cache_stats += summaryMgr->getBackingStore().getCacheStats(); - memoryUsage += summaryMgr->getBackingStore().memoryUsed(); - } - } - metrics.memoryUsage.set(memoryUsage); - updateCountMetric(cache_stats.lookups(), lastCacheStats.lookups(), metrics.cacheLookups); - updateDocumentStoreCacheHitRate(cache_stats, lastCacheStats, metrics.cacheHitRate); - metrics.cacheElements.set(cache_stats.elements); - metrics.cacheMemoryUsed.set(cache_stats.memory_used); - lastCacheStats = cache_stats; -} - -void -updateDocumentStoreMetrics(DocumentDBTaggedMetrics::SubDBMetrics::DocumentStoreMetrics &metrics, - IDocumentSubDB *subDb, - CacheStats &lastCacheStats, - MemoryUsage &totalMemoryUsage) -{ - const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager(); - search::IDocumentStore &backingStore = summaryMgr->getBackingStore(); - search::DataStoreStorageStats storageStats(backingStore.getStorageStats()); - metrics.diskUsage.set(storageStats.diskUsage()); - metrics.diskBloat.set(storageStats.diskBloat()); - metrics.maxBucketSpread.set(storageStats.maxBucketSpread()); - updateMemoryUsageMetrics(metrics.memoryUsage, backingStore.getMemoryUsage(), totalMemoryUsage); - - search::CacheStats cacheStats = backingStore.getCacheStats(); - totalMemoryUsage.incAllocatedBytes(cacheStats.memory_used); - metrics.cache.memoryUsage.set(cacheStats.memory_used); - metrics.cache.elements.set(cacheStats.elements); - updateDocumentStoreCacheHitRate(cacheStats, lastCacheStats, metrics.cache.hitRate); - updateCountMetric(cacheStats.lookups(), lastCacheStats.lookups(), metrics.cache.lookups); - updateCountMetric(cacheStats.invalidations, lastCacheStats.invalidations, metrics.cache.invalidations); - lastCacheStats = cacheStats; -} - -template <typename MetricSetType> -void -updateLidSpaceMetrics(MetricSetType &metrics, const search::IDocumentMetaStore &metaStore) -{ - LidUsageStats stats = metaStore.getLidUsageStats(); - metrics.lidLimit.set(stats.getLidLimit()); - metrics.usedLids.set(stats.getUsedLids()); - metrics.lowestFreeLid.set(stats.getLowestFreeLid()); - metrics.highestUsedLid.set(stats.getHighestUsedLid()); - metrics.lidBloatFactor.set(stats.getLidBloatFactor()); - metrics.lidFragmentationFactor.set(stats.getLidFragmentationFactor()); -} - -} // namespace - void DocumentDB::updateMetrics(DocumentDBMetricsCollection &metrics) { if (_state.getState() < DDBState::State::REPLAY_TRANSACTION_LOG) { return; } - - MemoryUsage totalMemoryUsage; - ExecutorThreadingServiceStats threadingServiceStats = _writeService.getStats(); - updateLegacyMetrics(metrics.getLegacyMetrics(), threadingServiceStats); - updateIndexMetrics(metrics, _subDBs.getReadySubDB()->getSearchableStats(), totalMemoryUsage); - updateAttributeMetrics(metrics, _subDBs, totalMemoryUsage); - updateMatchingMetrics(metrics, *_subDBs.getReadySubDB()); - updateSessionCacheMetrics(metrics, *_sessionManager); - updateDocumentsMetrics(metrics, _subDBs); - updateMiscMetrics(metrics.getTaggedMetrics(), threadingServiceStats, totalMemoryUsage); - metrics.getTaggedMetrics().totalMemoryUsage.update(totalMemoryUsage); -} - -void -DocumentDB::updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats) -{ - metrics.executor.update(threadingServiceStats.getMasterExecutorStats()); - metrics.summaryExecutor.update(threadingServiceStats.getSummaryExecutorStats()); - metrics.indexExecutor.update(threadingServiceStats.getIndexExecutorStats()); - updateDocstoreMetrics(metrics.docstore, _subDBs, _lastDocStoreCacheStats.total); - metrics.numDocs.set(getNumDocs()); - - DocumentMetaStoreReadGuards dmss(_subDBs); - updateLidSpaceMetrics(metrics.ready.docMetaStore, dmss.readydms->get()); - updateLidSpaceMetrics(metrics.notReady.docMetaStore, dmss.notreadydms->get()); - updateLidSpaceMetrics(metrics.removed.docMetaStore, dmss.remdms->get()); - - metrics.numBadConfigs.set(_state.getDelayedConfig() ? 1u : 0u); -} - -void -DocumentDB:: -updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics) -{ - AttributeUsageFilter &writeFilter(_writeFilter); - AttributeUsageStats attributeUsageStats = writeFilter.getAttributeUsageStats(); - bool feedBlocked = !writeFilter.acceptWriteOperation(); - double enumStoreUsed = attributeUsageStats.enumStoreUsage().getUsage().usage(); - double multiValueUsed = attributeUsageStats.multiValueUsage().getUsage().usage(); - metrics.resourceUsage.enumStore.set(enumStoreUsed); - metrics.resourceUsage.multiValue.set(multiValueUsed); - metrics.resourceUsage.feedingBlocked.set(feedBlocked ? 1 : 0); -} - -void -DocumentDB::updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats, MemoryUsage &totalMemoryUsage) -{ - metrics.threadingService.update(threadingServiceStats); - _jobTrackers.updateMetrics(metrics.job); - - updateAttributeResourceUsageMetrics(metrics.attribute); - updateDocumentStoreMetrics(metrics.ready.documentStore, _subDBs.getReadySubDB(), _lastDocStoreCacheStats.readySubDb, totalMemoryUsage); - updateDocumentStoreMetrics(metrics.removed.documentStore, _subDBs.getRemSubDB(), _lastDocStoreCacheStats.removedSubDb, totalMemoryUsage); - updateDocumentStoreMetrics(metrics.notReady.documentStore, _subDBs.getNotReadySubDB(), _lastDocStoreCacheStats.notReadySubDb, totalMemoryUsage); - - DocumentMetaStoreReadGuards dmss(_subDBs); - updateLidSpaceMetrics(metrics.ready.lidSpace, dmss.readydms->get()); - updateLidSpaceMetrics(metrics.notReady.lidSpace, dmss.notreadydms->get()); - updateLidSpaceMetrics(metrics.removed.lidSpace, dmss.remdms->get()); + _metricsUpdater.updateMetrics(metrics); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 66dc47e78c0..fb23b113f3b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -6,11 +6,12 @@ #include "configstore.h" #include "ddbstate.h" #include "disk_mem_usage_forwarder.h" +#include "documentdb_metrics_updater.h" +#include "document_db_config_owner.h" #include "documentdbconfig.h" #include "documentsubdbcollection.h" #include "executorthreadingservice.h" #include "feedhandler.h" -#include "document_db_config_owner.h" #include "i_document_subdb_owner.h" #include "i_feed_handler_owner.h" #include "i_lid_space_compaction_handler.h" @@ -135,10 +136,8 @@ private: VisibilityHandler _visibility; ILidSpaceCompactionHandler::Vector _lidSpaceCompactionHandlers; DocumentDBJobTrackers _jobTrackers; - - // Last updated document store cache statistics. Necessary due to metrics implementation is upside down. - DocumentStoreCacheStats _lastDocStoreCacheStats; IBucketStateCalculator::SP _calc; + DocumentDBMetricsUpdater _metricsUpdater; void registerReference(); void setActiveConfig(const DocumentDBConfig::SP &config, SerialNum serialNum, int64_t generation); @@ -206,10 +205,6 @@ private: virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; void notifyAllBucketsChanged(); - void updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats); - void updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats, search::MemoryUsage &totalMemoryUsage); - void updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics); - /* * Tear down references to this document db (e.g. listeners for * gid to lid changes) from other document dbs. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp new file mode 100644 index 00000000000..684775172aa --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -0,0 +1,393 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ddbstate.h" +#include "document_meta_store_read_guards.h" +#include "documentdb_metrics_updater.h" +#include "documentsubdbcollection.h" +#include "executorthreadingservice.h" +#include "idocumentsubdb.h" +#include <vespa/searchcommon/attribute/status.h> +#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> +#include <vespa/searchcore/proton/attribute/i_attribute_manager.h> +#include <vespa/searchcore/proton/docsummary/isummarymanager.h> +#include <vespa/searchcore/proton/matching/matching_stats.h> +#include <vespa/searchcore/proton/matching/matching_stats.h> +#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> +#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> +#include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/docstore/cachestats.h> +#include <vespa/searchlib/util/memoryusage.h> +#include <vespa/searchlib/util/searchable_stats.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.documentdb_metrics_updater"); + +using search::LidUsageStats; +using search::CacheStats; +using search::MemoryUsage; + +namespace proton { + +using matching::MatchingStats; + +DocumentDBMetricsUpdater::DocumentDBMetricsUpdater(const DocumentSubDBCollection &subDBs, + ExecutorThreadingService &writeService, + DocumentDBJobTrackers &jobTrackers, + matching::SessionManager &sessionManager, + const AttributeUsageFilter &writeFilter, + const DDBState &state) + : _subDBs(subDBs), + _writeService(writeService), + _jobTrackers(jobTrackers), + _sessionManager(sessionManager), + _writeFilter(writeFilter), + _state(state) +{ +} + +DocumentDBMetricsUpdater::~DocumentDBMetricsUpdater() = default; + +namespace { + +void +updateMemoryUsageMetrics(MemoryUsageMetrics &metrics, const MemoryUsage &memoryUsage, MemoryUsage &totalMemoryUsage) +{ + metrics.update(memoryUsage); + totalMemoryUsage.merge(memoryUsage); +} + +void +updateIndexMetrics(DocumentDBMetricsCollection &metrics, const search::SearchableStats &stats, MemoryUsage &totalMemoryUsage) +{ + DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.getTaggedMetrics().index; + indexMetrics.diskUsage.set(stats.sizeOnDisk()); + updateMemoryUsageMetrics(indexMetrics.memoryUsage, stats.memoryUsage(), totalMemoryUsage); + indexMetrics.docsInMemory.set(stats.docsInMemory()); + + LegacyDocumentDBMetrics::IndexMetrics &legacyIndexMetrics = metrics.getLegacyMetrics().index; + legacyIndexMetrics.memoryUsage.set(stats.memoryUsage().allocatedBytes()); + legacyIndexMetrics.docsInMemory.set(stats.docsInMemory()); + legacyIndexMetrics.diskUsage.set(stats.sizeOnDisk()); +} + +struct TempAttributeMetric +{ + MemoryUsage memoryUsage; + uint64_t bitVectors; + + TempAttributeMetric() + : memoryUsage(), + bitVectors(0) + {} +}; + +struct TempAttributeMetrics +{ + typedef std::map<vespalib::string, TempAttributeMetric> AttrMap; + TempAttributeMetric total; + AttrMap attrs; +}; + +bool +isReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &subDbs) +{ + return subDb == subDbs.getReadySubDB(); +} + +bool +isNotReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &subDbs) +{ + return subDb == subDbs.getNotReadySubDB(); +} + +void +fillTempAttributeMetrics(TempAttributeMetrics &metrics, const vespalib::string &attrName, + const MemoryUsage &memoryUsage, uint32_t bitVectors) +{ + metrics.total.memoryUsage.merge(memoryUsage); + metrics.total.bitVectors += bitVectors; + TempAttributeMetric &m = metrics.attrs[attrName]; + m.memoryUsage.merge(memoryUsage); + m.bitVectors += bitVectors; +} + +void +fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics, + TempAttributeMetrics &readyMetrics, + TempAttributeMetrics ¬ReadyMetrics, + const DocumentSubDBCollection &subDbs) +{ + for (const auto subDb : subDbs) { + proton::IAttributeManager::SP attrMgr(subDb->getAttributeManager()); + if (attrMgr) { + TempAttributeMetrics *subMetrics = + (isReadySubDB(subDb, subDbs) ? &readyMetrics : + (isNotReadySubDB(subDb, subDbs) ? ¬ReadyMetrics : nullptr)); + std::vector<search::AttributeGuard> list; + attrMgr->getAttributeListAll(list); + for (const auto &attr : list) { + const search::attribute::Status &status = attr->getStatus(); + MemoryUsage memoryUsage(status.getAllocated(), status.getUsed(), status.getDead(), status.getOnHold()); + uint32_t bitVectors = status.getBitVectors(); + fillTempAttributeMetrics(totalMetrics, attr->getName(), memoryUsage, bitVectors); + if (subMetrics != nullptr) { + fillTempAttributeMetrics(*subMetrics, attr->getName(), memoryUsage, bitVectors); + } + } + } + } +} + +void +updateLegacyAttributeMetrics(LegacyAttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) +{ + for (const auto &attr : tmpMetrics.attrs) { + LegacyAttributeMetrics::List::Entry *entry = metrics.list.get(attr.first); + if (entry) { + entry->memoryUsage.set(attr.second.memoryUsage.allocatedBytes()); + entry->bitVectors.set(attr.second.bitVectors); + } else { + LOG(debug, "Could not update metrics for attribute: '%s'", attr.first.c_str()); + } + } + metrics.memoryUsage.set(tmpMetrics.total.memoryUsage.allocatedBytes()); + metrics.bitVectors.set(tmpMetrics.total.bitVectors); +} + +void +updateAttributeMetrics(AttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) +{ + for (const auto &attr : tmpMetrics.attrs) { + auto entry = metrics.get(attr.first); + if (entry) { + entry->memoryUsage.update(attr.second.memoryUsage); + } + } +} + +void +updateAttributeMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDBCollection &subDbs, MemoryUsage &totalMemoryUsage) +{ + TempAttributeMetrics totalMetrics; + TempAttributeMetrics readyMetrics; + TempAttributeMetrics notReadyMetrics; + fillTempAttributeMetrics(totalMetrics, readyMetrics, notReadyMetrics, subDbs); + + updateLegacyAttributeMetrics(metrics.getLegacyMetrics().attributes, totalMetrics); + updateLegacyAttributeMetrics(metrics.getLegacyMetrics().ready.attributes, readyMetrics); + updateLegacyAttributeMetrics(metrics.getLegacyMetrics().notReady.attributes, notReadyMetrics); + + updateAttributeMetrics(metrics.getTaggedMetrics().ready.attributes, readyMetrics); + updateAttributeMetrics(metrics.getTaggedMetrics().notReady.attributes, notReadyMetrics); + updateMemoryUsageMetrics(metrics.getTaggedMetrics().attribute.totalMemoryUsage, totalMetrics.total.memoryUsage, totalMemoryUsage); +} + +void +updateLegacyRankProfileMetrics(LegacyDocumentDBMetrics::MatchingMetrics &matchingMetrics, + const vespalib::string &rankProfileName, + const MatchingStats &stats) +{ + auto itr = matchingMetrics.rank_profiles.find(rankProfileName); + assert(itr != matchingMetrics.rank_profiles.end()); + itr->second->update(stats); +} + +void +updateMatchingMetrics(DocumentDBMetricsCollection &metrics, const IDocumentSubDB &ready) +{ + MatchingStats totalStats; + for (const auto &rankProfile : metrics.getTaggedMetrics().matching.rank_profiles) { + MatchingStats matchingStats = ready.getMatcherStats(rankProfile.first); + rankProfile.second->update(matchingStats); + updateLegacyRankProfileMetrics(metrics.getLegacyMetrics().matching, rankProfile.first, matchingStats); + + totalStats.add(matchingStats); + } + metrics.getTaggedMetrics().matching.update(totalStats); + metrics.getLegacyMetrics().matching.update(totalStats); +} + +void +updateSessionCacheMetrics(DocumentDBMetricsCollection &metrics, proton::matching::SessionManager &sessionManager) +{ + auto searchStats = sessionManager.getSearchStats(); + metrics.getTaggedMetrics().sessionCache.search.update(searchStats); + + auto groupingStats = sessionManager.getGroupingStats(); + metrics.getTaggedMetrics().sessionCache.grouping.update(groupingStats); + metrics.getLegacyMetrics().sessionManager.update(groupingStats); +} + +void +updateDocumentsMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDBCollection &subDbs) +{ + DocumentMetaStoreReadGuards dms(subDbs); + uint32_t active = dms.numActiveDocs(); + uint32_t ready = dms.numReadyDocs(); + uint32_t total = dms.numTotalDocs(); + uint32_t removed = dms.numRemovedDocs(); + + auto &docsMetrics = metrics.getTaggedMetrics().documents; + docsMetrics.active.set(active); + docsMetrics.ready.set(ready); + docsMetrics.total.set(total); + docsMetrics.removed.set(removed); + + auto &legacyMetrics = metrics.getLegacyMetrics(); + legacyMetrics.numDocs.set(ready); + legacyMetrics.numActiveDocs.set(active); + legacyMetrics.numIndexedDocs.set(ready); + legacyMetrics.numStoredDocs.set(total); + legacyMetrics.numRemovedDocs.set(removed); +} + +void +updateDocumentStoreCacheHitRate(const CacheStats ¤t, const CacheStats &last, + metrics::LongAverageMetric &cacheHitRate) +{ + if (current.lookups() < last.lookups() || current.hits < last.hits) { + LOG(warning, "Not adding document store cache hit rate metrics as values calculated " + "are corrupt. current.lookups=%" PRIu64 ", last.lookups=%" PRIu64 ", current.hits=%" PRIu64 ", last.hits=%" PRIu64 ".", + current.lookups(), last.lookups(), current.hits, last.hits); + } else { + if ((current.lookups() - last.lookups()) > 0xffffffffull + || (current.hits - last.hits) > 0xffffffffull) + { + LOG(warning, "Document store cache hit rate metrics to add are suspiciously high." + " lookups diff=%" PRIu64 ", hits diff=%" PRIu64 ".", + current.lookups() - last.lookups(), current.hits - last.hits); + } + cacheHitRate.addTotalValueWithCount(current.hits - last.hits, current.lookups() - last.lookups()); + } +} + +void +updateCountMetric(uint64_t currVal, uint64_t lastVal, metrics::LongCountMetric &metric) +{ + uint64_t delta = (currVal >= lastVal) ? (currVal - lastVal) : 0; + metric.inc(delta); +} + +void +updateDocstoreMetrics(LegacyDocumentDBMetrics::DocstoreMetrics &metrics, + const DocumentSubDBCollection &sub_dbs, + CacheStats &lastCacheStats) +{ + size_t memoryUsage = 0; + CacheStats cache_stats; + for (const auto subDb : sub_dbs) { + const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager(); + if (summaryMgr) { + cache_stats += summaryMgr->getBackingStore().getCacheStats(); + memoryUsage += summaryMgr->getBackingStore().memoryUsed(); + } + } + metrics.memoryUsage.set(memoryUsage); + updateCountMetric(cache_stats.lookups(), lastCacheStats.lookups(), metrics.cacheLookups); + updateDocumentStoreCacheHitRate(cache_stats, lastCacheStats, metrics.cacheHitRate); + metrics.cacheElements.set(cache_stats.elements); + metrics.cacheMemoryUsed.set(cache_stats.memory_used); + lastCacheStats = cache_stats; +} + +void +updateDocumentStoreMetrics(DocumentDBTaggedMetrics::SubDBMetrics::DocumentStoreMetrics &metrics, + const IDocumentSubDB *subDb, + CacheStats &lastCacheStats, + MemoryUsage &totalMemoryUsage) +{ + const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager(); + search::IDocumentStore &backingStore = summaryMgr->getBackingStore(); + search::DataStoreStorageStats storageStats(backingStore.getStorageStats()); + metrics.diskUsage.set(storageStats.diskUsage()); + metrics.diskBloat.set(storageStats.diskBloat()); + metrics.maxBucketSpread.set(storageStats.maxBucketSpread()); + updateMemoryUsageMetrics(metrics.memoryUsage, backingStore.getMemoryUsage(), totalMemoryUsage); + + search::CacheStats cacheStats = backingStore.getCacheStats(); + totalMemoryUsage.incAllocatedBytes(cacheStats.memory_used); + metrics.cache.memoryUsage.set(cacheStats.memory_used); + metrics.cache.elements.set(cacheStats.elements); + updateDocumentStoreCacheHitRate(cacheStats, lastCacheStats, metrics.cache.hitRate); + updateCountMetric(cacheStats.lookups(), lastCacheStats.lookups(), metrics.cache.lookups); + updateCountMetric(cacheStats.invalidations, lastCacheStats.invalidations, metrics.cache.invalidations); + lastCacheStats = cacheStats; +} + +template <typename MetricSetType> +void +updateLidSpaceMetrics(MetricSetType &metrics, const search::IDocumentMetaStore &metaStore) +{ + LidUsageStats stats = metaStore.getLidUsageStats(); + metrics.lidLimit.set(stats.getLidLimit()); + metrics.usedLids.set(stats.getUsedLids()); + metrics.lowestFreeLid.set(stats.getLowestFreeLid()); + metrics.highestUsedLid.set(stats.getHighestUsedLid()); + metrics.lidBloatFactor.set(stats.getLidBloatFactor()); + metrics.lidFragmentationFactor.set(stats.getLidFragmentationFactor()); +} + +} + +void +DocumentDBMetricsUpdater::updateMetrics(DocumentDBMetricsCollection &metrics) +{ + MemoryUsage totalMemoryUsage; + ExecutorThreadingServiceStats threadingServiceStats = _writeService.getStats(); + updateLegacyMetrics(metrics.getLegacyMetrics(), threadingServiceStats); + updateIndexMetrics(metrics, _subDBs.getReadySubDB()->getSearchableStats(), totalMemoryUsage); + updateAttributeMetrics(metrics, _subDBs, totalMemoryUsage); + updateMatchingMetrics(metrics, *_subDBs.getReadySubDB()); + updateSessionCacheMetrics(metrics, _sessionManager); + updateDocumentsMetrics(metrics, _subDBs); + updateMiscMetrics(metrics.getTaggedMetrics(), threadingServiceStats, totalMemoryUsage); + metrics.getTaggedMetrics().totalMemoryUsage.update(totalMemoryUsage); +} + +void +DocumentDBMetricsUpdater::updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats) +{ + metrics.executor.update(threadingServiceStats.getMasterExecutorStats()); + metrics.summaryExecutor.update(threadingServiceStats.getSummaryExecutorStats()); + metrics.indexExecutor.update(threadingServiceStats.getIndexExecutorStats()); + updateDocstoreMetrics(metrics.docstore, _subDBs, _lastDocStoreCacheStats.total); + + DocumentMetaStoreReadGuards dmss(_subDBs); + updateLidSpaceMetrics(metrics.ready.docMetaStore, dmss.readydms->get()); + updateLidSpaceMetrics(metrics.notReady.docMetaStore, dmss.notreadydms->get()); + updateLidSpaceMetrics(metrics.removed.docMetaStore, dmss.remdms->get()); + + metrics.numBadConfigs.set(_state.getDelayedConfig() ? 1u : 0u); +} + +void +DocumentDBMetricsUpdater::updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics) +{ + AttributeUsageStats attributeUsageStats = _writeFilter.getAttributeUsageStats(); + bool feedBlocked = !_writeFilter.acceptWriteOperation(); + double enumStoreUsed = attributeUsageStats.enumStoreUsage().getUsage().usage(); + double multiValueUsed = attributeUsageStats.multiValueUsage().getUsage().usage(); + metrics.resourceUsage.enumStore.set(enumStoreUsed); + metrics.resourceUsage.multiValue.set(multiValueUsed); + metrics.resourceUsage.feedingBlocked.set(feedBlocked ? 1 : 0); +} + +void +DocumentDBMetricsUpdater::updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats, MemoryUsage &totalMemoryUsage) +{ + metrics.threadingService.update(threadingServiceStats); + _jobTrackers.updateMetrics(metrics.job); + + updateAttributeResourceUsageMetrics(metrics.attribute); + updateDocumentStoreMetrics(metrics.ready.documentStore, _subDBs.getReadySubDB(), _lastDocStoreCacheStats.readySubDb, totalMemoryUsage); + updateDocumentStoreMetrics(metrics.removed.documentStore, _subDBs.getRemSubDB(), _lastDocStoreCacheStats.removedSubDb, totalMemoryUsage); + updateDocumentStoreMetrics(metrics.notReady.documentStore, _subDBs.getNotReadySubDB(), _lastDocStoreCacheStats.notReadySubDb, totalMemoryUsage); + + DocumentMetaStoreReadGuards dmss(_subDBs); + updateLidSpaceMetrics(metrics.ready.lidSpace, dmss.readydms->get()); + updateLidSpaceMetrics(metrics.notReady.lidSpace, dmss.notreadydms->get()); + updateLidSpaceMetrics(metrics.removed.lidSpace, dmss.remdms->get()); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h new file mode 100644 index 00000000000..755d429d553 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h @@ -0,0 +1,59 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> +#include <vespa/searchlib/docstore/cachestats.h> + +namespace proton { + +namespace matching { class SessionManager; } + +class AttributeUsageFilter; +class DDBState; +class DocumentDBJobTrackers; +class DocumentDBMetricsCollection; +class DocumentSubDBCollection; +class ExecutorThreadingService; +class ExecutorThreadingServiceStats; +class LegacyDocumentDBMetrics; + +/** + * Class used to update metrics for a document db. + */ +class DocumentDBMetricsUpdater { +private: + struct DocumentStoreCacheStats { + search::CacheStats total; + search::CacheStats readySubDb; + search::CacheStats notReadySubDb; + search::CacheStats removedSubDb; + DocumentStoreCacheStats() : total(), readySubDb(), notReadySubDb(), removedSubDb() {} + }; + + const DocumentSubDBCollection &_subDBs; + ExecutorThreadingService &_writeService; + DocumentDBJobTrackers &_jobTrackers; + matching::SessionManager &_sessionManager; + const AttributeUsageFilter &_writeFilter; + const DDBState &_state; + // Last updated document store cache statistics. Necessary due to metrics implementation is upside down. + DocumentStoreCacheStats _lastDocStoreCacheStats; + + void updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats); + void updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats, search::MemoryUsage &totalMemoryUsage); + void updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics); + +public: + DocumentDBMetricsUpdater(const DocumentSubDBCollection &subDBs, + ExecutorThreadingService &writeService, + DocumentDBJobTrackers &jobTrackers, + matching::SessionManager &sessionManager, + const AttributeUsageFilter &writeFilter, + const DDBState &state); + ~DocumentDBMetricsUpdater(); + + void updateMetrics(DocumentDBMetricsCollection &metrics); + +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index d41c2088518..736edcf3c5e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -84,6 +84,7 @@ public: virtual const std::shared_ptr<ISummaryAdapter> &getSummaryAdapter() const = 0; virtual const std::shared_ptr<IIndexWriter> &getIndexWriter() const = 0; virtual IDocumentMetaStoreContext &getDocumentMetaStoreContext() = 0; + virtual const IDocumentMetaStoreContext &getDocumentMetaStoreContext() const = 0; virtual IFlushTargetList getFlushTargets() = 0; virtual size_t getNumDocs() const = 0; virtual size_t getNumActiveDocs() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 4a21537f134..a54525377a9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -228,6 +228,7 @@ public: const ISummaryAdapter::SP & getSummaryAdapter() const override { return _summaryAdapter; } const std::shared_ptr<IIndexWriter> & getIndexWriter() const override; IDocumentMetaStoreContext & getDocumentMetaStoreContext() override { return *_metaStoreCtx; } + const IDocumentMetaStoreContext &getDocumentMetaStoreContext() const override { return *_metaStoreCtx; } size_t getNumDocs() const override; size_t getNumActiveDocs() const override; bool hasDocument(const document::DocumentId &id) override; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 66ae296566f..c2595a85fb7 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -67,6 +67,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; } const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; } IDocumentMetaStoreContext &getDocumentMetaStoreContext() override { return _metaStoreCtx; } + const IDocumentMetaStoreContext &getDocumentMetaStoreContext() const override { return _metaStoreCtx; } IFlushTargetList getFlushTargets() override { return IFlushTargetList(); } size_t getNumDocs() const override { return 0; } size_t getNumActiveDocs() const override { return 0; } |