diff options
Diffstat (limited to 'searchcore/src')
23 files changed, 320 insertions, 18 deletions
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 1f0f3566b1d..6eaa1bd373a 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -3,11 +3,14 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/fastos/thread.h> +#include <vespa/config-attributes.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> #include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> @@ -60,6 +63,7 @@ using storage::spi::Timestamp; using vespalib::Slime; using vespalib::makeClosure; using vespalib::makeTask; +using vespa::config::search::AttributesConfigBuilder; using BlockedReason = IBlockableMaintenanceJob::BlockedReason; @@ -885,6 +889,8 @@ MaintenanceControllerFixture::injectMaintenanceJobs() _jobTrackers, *this, _readyAttributeManager, _notReadyAttributeManager, + std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()), + std::make_shared<TransientMemoryUsageProvider>(), _attributeUsageFilter); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 550d0a1c4c4..e0b0cd9f0fd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_aspect_delayer.cpp attribute_collection_spec_factory.cpp attribute_collection_spec.cpp + attribute_config_inspector.cpp attribute_directory.cpp attribute_factory.cpp attribute_initializer.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp new file mode 100644 index 00000000000..d950b5e8ed3 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_config_inspector.h" +#include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/config-attributes.h> +#include <vespa/vespalib/stllike/hash_map.hpp> + +using search::attribute::ConfigConverter; + +namespace proton { + +AttributeConfigInspector::AttributeConfigInspector(const AttributesConfig& config) + : _hash() +{ + for (auto& attr : config.attribute) { + auto res = _hash.insert(std::make_pair(attr.name, ConfigConverter::convert(attr))); + assert(res.second); + } +} + +AttributeConfigInspector::~AttributeConfigInspector() = default; + +const search::attribute::Config* +AttributeConfigInspector::get_config(const vespalib::string& name) const +{ + auto itr = _hash.find(name); + return (itr != _hash.end()) ? &itr->second : nullptr; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h new file mode 100644 index 00000000000..8952b34c6ff --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/searchcommon/attribute/config.h> + +namespace vespa::config::search::internal { class InternalAttributesType; } + +namespace proton { + +/* + * Class used to find attribute config given attribute name based on config + * from config server. + */ +class AttributeConfigInspector { + vespalib::hash_map<vespalib::string, search::attribute::Config> _hash; +public: + using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; + AttributeConfigInspector(const AttributesConfig& config); + ~AttributeConfigInspector(); + const search::attribute::Config* get_config(const vespalib::string& name) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index 39235fb8d00..43c9e52315b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -2,28 +2,35 @@ #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> namespace proton { -AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter) +AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) : _usage(), + _transient_memory_usage(0u), _lock(), - _filter(filter) + _filter(filter), + _attribute_config_inspector(std::move(attribute_config_inspector)), + _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) { } AttributeUsageSamplerContext::~AttributeUsageSamplerContext() { _filter.setAttributeStats(_usage); + _transient_memory_usage_provider->set_transient_memory_usage(_transient_memory_usage); } void AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage, + size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { Guard guard(_lock); _usage.merge(usage, attributeName, subDbName); + _transient_memory_usage = std::max(_transient_memory_usage, transient_memory_usage); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h index 21e4ff6a90f..b5693aef1bc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h @@ -8,6 +8,8 @@ namespace proton { class AttributeUsageFilter; +class AttributeConfigInspector; +class TransientMemoryUsageProvider; /* * Context for sampling attribute usage stats. When instance is @@ -19,16 +21,22 @@ class AttributeUsageSamplerContext using Guard = std::lock_guard<Mutex>; AttributeUsageStats _usage; + size_t _transient_memory_usage; Mutex _lock; AttributeUsageFilter &_filter; + std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; public: - AttributeUsageSamplerContext(AttributeUsageFilter &filter); + AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); ~AttributeUsageSamplerContext(); void merge(const search::AddressSpaceUsage &usage, + size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName); + const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; } const AttributeUsageStats & getUsage() const { return _usage; } + size_t get_transient_memory_usage() const { return _transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index a1a8409e93b..f8594a4477b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -2,10 +2,49 @@ #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" +#include "attribute_config_inspector.h" #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/loadedenumvalue.h> +#include <vespa/searchlib/attribute/loadedvalue.h> + +using search::attribute::BasicType; namespace proton { +namespace { + +size_t +get_transient_memory_usage(const search::attribute::Config& old_config, + const search::attribute::Config& current_config, + uint64_t total_value_count) +{ + if (current_config.fastSearch()) { + if (old_config.fastSearch()) { + return sizeof(search::attribute::LoadedEnumAttribute) * total_value_count; + } else { + switch (old_config.basicType().type()) { + case BasicType::Type::INT8: + return sizeof(search::attribute::LoadedValue<int8_t>) * total_value_count; + case BasicType::Type::INT16: + return sizeof(search::attribute::LoadedValue<int16_t>) * total_value_count; + case BasicType::Type::INT32: + return sizeof(search::attribute::LoadedValue<int32_t>) * total_value_count; + case BasicType::Type::INT64: + return sizeof(search::attribute::LoadedValue<int64_t>) * total_value_count; + case BasicType::Type::FLOAT: + return sizeof(search::attribute::LoadedValue<float>) * total_value_count; + case BasicType::Type::DOUBLE: + return sizeof(search::attribute::LoadedValue<double>) * total_value_count; + default: + ; + } + } + } + return 0u; +} + +} + AttributeUsageSamplerFunctor::AttributeUsageSamplerFunctor( std::shared_ptr<AttributeUsageSamplerContext> samplerContext, const std::string &subDbName) @@ -23,7 +62,15 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector); search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage(); vespalib::string attributeName = attributeVector.getName(); - _samplerContext->merge(usage, attributeName, _subDbName); + size_t transient_memory_usage = 0; + auto& old_config = attributeVector.getConfig(); + auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName); + if (current_config == nullptr) { + current_config = &old_config; + } + uint64_t total_value_count = attributeVector.getStatus().getNumValues(); + transient_memory_usage = get_transient_memory_usage(old_config, *current_config, total_value_count); + _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index f3303f36199..7a9426ae716 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp + transient_memory_usage_provider.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h new file mode 100644 index 00000000000..e00ccfd3c27 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> + +namespace proton { + +/* + * Interface class providing transient memory usage, e.g. extra memory needed + * for loading or saving an attribute vector. + */ +class ITransientMemoryUsageProvider { +public: + virtual ~ITransientMemoryUsageProvider() = default; + virtual size_t get_transient_memory_usage() const = 0; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp new file mode 100644 index 00000000000..0752af29a0a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "transient_memory_usage_provider.h" + +namespace proton { + +TransientMemoryUsageProvider::TransientMemoryUsageProvider() + : ITransientMemoryUsageProvider(), + _transient_memory_usage(0u) +{ +} + +TransientMemoryUsageProvider::~TransientMemoryUsageProvider() = default; + +size_t +TransientMemoryUsageProvider::get_transient_memory_usage() const +{ + return _transient_memory_usage.load(std::memory_order_relaxed); +} + +void +TransientMemoryUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) +{ + _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h new file mode 100644 index 00000000000..42c6933677a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_transient_memory_usage_provider.h" + +#include <atomic> + +namespace proton { + +/* + * Class providing transient memory usage, e.g. extra memory needed + * for loading or saving an attribute vector. + */ +class TransientMemoryUsageProvider : public ITransientMemoryUsageProvider { + std::atomic<size_t> _transient_memory_usage; +public: + TransientMemoryUsageProvider(); + virtual ~TransientMemoryUsageProvider(); + size_t get_transient_memory_usage() const override; + void set_transient_memory_usage(size_t transient_memory_usage); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 3191d82bf11..daefaf2883e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -156,6 +156,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) _hwInfo(hwInfo), _memoryStats(), _diskUsedSizeBytes(), + _transient_memory_usage(0u), _config(), _state(), _acceptWrite(true), @@ -182,6 +183,13 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) } void +DiskMemUsageFilter::set_transient_memory_usage(size_t transient_memory_usage) +{ + Guard guard(_lock); + _transient_memory_usage = transient_memory_usage; +} + +void DiskMemUsageFilter::setConfig(Config config_in) { Guard guard(_lock); @@ -203,6 +211,13 @@ DiskMemUsageFilter::getDiskUsedSize() const return _diskUsedSizeBytes; } +size_t +DiskMemUsageFilter::get_transient_memory_usage() const +{ + Guard guard(_lock); + return _transient_memory_usage; +} + DiskMemUsageFilter::Config DiskMemUsageFilter::getConfig() const { @@ -230,7 +245,6 @@ DiskMemUsageFilter::getAcceptState() const return _state; } - void DiskMemUsageFilter::addDiskMemUsageListener(IDiskMemUsageListener *listener) { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index afb8c3d2c7a..f3009c2ce97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -43,6 +43,7 @@ private: HwInfo _hwInfo; vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; + size_t _transient_memory_usage; Config _config; State _state; std::atomic<bool> _acceptWrite; @@ -59,9 +60,11 @@ public: ~DiskMemUsageFilter() override; void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); void setDiskUsedSize(uint64_t diskUsedSizeBytes); + void set_transient_memory_usage(size_t transient_memory_usage); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; + size_t get_transient_memory_usage() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index a3fb1244c45..f7f505cd754 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -3,6 +3,7 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/searchcore/proton/common/i_transient_memory_usage_provider.h> #include <filesystem> using vespalib::makeLambdaTask; @@ -14,7 +15,9 @@ DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Confi _path(path_in), _sampleInterval(60s), _lastSampleTime(vespalib::steady_clock::now()), - _periodicTimer() + _periodicTimer(), + _lock(), + _transient_memory_usage_providers() { setConfig(config); } @@ -49,6 +52,7 @@ DiskMemUsageSampler::sampleUsage() { sampleMemoryUsage(); sampleDiskUsage(); + sample_transient_memory_usage(); } namespace { @@ -118,4 +122,37 @@ DiskMemUsageSampler::sampleMemoryUsage() _filter.setMemoryStats(vespalib::ProcessMemoryStats::create()); } +void +DiskMemUsageSampler::sample_transient_memory_usage() +{ + size_t max_transient_memory_usage = 0; + { + std::lock_guard<std::mutex> guard(_lock); + for (auto provider : _transient_memory_usage_providers) { + auto transient_memory_usage = provider->get_transient_memory_usage(); + max_transient_memory_usage = std::max(max_transient_memory_usage, transient_memory_usage); + } + } + _filter.set_transient_memory_usage(max_transient_memory_usage); +} + +void +DiskMemUsageSampler::add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +{ + std::lock_guard<std::mutex> guard(_lock); + _transient_memory_usage_providers.push_back(provider); +} + +void +DiskMemUsageSampler::remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +{ + std::lock_guard<std::mutex> guard(_lock); + for (auto itr = _transient_memory_usage_providers.begin(); itr != _transient_memory_usage_providers.end(); ++itr) { + if (*itr == provider) { + _transient_memory_usage_providers.erase(itr); + break; + } + } +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index bbfc95ff82c..38e80a6cec1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -9,6 +9,8 @@ namespace vespalib { class ScheduledExecutor; } namespace proton { +class ITransientMemoryUsageProvider; + /* * Class to sample disk and memory usage used for filtering write operations. */ @@ -18,10 +20,13 @@ class DiskMemUsageSampler { vespalib::duration _sampleInterval; vespalib::steady_time _lastSampleTime; std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer; + std::mutex _lock; + std::vector<std::shared_ptr<const ITransientMemoryUsageProvider>> _transient_memory_usage_providers; void sampleUsage(); void sampleDiskUsage(); void sampleMemoryUsage(); + void sample_transient_memory_usage(); public: struct Config { DiskMemUsageFilter::Config filterConfig; @@ -55,6 +60,8 @@ public: const DiskMemUsageFilter &writeFilter() const { return _filter; } IDiskMemUsageNotifier ¬ifier() { return _filter; } + void add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); + void remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 55f95ce0518..7567aa9c322 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -13,10 +13,12 @@ #include "maintenance_jobs_injector.h" #include "reconfig_params.h" #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> #include <vespa/searchcore/proton/docsummary/isummarymanager.h> #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -163,6 +165,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _state(), _dmUsageForwarder(_writeService.master()), _writeFilter(), + _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()), _feedHandler(_writeService, tlsSpec, docTypeName, _state, *this, _writeFilter, *this, tlsDirectWriter), _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, @@ -929,7 +932,7 @@ DocumentDB::hasDocument(const document::DocumentId &id) } void -DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) +DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector) { // Called by executor thread _maintenanceController.killJobs(); @@ -954,6 +957,8 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) _visibility, // ICommitable _subDBs.getReadySubDB()->getAttributeManager(), _subDBs.getNotReadySubDB()->getAttributeManager(), + std::move(attribute_config_inspector), + _transient_memory_usage_provider, _writeFilter); } @@ -963,18 +968,21 @@ DocumentDB::performStartMaintenance() // Called by executor thread // Only start once, after replay done - DocumentDBMaintenanceConfig::SP maintenanceConfig; + std::shared_ptr<DocumentDBConfig> activeConfig; { lock_guard guard(_configMutex); if (_state.getClosed()) return; - assert(_activeConfigSnapshot); - maintenanceConfig = _activeConfigSnapshot->getMaintenanceConfigSP(); + activeConfig = _activeConfigSnapshot; } + assert(activeConfig); if (_maintenanceController.getStopping()) { return; } - injectMaintenanceJobs(*maintenanceConfig); + auto maintenanceConfig = activeConfig->getMaintenanceConfigSP(); + const auto &attributes_config = activeConfig->getAttributesConfig(); + auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); + injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); _maintenanceController.start(maintenanceConfig); } @@ -992,10 +1000,12 @@ DocumentDB::forwardMaintenanceConfig() assert(activeConfig); DocumentDBMaintenanceConfig::SP maintenanceConfig(activeConfig->getMaintenanceConfigSP()); + const auto &attributes_config = activeConfig->getAttributesConfig(); + auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); if (!_state.getClosed()) { if (_maintenanceController.getStarted() && !_maintenanceController.getStopping()) { - injectMaintenanceJobs(*maintenanceConfig); + injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); } _maintenanceController.newConfig(maintenanceConfig); } @@ -1086,4 +1096,10 @@ DocumentDB::getDistributionKey() const return _owner.getDistributionKey(); } +std::shared_ptr<const ITransientMemoryUsageProvider> +DocumentDB::transient_memory_usage_provider() +{ + return _transient_memory_usage_provider; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 917d753683a..33dfa85b628 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -44,10 +44,13 @@ namespace search { namespace vespa::config::search::core::internal { class InternalProtonType; } namespace proton { +class AttributeConfigInspector; class IDocumentDBOwner; +class ITransientMemoryUsageProvider; struct MetricsWireService; class StatusReport; class ExecutorThreadingServiceStats; +class TransientMemoryUsageProvider; namespace matching { class SessionManager; } @@ -132,6 +135,7 @@ private: DDBState _state; DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; FeedHandler _feedHandler; DocumentSubDBCollection _subDBs; @@ -412,7 +416,7 @@ public: /** * Implements IFeedHandlerOwner **/ - void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); + void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector); void performStartMaintenance(); void stopMaintenance(); void forwardMaintenanceConfig(); @@ -435,6 +439,7 @@ public: void enterOnlineState(); void waitForOnlineState(); IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } + std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index 483497eb008..d69f0365fc0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -9,6 +9,7 @@ #include "prune_session_cache_job.h" #include "pruneremoveddocumentsjob.h" #include "sample_attribute_usage_job.h" +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> using vespalib::system_clock; @@ -99,6 +100,8 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, ICommitable &commit, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval())); @@ -123,7 +126,9 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, notReadyAttributeManager, attributeUsageFilter, docTypeName, - config.getAttributeUsageSampleInterval())); + config.getAttributeUsageSampleInterval(), + std::move(attribute_config_inspector), + transient_memory_usage_provider)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 94203e144dc..f0859335919 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -12,6 +12,7 @@ namespace proton { +class AttributeConfigInspector; class IPruneRemovedDocumentsHandler; struct IDocumentMoveHandler; class IBucketModifiedHandler; @@ -21,6 +22,7 @@ struct IBucketStateCalculator; struct IAttributeManager; class AttributeUsageFilter; class IDiskMemUsageNotifier; +class TransientMemoryUsageProvider; namespace bucketdb { class IBucketCreateNotifier; } /** @@ -53,6 +55,8 @@ struct MaintenanceJobsInjector ICommitable & commit, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, AttributeUsageFilter &attributeUsageFilter); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c886b371064..cd9c1467da9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -606,6 +606,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, auto flushHandler = std::make_shared<FlushHandlerProxy>(ret); _flushEngine->putFlushHandler(docTypeName, flushHandler); _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener()); + _diskMemUsageSampler->add_transient_memory_usage_provider(ret->transient_memory_usage_provider()); return ret; } @@ -643,6 +644,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook()); _metricsEngine->removeDocumentDBMetrics(old->getMetrics()); _diskMemUsageSampler->notifier().removeDiskMemUsageListener(old->diskMemUsageListener()); + _diskMemUsageSampler->remove_transient_memory_usage_provider(old->transient_memory_usage_provider()); // Caller should have removed & drained relevant timer tasks old->close(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 6f1fd8b90e0..83c268fc755 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -47,6 +47,8 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool memory.setDouble("utilization", usageState.memoryState().utilization()); memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats")); + size_t transient_memory = _usageFilter.get_transient_memory_usage(); + memory.setLong("transient", transient_memory); } else { object.setDouble("disk", usageState.diskState().usage()); object.setDouble("memory", usageState.memoryState().usage()); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 1f5f29c7708..a6497966891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -2,6 +2,7 @@ #include "sample_attribute_usage_job.h" #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> @@ -13,11 +14,15 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval) + vespalib::duration interval, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), - _attributeUsageFilter(attributeUsageFilter) + _attributeUsageFilter(attributeUsageFilter), + _attribute_config_inspector(std::move(attribute_config_inspector)), + _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) { } @@ -26,7 +31,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default; bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_memory_usage_provider); _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h index 72a0bf1a665..6efbfb67c4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h @@ -7,7 +7,9 @@ namespace proton { struct IAttributeManager; +class AttributeConfigInspector; class AttributeUsageFilter; +class TransientMemoryUsageProvider; /** * Class used to sample attribute resource usage and pass aggregated @@ -21,12 +23,16 @@ class SampleAttributeUsageJob : public IMaintenanceJob IAttributeManagerSP _readyAttributeManager; IAttributeManagerSP _notReadyAttributeManager; AttributeUsageFilter &_attributeUsageFilter; + std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; public: SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval); + vespalib::duration interval, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); ~SampleAttributeUsageJob() override; bool run() override; |