diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2020-05-13 10:54:07 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2020-05-13 11:26:51 +0200 |
commit | d44ef62fdbd6b0f5b67ab217e3ec8fbe16469f1c (patch) | |
tree | 9e97ecf7458be092699d62a1eacaa3c57115f639 | |
parent | 0fa71866cd87e65331169700c656313f41f85eaa (diff) |
Consolidate code to calculate transient memory usage for attribute
vector load.
5 files changed, 104 insertions, 64 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index e0b0cd9f0fd..c20f279503f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_manager_initializer.cpp attribute_populator.cpp attribute_spec.cpp + attribute_transient_memory_calculator.cpp attribute_type_matcher.cpp attribute_usage_filter.cpp attribute_usage_sampler_context.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 88e4d7a2191..368e4ce12f2 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -4,6 +4,7 @@ #include "attributedisklayout.h" #include "attribute_directory.h" #include "i_attribute_factory.h" +#include "attribute_transient_memory_calculator.h" #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -12,8 +13,6 @@ #include <vespa/searchlib/util/fileutil.h> #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/loadedenumvalue.h> -#include <vespa/searchlib/attribute/loadedvalue.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> @@ -263,29 +262,8 @@ size_t AttributeInitializer::get_transient_memory_usage() const { if (_header_ok) { - auto &header = *_header; - if (_spec.getConfig().fastSearch()) { - if (header.getEnumerated()) { - return sizeof(search::attribute::LoadedEnumAttribute) * header.get_total_value_count(); - } else { - switch (_spec.getConfig().basicType().type()) { - case BasicType::Type::INT8: - return sizeof(search::attribute::LoadedValue<int8_t>) * header.get_total_value_count(); - case BasicType::Type::INT16: - return sizeof(search::attribute::LoadedValue<int16_t>) * header.get_total_value_count(); - case BasicType::Type::INT32: - return sizeof(search::attribute::LoadedValue<int32_t>) * header.get_total_value_count(); - case BasicType::Type::INT64: - return sizeof(search::attribute::LoadedValue<int64_t>) * header.get_total_value_count(); - case BasicType::Type::FLOAT: - return sizeof(search::attribute::LoadedValue<float>) * header.get_total_value_count(); - case BasicType::Type::DOUBLE: - return sizeof(search::attribute::LoadedValue<double>) * header.get_total_value_count(); - default: - return 0u; - } - } - } + AttributeTransientMemoryCalculator get_transient_memory_usage; + return get_transient_memory_usage(*_header, _spec.getConfig()); } return 0u; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp new file mode 100644 index 00000000000..37706328149 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_transient_memory_calculator.h" +#include <vespa/searchlib/attribute/attribute_header.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/loadedenumvalue.h> +#include <vespa/searchlib/attribute/loadedvalue.h> +#include <vespa/searchlib/attribute/attributevector.h> + +using search::attribute::BasicType; + +namespace proton { + +namespace { + +size_t +get_transient_memory_usage(bool old_enumerated, + const search::attribute::Config& new_config, + uint64_t total_value_count) +{ + if (new_config.fastSearch()) { + if (old_enumerated) { + return sizeof(search::attribute::LoadedEnumAttribute) * total_value_count; + } else { + switch (new_config.basicType().type()) { + case BasicType::Type::INT8: + return sizeof(search::attribute::LoadedValue<int8_t>) * total_value_count; + case BasicType::Type::INT16: + return sizeof(search::attribute::LoadedValue<int16_t>) * total_value_count; + case BasicType::Type::INT32: + return sizeof(search::attribute::LoadedValue<int32_t>) * total_value_count; + case BasicType::Type::INT64: + return sizeof(search::attribute::LoadedValue<int64_t>) * total_value_count; + case BasicType::Type::FLOAT: + return sizeof(search::attribute::LoadedValue<float>) * total_value_count; + case BasicType::Type::DOUBLE: + return sizeof(search::attribute::LoadedValue<double>) * total_value_count; + default: + ; + } + } + } + return 0u; +} + +} +size_t +AttributeTransientMemoryCalculator::operator()(const search::AttributeVector& attribute_vector, + const search::attribute::Config& new_config) const +{ + uint64_t total_value_count = attribute_vector.getStatus().getNumValues(); + bool old_enumerated = attribute_vector.getEnumeratedSave(); + return get_transient_memory_usage(old_enumerated, new_config, total_value_count); +} + +size_t +AttributeTransientMemoryCalculator::operator()(const search::attribute::AttributeHeader& old_header, + const search::attribute::Config& new_config) const +{ + return get_transient_memory_usage(old_header.getEnumerated(), new_config, old_header.get_total_value_count()); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h new file mode 100644 index 00000000000..34ab8f02768 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> +#include <cstdint> + +namespace search { class AttributeVector; } + +namespace search::attribute { + +class AttributeHeader; +class Config; + +} + +namespace proton { + +/** + * Class to calculate transient memory during load of attribute vector + * in the future based on current attribute vector and new config. + */ +class AttributeTransientMemoryCalculator +{ +public: + AttributeTransientMemoryCalculator() = default; + ~AttributeTransientMemoryCalculator() = default; + size_t operator()(const search::AttributeVector& attribute_vector, + const search::attribute::Config& new_config) const; + size_t operator()(const search::attribute::AttributeHeader& old_header, + const search::attribute::Config& new_config) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index f8594a4477b..685433cd5c7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -3,48 +3,13 @@ #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" #include "attribute_config_inspector.h" +#include "attribute_transient_memory_calculator.h" #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/loadedenumvalue.h> -#include <vespa/searchlib/attribute/loadedvalue.h> using search::attribute::BasicType; namespace proton { -namespace { - -size_t -get_transient_memory_usage(const search::attribute::Config& old_config, - const search::attribute::Config& current_config, - uint64_t total_value_count) -{ - if (current_config.fastSearch()) { - if (old_config.fastSearch()) { - return sizeof(search::attribute::LoadedEnumAttribute) * total_value_count; - } else { - switch (old_config.basicType().type()) { - case BasicType::Type::INT8: - return sizeof(search::attribute::LoadedValue<int8_t>) * total_value_count; - case BasicType::Type::INT16: - return sizeof(search::attribute::LoadedValue<int16_t>) * total_value_count; - case BasicType::Type::INT32: - return sizeof(search::attribute::LoadedValue<int32_t>) * total_value_count; - case BasicType::Type::INT64: - return sizeof(search::attribute::LoadedValue<int64_t>) * total_value_count; - case BasicType::Type::FLOAT: - return sizeof(search::attribute::LoadedValue<float>) * total_value_count; - case BasicType::Type::DOUBLE: - return sizeof(search::attribute::LoadedValue<double>) * total_value_count; - default: - ; - } - } - } - return 0u; -} - -} - AttributeUsageSamplerFunctor::AttributeUsageSamplerFunctor( std::shared_ptr<AttributeUsageSamplerContext> samplerContext, const std::string &subDbName) @@ -62,14 +27,13 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector); search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage(); vespalib::string attributeName = attributeVector.getName(); - size_t transient_memory_usage = 0; auto& old_config = attributeVector.getConfig(); auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName); if (current_config == nullptr) { current_config = &old_config; } - uint64_t total_value_count = attributeVector.getStatus().getNumValues(); - transient_memory_usage = get_transient_memory_usage(old_config, *current_config, total_value_count); + AttributeTransientMemoryCalculator get_transient_memory_usage; + size_t transient_memory_usage = get_transient_memory_usage(attributeVector, *current_config); _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName); } |