diff options
32 files changed, 314 insertions, 167 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index a7988b0ed70..f3519f3ba0b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -446,6 +446,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); + metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index d77bc69ba62..55b3b66a75c 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -143,6 +143,7 @@ vespa_define_module( src/tests/proton/reprocessing/reprocessing_runner src/tests/proton/server src/tests/proton/server/disk_mem_usage_filter + src/tests/proton/server/disk_mem_usage_sampler src/tests/proton/server/health_adapter src/tests/proton/server/memory_flush_config_updater src/tests/proton/server/memoryflush diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp index cb092aaa910..c99abcdaa59 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> -#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> +#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/string.h> @@ -61,18 +61,18 @@ std::shared_ptr<AttributeVector> build_attribute_vector(const vespalib::string& class AttributeUsageSamplerFunctorTest : public ::testing::Test { protected: AttributeUsageFilter _filter; - std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; + std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; public: AttributeUsageSamplerFunctorTest(); ~AttributeUsageSamplerFunctorTest(); protected: void sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search = true); - size_t get_transient_memory_usage() const { return _transient_memory_usage_provider->get_transient_memory_usage(); } + size_t get_transient_memory_usage() const { return _transient_usage_provider->get_transient_memory_usage(); } }; AttributeUsageSamplerFunctorTest::AttributeUsageSamplerFunctorTest() : _filter(), - _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()) + _transient_usage_provider(std::make_shared<TransientResourceUsageProvider>()) { } @@ -88,7 +88,7 @@ AttributeUsageSamplerFunctorTest::sample_usage(bool sample_a1, bool sample_a2, b EXPECT_EQ(av1->getEnumeratedSave(), old_fast_search); auto new_config = build_config(new_fast_search); auto new_inspector = std::make_shared<AttributeConfigInspector>(new_config); - auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_memory_usage_provider); + auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_usage_provider); if (sample_a1) { AttributeUsageSamplerFunctor functor1(context, "ready"); functor1(*av1); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 476579a6816..ff1b29d646e 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> +#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> @@ -869,7 +869,7 @@ MaintenanceControllerFixture::injectMaintenanceJobs() _bmc, _clusterStateHandler, _bucketHandler, _calc, _diskMemUsageNotifier, _jobTrackers, _readyAttributeManager, _notReadyAttributeManager, std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()), - std::make_shared<TransientMemoryUsageProvider>(), + std::make_shared<TransientResourceUsageProvider>(), _attributeUsageFilter); } } diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/CMakeLists.txt new file mode 100644 index 00000000000..ce1e3f30ebc --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_disk_mem_usage_sampler_test_app TEST + SOURCES + disk_mem_usage_sampler_test.cpp + DEPENDS + searchcore_server + GTest::GTest +) +vespa_add_test(NAME searchcore_disk_mem_usage_sampler_test_app COMMAND searchcore_disk_mem_usage_sampler_test_app) diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp new file mode 100644 index 00000000000..666b4b50885 --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> +#include <vespa/searchcore/proton/server/disk_mem_usage_sampler.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <chrono> +#include <thread> + +#include <vespa/log/log.h> +LOG_SETUP("disk_mem_usage_sampler_test"); + +using namespace proton; +using namespace std::chrono_literals; + +constexpr uint64_t disk_size_bytes = 200000; +constexpr uint64_t memory_size_bytes = 100000; + +HwInfo +make_hw_info() +{ + return HwInfo(HwInfo::Disk(disk_size_bytes, false, true), + HwInfo::Memory(memory_size_bytes), + HwInfo::Cpu(1)); +} + +class MyProvider : public ITransientResourceUsageProvider { +private: + size_t _memory_usage; + size_t _disk_usage; + +public: + MyProvider(size_t memory_usage, size_t disk_usage) noexcept + : _memory_usage(memory_usage), + _disk_usage(disk_usage) + {} + size_t get_transient_memory_usage() const override { return _memory_usage; } + size_t get_transient_disk_usage() const override { return _disk_usage; } +}; + +struct DiskMemUsageSamplerTest : public ::testing::Test { + DiskMemUsageSampler sampler; + DiskMemUsageSamplerTest(): + sampler(".", + DiskMemUsageSampler::Config(0.8, 0.8, + 50ms, make_hw_info())) + { + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(99, 200)); + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 199)); + } + const DiskMemUsageFilter& filter() const { return sampler.writeFilter(); } +}; + +TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) +{ + // Poll for up to 20 seconds to get a sample. + size_t i = 0; + for (; i < (20s / 50ms); ++i) { + if (filter().get_transient_memory_usage() > 0) { + break; + } + std::this_thread::sleep_for(50ms); + } + LOG(info, "Polled %zu times (%zu ms) to get a sample", i, i * 50); + EXPECT_GT(filter().getMemoryStats().getAnonymousRss(), 0); + EXPECT_GT(filter().getDiskUsedSize(), 0); + EXPECT_EQ(100, filter().get_transient_memory_usage()); + EXPECT_EQ(100.0 / memory_size_bytes, filter().get_relative_transient_memory_usage()); + EXPECT_EQ(200, filter().get_transient_disk_usage()); + EXPECT_EQ(200.0 / disk_size_bytes, filter().get_relative_transient_disk_usage()); +} + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index 43c9e52315b..c299581835d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -2,24 +2,26 @@ #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" -#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> +#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> namespace proton { -AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) +AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter& filter, + std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) : _usage(), _transient_memory_usage(0u), _lock(), _filter(filter), _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) + _transient_usage_provider(std::move(transient_usage_provider)) { } AttributeUsageSamplerContext::~AttributeUsageSamplerContext() { _filter.setAttributeStats(_usage); - _transient_memory_usage_provider->set_transient_memory_usage(_transient_memory_usage); + _transient_usage_provider->set_transient_memory_usage(_transient_memory_usage); } void diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h index 994e4fd40f1..becff58fdd7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h @@ -10,7 +10,7 @@ namespace proton { class AttributeUsageFilter; class AttributeConfigInspector; -class TransientMemoryUsageProvider; +class TransientResourceUsageProvider; /* * Context for sampling attribute usage stats and transient memory usage. @@ -27,9 +27,11 @@ class AttributeUsageSamplerContext Mutex _lock; AttributeUsageFilter &_filter; std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; + std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; public: - AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); + AttributeUsageSamplerContext(AttributeUsageFilter& filter, + std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); ~AttributeUsageSamplerContext(); void merge(const search::AddressSpaceUsage &usage, size_t transient_memory_usage, @@ -38,7 +40,6 @@ public: const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; } const AttributeUsageStats & getUsage() const { return _usage; } - size_t get_transient_memory_usage() const { return _transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index a91c35f0485..9838b598df9 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -24,7 +24,7 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp - transient_memory_usage_provider.cpp + transient_resource_usage_provider.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h deleted file mode 100644 index cd87150195c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstddef> - -namespace proton { - -/* - * Interface class providing transient memory usage, e.g. extra memory needed - * for loading or saving an attribute vector. It provides an aggregated view - * over several components (e.g. all attribute vectors for a document type). - */ -class ITransientMemoryUsageProvider { -public: - virtual ~ITransientMemoryUsageProvider() = default; - virtual size_t get_transient_memory_usage() const = 0; -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h new file mode 100644 index 00000000000..035a2b19d6d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> + +namespace proton { + +/** + * Interface class providing transient resource usage. + * + * E.g. extra memory needed for loading or saving an attribute vectors or extra disk needed for running disk index fusion. + * It provides an aggregated max view over several components (e.g. all attribute vectors for a document type). + */ +class ITransientResourceUsageProvider { +public: + virtual ~ITransientResourceUsageProvider() = default; + virtual size_t get_transient_memory_usage() const = 0; + virtual size_t get_transient_disk_usage() const = 0; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp deleted file mode 100644 index 0752af29a0a..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "transient_memory_usage_provider.h" - -namespace proton { - -TransientMemoryUsageProvider::TransientMemoryUsageProvider() - : ITransientMemoryUsageProvider(), - _transient_memory_usage(0u) -{ -} - -TransientMemoryUsageProvider::~TransientMemoryUsageProvider() = default; - -size_t -TransientMemoryUsageProvider::get_transient_memory_usage() const -{ - return _transient_memory_usage.load(std::memory_order_relaxed); -} - -void -TransientMemoryUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) -{ - _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h deleted file mode 100644 index 8cda4278ad8..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "i_transient_memory_usage_provider.h" - -#include <atomic> - -namespace proton { - -/* - * Class providing transient memory usage, e.g. extra memory needed - * for loading or saving an attribute vector. It provides an aggregated view - * over several components (e.g. all attribute vectors for a document type). - */ -class TransientMemoryUsageProvider : public ITransientMemoryUsageProvider { - std::atomic<size_t> _transient_memory_usage; -public: - TransientMemoryUsageProvider(); - virtual ~TransientMemoryUsageProvider(); - size_t get_transient_memory_usage() const override; - void set_transient_memory_usage(size_t transient_memory_usage); -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp new file mode 100644 index 00000000000..7643b82855d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "transient_resource_usage_provider.h" + +namespace proton { + +TransientResourceUsageProvider::TransientResourceUsageProvider() + : ITransientResourceUsageProvider(), + _transient_memory_usage(0u) +{ +} + +TransientResourceUsageProvider::~TransientResourceUsageProvider() = default; + +size_t +TransientResourceUsageProvider::get_transient_memory_usage() const +{ + return _transient_memory_usage.load(std::memory_order_relaxed); +} + +void +TransientResourceUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) +{ + _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h new file mode 100644 index 00000000000..556c874caed --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_transient_resource_usage_provider.h" + +#include <atomic> + +namespace proton { + +/** + * Class providing transient resource usage. + * E.g. extra memory needed for loading or saving an attribute vector. + * It provides an aggregated view over several components (e.g. all attribute vectors for a document type). + */ +class TransientResourceUsageProvider : public ITransientResourceUsageProvider { + std::atomic<size_t> _transient_memory_usage; +public: + TransientResourceUsageProvider(); + virtual ~TransientResourceUsageProvider(); + size_t get_transient_memory_usage() const override; + size_t get_transient_disk_usage() const override { return 0; } + void set_transient_memory_usage(size_t transient_memory_usage); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp index c890592b411..85b2a7a4b99 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp @@ -10,7 +10,8 @@ ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent) diskUtilization("disk_utilization", {}, "The relative amount of disk used compared to the disk resource limit", this), memory("memory", {}, "The relative amount of memory used by this process (value in the range [0, 1])", this), memoryUtilization("memory_utilization", {}, "The relative amount of memory used compared to the memory resource limit", this), - transient_memory("transient_memory", {}, "The relative amount of transient memory needed for load (value in the range [0, 1])", this), + transient_memory("transient_memory", {}, "The relative amount of transient memory needed for loading attributes. Max value among all attributes (value in the range [0, 1])", this), + transient_disk("transient_disk", {}, "The relative amount of transient disk needed for running disk index fusion. Max value among all disk indexes (value in the range [0, 1])", this), memoryMappings("memory_mappings", {}, "The number of mapped memory areas", this), openFileDescriptors("open_file_descriptors", {}, "The number of open files", this), feedingBlocked("feeding_blocked", {}, "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h index 1878cd4b7c9..038d9149b3b 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h @@ -17,6 +17,7 @@ struct ResourceUsageMetrics : metrics::MetricSet metrics::DoubleValueMetric memory; metrics::DoubleValueMetric memoryUtilization; metrics::DoubleValueMetric transient_memory; + metrics::DoubleValueMetric transient_disk; metrics::LongValueMetric memoryMappings; metrics::LongValueMetric openFileDescriptors; metrics::LongValueMetric feedingBlocked; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 4855577e712..ef77081132b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -183,10 +183,11 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) } void -DiskMemUsageFilter::set_transient_memory_usage(size_t transient_memory_usage) +DiskMemUsageFilter::set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage) { Guard guard(_lock); _transient_memory_usage = transient_memory_usage; + _transient_disk_usage = transient_disk_usage; } void @@ -225,6 +226,20 @@ DiskMemUsageFilter::get_relative_transient_memory_usage() const return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); } +size_t +DiskMemUsageFilter::get_transient_disk_usage() const +{ + Guard guard(_lock); + return _transient_disk_usage; +} + +double +DiskMemUsageFilter::get_relative_transient_disk_usage() const +{ + Guard guard(_lock); + return static_cast<double>(_transient_disk_usage) / _hwInfo.disk().sizeBytes(); +} + DiskMemUsageFilter::Config DiskMemUsageFilter::getConfig() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index badc363e5bd..cdb9fc5f4cb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -45,6 +45,7 @@ private: vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; size_t _transient_memory_usage; + size_t _transient_disk_usage; Config _config; State _state; std::atomic<bool> _acceptWrite; @@ -61,12 +62,14 @@ public: ~DiskMemUsageFilter() override; void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); void setDiskUsedSize(uint64_t diskUsedSizeBytes); - void set_transient_memory_usage(size_t transient_memory_usage); + void set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; size_t get_transient_memory_usage() const; double get_relative_transient_memory_usage() const; + size_t get_transient_disk_usage() const; + double get_relative_transient_disk_usage() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index f7f505cd754..abd89345401 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -3,7 +3,7 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/searchcore/proton/common/i_transient_memory_usage_provider.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <filesystem> using vespalib::makeLambdaTask; @@ -17,7 +17,7 @@ DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Confi _lastSampleTime(vespalib::steady_clock::now()), _periodicTimer(), _lock(), - _transient_memory_usage_providers() + _transient_usage_providers() { setConfig(config); } @@ -52,7 +52,7 @@ DiskMemUsageSampler::sampleUsage() { sampleMemoryUsage(); sampleDiskUsage(); - sample_transient_memory_usage(); + sample_transient_resource_usage(); } namespace { @@ -123,33 +123,34 @@ DiskMemUsageSampler::sampleMemoryUsage() } void -DiskMemUsageSampler::sample_transient_memory_usage() +DiskMemUsageSampler::sample_transient_resource_usage() { size_t max_transient_memory_usage = 0; + size_t max_transient_disk_usage = 0; { std::lock_guard<std::mutex> guard(_lock); - for (auto provider : _transient_memory_usage_providers) { - auto transient_memory_usage = provider->get_transient_memory_usage(); - max_transient_memory_usage = std::max(max_transient_memory_usage, transient_memory_usage); + for (auto provider : _transient_usage_providers) { + max_transient_memory_usage = std::max(max_transient_memory_usage, provider->get_transient_memory_usage()); + max_transient_disk_usage = std::max(max_transient_disk_usage, provider->get_transient_disk_usage()); } } - _filter.set_transient_memory_usage(max_transient_memory_usage); + _filter.set_transient_resource_usage(max_transient_memory_usage, max_transient_disk_usage); } void -DiskMemUsageSampler::add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +DiskMemUsageSampler::add_transient_usage_provider(std::shared_ptr<const ITransientResourceUsageProvider> provider) { std::lock_guard<std::mutex> guard(_lock); - _transient_memory_usage_providers.push_back(provider); + _transient_usage_providers.push_back(provider); } void -DiskMemUsageSampler::remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +DiskMemUsageSampler::remove_transient_usage_provider(std::shared_ptr<const ITransientResourceUsageProvider> provider) { std::lock_guard<std::mutex> guard(_lock); - for (auto itr = _transient_memory_usage_providers.begin(); itr != _transient_memory_usage_providers.end(); ++itr) { + for (auto itr = _transient_usage_providers.begin(); itr != _transient_usage_providers.end(); ++itr) { if (*itr == provider) { - _transient_memory_usage_providers.erase(itr); + _transient_usage_providers.erase(itr); break; } } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index 38e80a6cec1..2248db87960 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -9,7 +9,7 @@ namespace vespalib { class ScheduledExecutor; } namespace proton { -class ITransientMemoryUsageProvider; +class ITransientResourceUsageProvider; /* * Class to sample disk and memory usage used for filtering write operations. @@ -21,12 +21,12 @@ class DiskMemUsageSampler { vespalib::steady_time _lastSampleTime; std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer; std::mutex _lock; - std::vector<std::shared_ptr<const ITransientMemoryUsageProvider>> _transient_memory_usage_providers; + std::vector<std::shared_ptr<const ITransientResourceUsageProvider>> _transient_usage_providers; void sampleUsage(); void sampleDiskUsage(); void sampleMemoryUsage(); - void sample_transient_memory_usage(); + void sample_transient_resource_usage(); public: struct Config { DiskMemUsageFilter::Config filterConfig; @@ -60,8 +60,8 @@ public: const DiskMemUsageFilter &writeFilter() const { return _filter; } IDiskMemUsageNotifier ¬ifier() { return _filter; } - void add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); - void remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); + void add_transient_usage_provider(std::shared_ptr<const ITransientResourceUsageProvider> provider); + void remove_transient_usage_provider(std::shared_ptr<const ITransientResourceUsageProvider> provider); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 46d4c09ea67..3cb5c31e13c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -20,7 +20,7 @@ #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> -#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> +#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/searchcore/proton/docsummary/isummarymanager.h> #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -88,6 +88,21 @@ public: } }; +class DocumentDBResourceUsageProvider : public TransientResourceUsageProvider { +private: + const DocumentDB& _doc_db; + +public: + DocumentDBResourceUsageProvider(const DocumentDB& doc_db) + : _doc_db(doc_db) + {} + size_t get_transient_disk_usage() const override { + // We estimate the transient disk usage for the next disk index fusion + // as the size of the largest disk index. + return _doc_db.getReadySubDB()->getSearchableStats().max_component_size_on_disk(); + } +}; + } template <typename FunctionType> @@ -152,7 +167,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _state(), _dmUsageForwarder(_writeService.master()), _writeFilter(), - _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()), + _transient_usage_provider(std::make_shared<DocumentDBResourceUsageProvider>(*this)), _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, @@ -938,7 +953,7 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std _subDBs.getReadySubDB()->getAttributeManager(), _subDBs.getNotReadySubDB()->getAttributeManager(), std::move(attribute_config_inspector), - _transient_memory_usage_provider, + _transient_usage_provider, _writeFilter); } @@ -1085,10 +1100,10 @@ DocumentDB::getDistributionKey() const return _owner.getDistributionKey(); } -std::shared_ptr<const ITransientMemoryUsageProvider> -DocumentDB::transient_memory_usage_provider() +std::shared_ptr<const ITransientResourceUsageProvider> +DocumentDB::transient_usage_provider() { - return _transient_memory_usage_provider; + return _transient_usage_provider; } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 3fec997d1bc..d8bba30e650 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -50,11 +50,11 @@ namespace storage::spi { struct BucketExecutor; } namespace proton { class AttributeConfigInspector; class IDocumentDBOwner; -class ITransientMemoryUsageProvider; +class ITransientResourceUsageProvider; struct MetricsWireService; class StatusReport; class ExecutorThreadingServiceStats; -class TransientMemoryUsageProvider; +class TransientResourceUsageProvider; namespace matching { class SessionManager; } @@ -119,7 +119,7 @@ private: DDBState _state; DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; - std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; + std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; std::unique_ptr<FeedHandler> _feedHandler; DocumentSubDBCollection _subDBs; MaintenanceController _maintenanceController; @@ -414,7 +414,7 @@ public: void enterOnlineState(); void waitForOnlineState(); IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } - std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); + std::shared_ptr<const ITransientResourceUsageProvider> transient_usage_provider(); ExecutorThreadingService & getWriteService() { return _writeService; } void set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index ce6acb4795e..bd6c09934be 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -134,7 +134,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); @@ -160,7 +160,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, attributeUsageFilter, docTypeName, config.getAttributeUsageSampleInterval(), std::move(attribute_config_inspector), - transient_memory_usage_provider)); + transient_usage_provider)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 8ec479084d2..86049736a38 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -22,7 +22,7 @@ struct IBucketStateCalculator; struct IAttributeManager; class AttributeUsageFilter; class IDiskMemUsageNotifier; -class TransientMemoryUsageProvider; +class TransientResourceUsageProvider; namespace bucketdb { class IBucketCreateNotifier; } /** @@ -54,7 +54,7 @@ struct MaintenanceJobsInjector IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index f6c09b6b7c4..103805a41cd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -643,7 +643,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, auto flushHandler = std::make_shared<FlushHandlerProxy>(ret); _flushEngine->putFlushHandler(docTypeName, flushHandler); _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener()); - _diskMemUsageSampler->add_transient_memory_usage_provider(ret->transient_memory_usage_provider()); + _diskMemUsageSampler->add_transient_usage_provider(ret->transient_usage_provider()); return ret; } @@ -681,7 +681,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook()); _metricsEngine->removeDocumentDBMetrics(old->getMetrics()); _diskMemUsageSampler->notifier().removeDiskMemUsageListener(old->diskMemUsageListener()); - _diskMemUsageSampler->remove_transient_memory_usage_provider(old->transient_memory_usage_provider()); + _diskMemUsageSampler->remove_transient_usage_provider(old->transient_usage_provider()); // Caller should have removed & drained relevant timer tasks old->close(); } @@ -750,6 +750,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) metrics.resourceUsage.memory.set(usageState.memoryState().usage()); metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization()); metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); + metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index a6497966891..a8397d95313 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -16,13 +16,13 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, const vespalib::string &docTypeName, vespalib::duration interval, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), _attributeUsageFilter(attributeUsageFilter), _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) + _transient_usage_provider(std::move(transient_usage_provider)) { } @@ -31,7 +31,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default; bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_memory_usage_provider); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_usage_provider); _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h index 96cf82df362..bb5cf4154aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h @@ -9,7 +9,7 @@ namespace proton { struct IAttributeManager; class AttributeConfigInspector; class AttributeUsageFilter; -class TransientMemoryUsageProvider; +class TransientResourceUsageProvider; /** * Class used to sample attribute resource usage and pass aggregated @@ -24,7 +24,7 @@ class SampleAttributeUsageJob : public IMaintenanceJob IAttributeManagerSP _notReadyAttributeManager; AttributeUsageFilter &_attributeUsageFilter; std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; + std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; public: SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, @@ -32,7 +32,7 @@ public: const vespalib::string &docTypeName, vespalib::duration interval, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); + std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); ~SampleAttributeUsageJob() override; bool run() override; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp index 12128a3df18..031cf0178d8 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp @@ -113,7 +113,7 @@ IndexCollection::getSearchableStats() const { search::SearchableStats stats; for (size_t i = 0; i < _sources.size(); ++i) { - stats.add(_sources[i].source_wrapper->getSearchableStats()); + stats.merge(_sources[i].source_wrapper->getSearchableStats()); } return stats; } diff --git a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt index cbaabba41f3..786c316eb4f 100644 --- a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt +++ b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_searchable_stats_test_app TEST searchable_stats_test.cpp DEPENDS searchlib + GTest::GTest ) vespa_add_test(NAME searchlib_searchable_stats_test_app COMMAND searchlib_searchable_stats_test_app) diff --git a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp index c91214f5cfe..7f53eaddea5 100644 --- a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp +++ b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp @@ -1,41 +1,49 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchlib/util/searchable_stats.h> +#include <vespa/vespalib/gtest/gtest.h> + #include <vespa/log/log.h> LOG_SETUP("searchable_stats_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/util/searchable_stats.h> using namespace search; -class Test : public vespalib::TestApp { -public: - int Main() override; -}; - -int -Test::Main() +TEST(SearchableStatsTest, merge_also_tracks_max_size_on_disk_for_component) { - TEST_INIT("searchable_stats_test"); + SearchableStats stats; + EXPECT_EQ(0u, stats.memoryUsage().allocatedBytes()); + EXPECT_EQ(0u, stats.docsInMemory()); + EXPECT_EQ(0u, stats.sizeOnDisk()); + EXPECT_EQ(0u, stats.max_component_size_on_disk()); { - SearchableStats stats; - EXPECT_EQUAL(0u, stats.memoryUsage().allocatedBytes()); - EXPECT_EQUAL(0u, stats.docsInMemory()); - EXPECT_EQUAL(0u, stats.sizeOnDisk()); - { - SearchableStats rhs; - EXPECT_EQUAL(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs); - EXPECT_EQUAL(&rhs.docsInMemory(10), &rhs); - EXPECT_EQUAL(&rhs.sizeOnDisk(1000), &rhs); - EXPECT_EQUAL(&stats.add(rhs), &stats); - } - EXPECT_EQUAL(100u, stats.memoryUsage().allocatedBytes()); - EXPECT_EQUAL(10u, stats.docsInMemory()); - EXPECT_EQUAL(1000u, stats.sizeOnDisk()); - EXPECT_EQUAL(&stats.add(SearchableStats().memoryUsage(vespalib::MemoryUsage(100,0,0,0)).docsInMemory(10).sizeOnDisk(1000)), &stats); - EXPECT_EQUAL(200u, stats.memoryUsage().allocatedBytes()); - EXPECT_EQUAL(20u, stats.docsInMemory()); - EXPECT_EQUAL(2000u, stats.sizeOnDisk()); + SearchableStats rhs; + EXPECT_EQ(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs); + EXPECT_EQ(&rhs.docsInMemory(10), &rhs); + EXPECT_EQ(&rhs.sizeOnDisk(1000), &rhs); + EXPECT_EQ(1000u, rhs.max_component_size_on_disk()); + EXPECT_EQ(&stats.merge(rhs), &stats); } - TEST_DONE(); + EXPECT_EQ(100u, stats.memoryUsage().allocatedBytes()); + EXPECT_EQ(10u, stats.docsInMemory()); + EXPECT_EQ(1000u, stats.sizeOnDisk()); + EXPECT_EQ(1000u, stats.max_component_size_on_disk()); + + stats.merge(SearchableStats() + .memoryUsage(vespalib::MemoryUsage(150,0,0,0)) + .docsInMemory(15) + .sizeOnDisk(1500)); + EXPECT_EQ(250u, stats.memoryUsage().allocatedBytes()); + EXPECT_EQ(25u, stats.docsInMemory()); + EXPECT_EQ(2500u, stats.sizeOnDisk()); + EXPECT_EQ(1500u, stats.max_component_size_on_disk()); + + stats.merge(SearchableStats() + .memoryUsage(vespalib::MemoryUsage(120,0,0,0)) + .docsInMemory(12) + .sizeOnDisk(1200)); + EXPECT_EQ(370u, stats.memoryUsage().allocatedBytes()); + EXPECT_EQ(37u, stats.docsInMemory()); + EXPECT_EQ(3700u, stats.sizeOnDisk()); + EXPECT_EQ(1500u, stats.max_component_size_on_disk()); } -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/util/searchable_stats.h b/searchlib/src/vespa/searchlib/util/searchable_stats.h index 4772c04f6be..5138e636df5 100644 --- a/searchlib/src/vespa/searchlib/util/searchable_stats.h +++ b/searchlib/src/vespa/searchlib/util/searchable_stats.h @@ -6,9 +6,9 @@ namespace search { /** - * Simple statistics for a single Searchable component. Used for - * internal aggregation before inserting numbers into the metrics - * framework. + * Simple statistics for a single Searchable component or multiple components that are merged together. + * + * E.g. used for internal aggregation before inserting numbers into the metrics framework. **/ class SearchableStats { @@ -16,9 +16,10 @@ private: vespalib::MemoryUsage _memoryUsage; size_t _docsInMemory; size_t _sizeOnDisk; + size_t _max_component_size_on_disk; public: - SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0) {} + SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _max_component_size_on_disk(0) {} SearchableStats &memoryUsage(const vespalib::MemoryUsage &usage) { _memoryUsage = usage; return *this; @@ -31,13 +32,22 @@ public: size_t docsInMemory() const { return _docsInMemory; } SearchableStats &sizeOnDisk(size_t value) { _sizeOnDisk = value; + _max_component_size_on_disk = value; return *this; } size_t sizeOnDisk() const { return _sizeOnDisk; } - SearchableStats &add(const SearchableStats &rhs) { + + /** + * Returns the max disk size used by a single Searchable component, + * e.g. among the components that are merged into a SearchableStats instance via merge(). + */ + size_t max_component_size_on_disk() const { return _max_component_size_on_disk; } + + SearchableStats &merge(const SearchableStats &rhs) { _memoryUsage.merge(rhs._memoryUsage); _docsInMemory += rhs._docsInMemory; _sizeOnDisk += rhs._sizeOnDisk; + _max_component_size_on_disk = std::max(_max_component_size_on_disk, rhs._sizeOnDisk); return *this; } }; |