diff options
Diffstat (limited to 'searchcore')
29 files changed, 113 insertions, 243 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index c63ccfe4e24..7fb98cb2514 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -61,8 +61,8 @@ vespa_define_module( src/tests/proton/attribute/attribute_initializer src/tests/proton/attribute/attribute_manager src/tests/proton/attribute/attribute_populator + src/tests/proton/attribute/attribute_transient_memory_calculator src/tests/proton/attribute/attribute_usage_filter - src/tests/proton/attribute/attribute_usage_sampler_functor src/tests/proton/attribute/attribute_usage_stats src/tests/proton/attribute/attributes_state_explorer src/tests/proton/attribute/document_field_extractor diff --git a/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt new file mode 100644 index 00000000000..bc77f2d949b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_transient_memory_calculator_test_app TEST + SOURCES + attribute_transient_memory_calculator_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon + GTest::GTest +) +vespa_add_test(NAME searchcore_attribute_transient_memory_calculator_test_app COMMAND searchcore_attribute_transient_memory_calculator_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp index 31f90075421..786d1f6a801 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp @@ -5,10 +5,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> +#include <vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/string.h> @@ -58,73 +55,32 @@ std::shared_ptr<AttributeVector> build_attribute_vector(const vespalib::string& } -class AttributeUsageSamplerFunctorTest : public ::testing::Test { -protected: - AttributeUsageFilter _filter; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; -public: - AttributeUsageSamplerFunctorTest(); - ~AttributeUsageSamplerFunctorTest(); -protected: - void sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search = true); - size_t get_transient_memory_usage() const { return _transient_usage_provider->get_transient_memory_usage(); } -}; - -AttributeUsageSamplerFunctorTest::AttributeUsageSamplerFunctorTest() - : _filter(), - _transient_usage_provider(std::make_shared<TransientResourceUsageProvider>()) -{ -} - -AttributeUsageSamplerFunctorTest::~AttributeUsageSamplerFunctorTest() = default; - -void -AttributeUsageSamplerFunctorTest::sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search) +size_t +sample_usage(bool old_fast_search, bool new_fast_search) { auto old_config = build_config(old_fast_search); auto old_inspector = std::make_shared<AttributeConfigInspector>(old_config); auto av1 = build_attribute_vector("a1", *old_inspector, 1); - auto av2 = build_attribute_vector("a2", *old_inspector, 3); EXPECT_EQ(av1->getEnumeratedSave(), old_fast_search); auto new_config = build_config(new_fast_search); auto new_inspector = std::make_shared<AttributeConfigInspector>(new_config); - auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_usage_provider); - if (sample_a1) { - AttributeUsageSamplerFunctor functor1(context, "ready"); - functor1(*av1); - } - if (sample_a2) { - AttributeUsageSamplerFunctor functor2(context, "ready"); - functor2(*av2); - } -} - -TEST_F(AttributeUsageSamplerFunctorTest, plain_attribute_vector_requires_no_transient_memory_for_load) -{ - sample_usage(true, true, false, false); - EXPECT_EQ(0u, get_transient_memory_usage()); + AttributeTransientMemoryCalculator calc; + return calc(*av1, *new_inspector->get_config("a1")); } -TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_transient_memory_for_load) +TEST(AttributeTransientMemoryCalculator, plain_attribute_vector_requires_no_transient_memory_for_load) { - sample_usage(true, false, true, true); - EXPECT_EQ(24u, get_transient_memory_usage()); + EXPECT_EQ(0, sample_usage(false, false)); } -TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated) +TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_transient_memory_for_load) { - sample_usage(true, false, false, true); - EXPECT_EQ(40u, get_transient_memory_usage()); + EXPECT_EQ(24u, sample_usage(true, true)); } -TEST_F(AttributeUsageSamplerFunctorTest, transient_memory_aggregation_function_for_attribute_usage_sampler_context_is_max) +TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated) { - sample_usage(true, false, true, true); - EXPECT_EQ(24u, get_transient_memory_usage()); - sample_usage(false, true, true, true); - EXPECT_EQ(72u, get_transient_memory_usage()); - sample_usage(true, true, true, true); - EXPECT_EQ(72u, get_transient_memory_usage()); + EXPECT_EQ(40u, sample_usage(false, true)); } } diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt deleted file mode 100644 index 6034f5a1309..00000000000 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_usage_sampler_functor_test_app TEST - SOURCES - attribute_usage_sampler_functor_test.cpp - DEPENDS - searchcore_attribute - searchcore_pcommon - GTest::GTest -) -vespa_add_test(NAME searchcore_attribute_usage_sampler_functor_test_app COMMAND searchcore_attribute_usage_sampler_functor_test_app) diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 227e885564d..9657619be40 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -1,14 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config-attributes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> +#include <vespa/fastos/thread.h> +#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> +#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> -#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> +#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> #include <vespa/searchcore/proton/feedoperation/putoperation.h> @@ -25,14 +30,9 @@ #include <vespa/searchcore/proton/server/maintenancecontroller.h> #include <vespa/searchcore/proton/test/buckethandler.h> #include <vespa/searchcore/proton/test/clusterstatehandler.h> -#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchcore/proton/test/test.h> -#include <vespa/config-attributes.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/vespalib/data/slime/slime.h> @@ -43,7 +43,6 @@ #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/fastos/thread.h> #include <unistd.h> #include <thread> @@ -824,8 +823,6 @@ MaintenanceControllerFixture::injectMaintenanceJobs() _bucketCreateNotifier, makeBucketSpace(), _fh, _fh, _bmc, _clusterStateHandler, _bucketHandler, _calc, _diskMemUsageNotifier, _jobTrackers, _readyAttributeManager, _notReadyAttributeManager, - std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()), - std::make_shared<TransientResourceUsageProvider>(), _attributeUsageFilter); } } diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index b83ce4d6d1b..072b76c6443 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -121,4 +121,11 @@ TEST_F(DiskMemUsageFilterTest, both_disk_limit_and_memory_limit_can_be_reached) "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } +TEST_F(DiskMemUsageFilterTest, transient_memory_usage_is_tracked_in_usage_state_and_metrics) +{ + _filter.set_transient_resource_usage(200, 0); + EXPECT_EQ(0.2, _filter.usageState().transient_memory_usage()); + EXPECT_EQ(0.2, _filter.get_metrics().get_transient_memory_usage()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp index 2731f51a85e..7fa51bb8b95 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -45,7 +45,7 @@ struct DiskMemUsageSamplerTest : public ::testing::Test { DiskMemUsageSampler::Config(0.8, 0.8, 50ms, make_hw_info())) { - sampler.add_transient_usage_provider(std::make_shared<MyProvider>(99, 200)); + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(50, 200)); sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 199)); } const DiskMemUsageFilter& filter() const { return sampler.writeFilter(); } @@ -70,8 +70,8 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) EXPECT_EQ(filter().getMemoryStats().getAnonymousRss(), 0); #endif EXPECT_GT(filter().getDiskUsedSize(), 0); - EXPECT_EQ(100, filter().get_transient_memory_usage()); - EXPECT_EQ(100.0 / memory_size_bytes, filter().get_relative_transient_memory_usage()); + EXPECT_EQ(150, filter().get_transient_memory_usage()); + EXPECT_EQ(150.0 / memory_size_bytes, filter().usageState().transient_memory_usage()); EXPECT_EQ(200, filter().get_transient_disk_usage()); EXPECT_EQ(200.0 / disk_size_bytes, filter().get_relative_transient_disk_usage()); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index 08432916d2b..16a4357d9f8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -2,37 +2,28 @@ #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> namespace proton { -AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter& filter, - std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) +AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter& filter) : _usage(), - _transient_memory_usage(0u), _lock(), - _filter(filter), - _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_usage_provider(std::move(transient_usage_provider)) + _filter(filter) { } AttributeUsageSamplerContext::~AttributeUsageSamplerContext() { _filter.setAttributeStats(_usage); - _transient_usage_provider->set_transient_memory_usage(_transient_memory_usage); } void AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage, - size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { Guard guard(_lock); _usage.merge(usage, attributeName, subDbName); - _transient_memory_usage = std::max(_transient_memory_usage, transient_memory_usage); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h index 71cdc0e7f1a..bc43fb1ce4b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h @@ -13,9 +13,8 @@ class AttributeConfigInspector; class TransientResourceUsageProvider; /* - * Context for sampling attribute usage stats and transient memory usage. - * When instance is destroyed, the aggregated stats is passed on to - * attribute usage filter and the transient memory usage provider. + * Context for sampling attribute usage stats. + * When instance is destroyed, the aggregated stats is passed on to attribute usage filter. */ class AttributeUsageSamplerContext { @@ -23,23 +22,16 @@ class AttributeUsageSamplerContext using Guard = std::lock_guard<Mutex>; AttributeUsageStats _usage; - size_t _transient_memory_usage; Mutex _lock; AttributeUsageFilter &_filter; - std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + public: - AttributeUsageSamplerContext(AttributeUsageFilter& filter, - std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); + AttributeUsageSamplerContext(AttributeUsageFilter& filter); ~AttributeUsageSamplerContext(); void merge(const search::AddressSpaceUsage &usage, - size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName); - const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; } - const AttributeUsageStats & - getUsage() const { return _usage; } + const AttributeUsageStats& getUsage() const { return _usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index 6db20ab82a6..0af77f3e8cf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -3,7 +3,6 @@ #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" #include "attribute_config_inspector.h" -#include "attribute_transient_memory_calculator.h" #include <vespa/searchlib/attribute/attributevector.h> using search::attribute::BasicType; @@ -27,14 +26,7 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector); search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage(); vespalib::string attributeName = attributeVector.getName(); - auto& old_config = attributeVector.getConfig(); - auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName); - if (current_config == nullptr) { - current_config = &old_config; - } - AttributeTransientMemoryCalculator get_transient_memory_usage; - size_t transient_memory_usage = get_transient_memory_usage(attributeVector, *current_config); - _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName); + _samplerContext->merge(usage, attributeName, _subDbName); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h index 963bd758494..d002173b205 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h @@ -9,9 +9,8 @@ namespace proton { class AttributeUsageSamplerContext; -/* - * Functor for sampling attribute usage and passing it on to sampler - * context. +/** + * Functor for sampling attribute usage and passing it on to sampler context. */ class AttributeUsageSamplerFunctor : public search::attribute::IConstAttributeFunctor { diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 219a2ea43a4..421e602a9cf 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -23,7 +23,6 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp - transient_resource_usage_provider.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h index fd50f46241c..5aeb84feb2c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h @@ -7,10 +7,10 @@ namespace proton { /** - * Interface class providing transient resource usage. + * Interface class providing a snapshot of transient resource usage. * - * E.g. extra memory needed for loading or saving an attribute vectors or extra disk needed for running disk index fusion. - * It provides an aggregated max view over several components (e.g. all attribute vectors for a document type). + * E.g. the memory used by the memory index and extra disk needed for running disk index fusion. + * This provides the total transient resource usage for the components this provider encapsulates. */ class ITransientResourceUsageProvider { public: diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp deleted file mode 100644 index 0d338cad9df..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "transient_resource_usage_provider.h" - -namespace proton { - -TransientResourceUsageProvider::TransientResourceUsageProvider() - : ITransientResourceUsageProvider(), - _transient_memory_usage(0u) -{ -} - -TransientResourceUsageProvider::~TransientResourceUsageProvider() = default; - -size_t -TransientResourceUsageProvider::get_transient_memory_usage() const -{ - return _transient_memory_usage.load(std::memory_order_relaxed); -} - -void -TransientResourceUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) -{ - _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h deleted file mode 100644 index 205e95d9fa6..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "i_transient_resource_usage_provider.h" - -#include <atomic> - -namespace proton { - -/** - * Class providing transient resource usage. - * E.g. extra memory needed for loading or saving an attribute vector. - * It provides an aggregated view over several components (e.g. all attribute vectors for a document type). - */ -class TransientResourceUsageProvider : public ITransientResourceUsageProvider { - std::atomic<size_t> _transient_memory_usage; -public: - TransientResourceUsageProvider(); - virtual ~TransientResourceUsageProvider(); - size_t get_transient_memory_usage() const override; - size_t get_transient_disk_usage() const override { return 0; } - void set_transient_memory_usage(size_t transient_memory_usage); -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp index d359583cb59..6307604598d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp @@ -94,6 +94,7 @@ void ResourceUsageTracker::notifyDiskMemUsage(DiskMemUsageState state) { std::lock_guard guard(_lock); + // TODO: Subtract transient resource (memory and disk) usage from the absolute numbers. _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_address_space_usage()); if (_listener != nullptr) { _listener->update_resource_usage(_resource_usage); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 3677b9165fb..27eb636b475 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -130,27 +130,33 @@ DiskMemUsageFilter::recalcState(const Guard &guard) _acceptWrite = true; } DiskMemUsageState dmstate(ResourceUsageState(_config._diskLimit, diskUsed), - ResourceUsageState(_config._memoryLimit, memoryUsed)); + ResourceUsageState(_config._memoryLimit, memoryUsed), + get_relative_transient_memory_usage(guard)); notifyDiskMemUsage(guard, dmstate); } double -DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const +DiskMemUsageFilter::getMemoryUsedRatio(const Guard&) const { - (void) guard; uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss(); return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memory().sizeBytes(); } double -DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const +DiskMemUsageFilter::getDiskUsedRatio(const Guard&) const { - (void) guard; double usedDiskSpaceRatio = static_cast<double>(_diskUsedSizeBytes) / static_cast<double>(_hwInfo.disk().sizeBytes()); return usedDiskSpaceRatio; } +double +DiskMemUsageFilter::get_relative_transient_memory_usage(const Guard&) const +{ + return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); +} + + DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), _hwInfo(hwInfo), @@ -189,6 +195,7 @@ DiskMemUsageFilter::set_transient_resource_usage(size_t transient_memory_usage, Guard guard(_lock); _transient_memory_usage = transient_memory_usage; _transient_disk_usage = transient_disk_usage; + recalcState(guard); } void @@ -220,13 +227,6 @@ DiskMemUsageFilter::get_transient_memory_usage() const return _transient_memory_usage; } -double -DiskMemUsageFilter::get_relative_transient_memory_usage() const -{ - Guard guard(_lock); - return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); -} - size_t DiskMemUsageFilter::get_transient_disk_usage() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index e2e94b1abd9..fb18a12084f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -15,9 +15,9 @@ namespace proton { -/* - * Class to filter write operations based on sampled disk and memory - * usage. If resource limit is reached then further writes are denied +/** + * Class to filter write operations based on sampled disk and memory usage. + * If resource limit is reached then further writes are denied * in order to prevent entering an unrecoverable state. */ class DiskMemUsageFilter : public IResourceWriteFilter, @@ -58,6 +58,7 @@ private: void recalcState(const Guard &guard); // called with _lock held double getMemoryUsedRatio(const Guard &guard) const; double getDiskUsedRatio(const Guard &guard) const; + double get_relative_transient_memory_usage(const Guard& guard) const; void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state); public: @@ -70,7 +71,6 @@ public: vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; size_t get_transient_memory_usage() const; - double get_relative_transient_memory_usage() const; size_t get_transient_disk_usage() const; double get_relative_transient_disk_usage() const; Config getConfig() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp index 60f45f536d6..562e20a864c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -15,7 +15,8 @@ DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) n : _disk_usage(usage_state.diskState().usage()), _disk_utilization(usage_state.diskState().utilization()), _memory_usage(usage_state.memoryState().usage()), - _memory_utilization(usage_state.memoryState().utilization()) + _memory_utilization(usage_state.memoryState().utilization()), + _transient_memory_usage(usage_state.transient_memory_usage()) { } @@ -26,6 +27,7 @@ DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); + _transient_memory_usage = std::max(_transient_memory_usage, usage_state.transient_memory_usage()); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h index 10687fd38a8..cb084b90a11 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -16,6 +16,7 @@ class DiskMemUsageMetrics double _disk_utilization; double _memory_usage; double _memory_utilization; + double _transient_memory_usage; public: DiskMemUsageMetrics() noexcept; @@ -25,6 +26,7 @@ public: double get_disk_utilization() const noexcept { return _disk_utilization; } double get_memory_usage() const noexcept { return _memory_usage; } double get_memory_utilization() const noexcept { return _memory_utilization; } + double get_transient_memory_usage() const noexcept { return _transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index d9f47c2944b..cfed4c0522a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -125,16 +125,16 @@ DiskMemUsageSampler::sampleMemoryUsage() void DiskMemUsageSampler::sample_transient_resource_usage() { - size_t max_transient_memory_usage = 0; + size_t transient_memory_usage_sum = 0; size_t max_transient_disk_usage = 0; { std::lock_guard<std::mutex> guard(_lock); for (auto provider : _transient_usage_providers) { - max_transient_memory_usage = std::max(max_transient_memory_usage, provider->get_transient_memory_usage()); + transient_memory_usage_sum += provider->get_transient_memory_usage(); max_transient_disk_usage = std::max(max_transient_disk_usage, provider->get_transient_disk_usage()); } } - _filter.set_transient_resource_usage(max_transient_memory_usage, max_transient_disk_usage); + _filter.set_transient_resource_usage(transient_memory_usage_sum, max_transient_disk_usage); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index 090c24b671e..769086574f0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -8,29 +8,35 @@ namespace proton { /** * Class used to describe state of disk and memory usage relative to configured limits. + * In addition relative transient memory usage is tracked. */ class DiskMemUsageState { ResourceUsageState _diskState; ResourceUsageState _memoryState; + double _transient_memory_usage; public: DiskMemUsageState() = default; DiskMemUsageState(const ResourceUsageState &diskState_, - const ResourceUsageState &memoryState_) + const ResourceUsageState &memoryState_, + double transient_memory_usage_ = 0) : _diskState(diskState_), - _memoryState(memoryState_) + _memoryState(memoryState_), + _transient_memory_usage(transient_memory_usage_) { } bool operator==(const DiskMemUsageState &rhs) const { return ((_diskState == rhs._diskState) && - (_memoryState == rhs._memoryState)); + (_memoryState == rhs._memoryState) && + (_transient_memory_usage == rhs._transient_memory_usage)); } bool operator!=(const DiskMemUsageState &rhs) const { return ! ((*this) == rhs); } const ResourceUsageState &diskState() const { return _diskState; } const ResourceUsageState &memoryState() const { return _memoryState; } + double transient_memory_usage() const { return _transient_memory_usage; } bool aboveDiskLimit(double resourceLimitFactor) const { return diskState().aboveLimit(resourceLimitFactor); } bool aboveMemoryLimit(double resourceLimitFactor) const { return memoryState().aboveLimit(resourceLimitFactor); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 53bdc356015..b9903d626aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -19,8 +19,8 @@ #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <vespa/searchcore/proton/common/statusreport.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/searchcore/proton/docsummary/isummarymanager.h> #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -89,17 +89,21 @@ public: } }; -class DocumentDBResourceUsageProvider : public TransientResourceUsageProvider { +class DocumentDBResourceUsageProvider : public ITransientResourceUsageProvider { private: const DocumentDB& _doc_db; public: - DocumentDBResourceUsageProvider(const DocumentDB& doc_db) + DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept : _doc_db(doc_db) {} + size_t get_transient_memory_usage() const override { + return _doc_db.getReadySubDB()->getSearchableStats().memoryUsage().allocatedBytes(); + } size_t get_transient_disk_usage() const override { // We estimate the transient disk usage for the next disk index fusion // as the size of the largest disk index. + // TODO: Change this to actually measure the size of the fusion disk index(es). return _doc_db.getReadySubDB()->getSearchableStats().max_component_size_on_disk(); } }; @@ -909,7 +913,7 @@ DocumentDB::hasDocument(const document::DocumentId &id) } void -DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector) +DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) { // Called by executor thread _maintenanceController.killJobs(); @@ -931,8 +935,6 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std _jobTrackers, _subDBs.getReadySubDB()->getAttributeManager(), _subDBs.getNotReadySubDB()->getAttributeManager(), - std::move(attribute_config_inspector), - _transient_usage_provider, _writeFilter); } @@ -954,9 +956,7 @@ DocumentDB::performStartMaintenance() return; } auto maintenanceConfig = activeConfig->getMaintenanceConfigSP(); - const auto &attributes_config = activeConfig->getAttributesConfig(); - auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); - injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); + injectMaintenanceJobs(*maintenanceConfig); _maintenanceController.start(maintenanceConfig); } @@ -973,11 +973,9 @@ DocumentDB::forwardMaintenanceConfig() DocumentDBConfig::SP activeConfig = getActiveConfig(); assert(activeConfig); auto maintenanceConfig(activeConfig->getMaintenanceConfigSP()); - const auto &attributes_config = activeConfig->getAttributesConfig(); - auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); if (!_state.getClosed()) { if (_maintenanceController.getPaused()) { - injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); + injectMaintenanceJobs(*maintenanceConfig); } _maintenanceController.newConfig(maintenanceConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index e829f477e8a..8e8391b2f31 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -52,7 +52,6 @@ class IDocumentDBOwner; class ISharedThreadingService; class ITransientResourceUsageProvider; class StatusReport; -class TransientResourceUsageProvider; struct MetricsWireService; namespace matching { class SessionManager; } @@ -117,7 +116,7 @@ private: DDBState _state; DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + std::shared_ptr<ITransientResourceUsageProvider> _transient_usage_provider; std::unique_ptr<FeedHandler> _feedHandler; DocumentSubDBCollection _subDBs; MaintenanceController _maintenanceController; @@ -384,7 +383,7 @@ public: /** * Implements IFeedHandlerOwner **/ - void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector); + void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); void performStartMaintenance(); void stopMaintenance(); void forwardMaintenanceConfig(); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index a832f00f911..f4b92876891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -9,7 +9,6 @@ #include "prune_session_cache_job.h" #include "pruneremoveddocumentsjob.h" #include "sample_attribute_usage_job.h" -#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> using vespalib::system_clock; @@ -87,8 +86,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, DocumentDBJobTrackers &jobTrackers, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); @@ -122,9 +119,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, controller.registerJobInMasterThread( std::make_unique<SampleAttributeUsageJob>(readyAttributeManager, notReadyAttributeManager, attributeUsageFilter, docTypeName, - config.getAttributeUsageSampleInterval(), - std::move(attribute_config_inspector), - transient_usage_provider)); + config.getAttributeUsageSampleInterval())); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 8d0b01465fb..1dcfafaa645 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -12,17 +12,15 @@ namespace storage::spi {struct BucketExecutor; } namespace proton { -class AttributeConfigInspector; -class IPruneRemovedDocumentsHandler; -struct IDocumentMoveHandler; +class AttributeUsageFilter; class IBucketModifiedHandler; -class IClusterStateChangedNotifier; class IBucketStateChangedNotifier; -struct IBucketStateCalculator; -struct IAttributeManager; -class AttributeUsageFilter; +class IClusterStateChangedNotifier; class IDiskMemUsageNotifier; -class TransientResourceUsageProvider; +class IPruneRemovedDocumentsHandler; +struct IAttributeManager; +struct IBucketStateCalculator; +struct IDocumentMoveHandler; namespace bucketdb { class IBucketCreateNotifier; } /** @@ -50,8 +48,6 @@ struct MaintenanceJobsInjector DocumentDBJobTrackers &jobTrackers, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index f697c4d4672..7c52d3d96ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -757,7 +757,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization()); metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); - metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); + metrics.resourceUsage.transient_memory.set(dm_metrics.get_transient_memory_usage()); metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 7dc96b92de9..e21e0366c4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -14,15 +14,11 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) + vespalib::duration interval) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), - _attributeUsageFilter(attributeUsageFilter), - _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_usage_provider(std::move(transient_usage_provider)) + _attributeUsageFilter(attributeUsageFilter) { } @@ -31,7 +27,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default; bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_usage_provider); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter); _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h index d72305496dd..9e42897f8e7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h @@ -7,9 +7,7 @@ namespace proton { struct IAttributeManager; -class AttributeConfigInspector; class AttributeUsageFilter; -class TransientResourceUsageProvider; /** * Class used to sample attribute resource usage and pass aggregated @@ -23,16 +21,13 @@ class SampleAttributeUsageJob : public IMaintenanceJob IAttributeManagerSP _readyAttributeManager; IAttributeManagerSP _notReadyAttributeManager; AttributeUsageFilter &_attributeUsageFilter; - std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + public: SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); + vespalib::duration interval); ~SampleAttributeUsageJob() override; bool run() override; |