From 853af98aee0cb4d8c33e35c65e1831a92dc5907c Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 3 Feb 2022 14:49:05 +0000 Subject: Use a conservative flush strategy before we reach the proton resource limits for memory and disk. This should improve protons ability to have enough headroom to handle transient resource usage (memory index and disk index fusion) before reaching feed blocked inside proton. --- .../memory_flush_config_updater_test.cpp | 28 +++++++++++++++++++--- searchcore/src/vespa/searchcore/config/proton.def | 4 ++++ .../proton/server/memory_flush_config_updater.cpp | 5 +++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index 7ff59a9f41a..d544f1cb1ab 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -11,6 +11,7 @@ ProtonConfig::Flush::Memory getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize, double conservativeMemoryLimitFactor = 0.5, double conservativeDiskLimitFactor = 0.6, + double high_watermark_factor = 0.9, double lowWatermarkFactor = 0.8) { ProtonConfig::Flush::Memory result; @@ -19,6 +20,7 @@ getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize, result.maxtlssize = maxTlsSize; result.conservative.memorylimitfactor = conservativeMemoryLimitFactor; result.conservative.disklimitfactor = conservativeDiskLimitFactor; + result.conservative.highwatermarkfactor = high_watermark_factor; result.conservative.lowwatermarkfactor = lowWatermarkFactor; return result; } @@ -32,14 +34,16 @@ getDefaultConfig() ResourceUsageState aboveLimit() { - return ResourceUsageState(0.7, 0.8); + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + return ResourceUsageState(0.7, 0.64); } ResourceUsageState belowLimit() { - // This is still over default low watermark of 0.56 (0.7 * 0.8) - return ResourceUsageState(0.7, 0.6); + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + // This is still over the low watermark limit of 0.56 (0.7 * 0.8 (factor)). + return ResourceUsageState(0.7, 0.62); } const HwInfo::Memory defaultMemory(8_Gi); @@ -175,6 +179,24 @@ TEST_F("require that last config if remembered when setting new disk/memory usag TEST_DO(f.assertStrategyConfig(6, 3, 18)); } +TEST_F("Use conservative settings when above high watermark for disk usage", Fixture) +{ + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.62), belowLimit()); + TEST_DO(f.assertStrategyConfig(4, 1, 20)); + f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.64), belowLimit()); + TEST_DO(f.assertStrategyConfig(4, 1, 12)); +} + +TEST_F("Use conservative settings when above high watermark for memory usage", Fixture) +{ + // The high watermark limit is 0.54 (0.6 * 0.9 (factor)). + f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.53)); + TEST_DO(f.assertStrategyConfig(4, 1, 20)); + f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.55)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); +} + TEST_F("require that we must go below low watermark for disk usage before using normal tls size value again", Fixture) { f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.8), belowLimit()); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index be581040af6..81f8b49eb49 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -81,6 +81,10 @@ flush.memory.conservative.memorylimitfactor double default=0.5 ## In this case this factor is multiplied with 'maxtlssize' to calculate a conservative value to use instead. flush.memory.conservative.disklimitfactor double default=0.5 +## The factor used to multiply with the resource limits for disk / memory to find the high +## watermark indicating when to from normal into conservative mode for the flush strategy. +flush.memory.conservative.highwatermarkfactor double default=0.95 + ## The factor used to multiply with the resource limits for disk / memory to find the low ## watermark indicating when to go back from conservative to normal mode for the flush strategy. flush.memory.conservative.lowwatermarkfactor double default=0.9 diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp index 51243b718ec..50a499c8a73 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp @@ -13,9 +13,10 @@ namespace { bool shouldUseConservativeMode(const ResourceUsageState &resourceState, bool currentlyUseConservativeMode, + double high_watermark_factor, double lowWatermarkFactor) { - return resourceState.aboveLimit() || + return resourceState.aboveLimit(high_watermark_factor) || (currentlyUseConservativeMode && resourceState.aboveLimit(lowWatermarkFactor)); } @@ -25,6 +26,7 @@ void MemoryFlushConfigUpdater::considerUseConservativeDiskMode(const LockGuard &guard, MemoryFlush::Config &newConfig) { if (shouldUseConservativeMode(_currState.diskState(), _useConservativeDiskMode, + _currConfig.conservative.highwatermarkfactor, _currConfig.conservative.lowwatermarkfactor)) { newConfig.maxGlobalTlsSize = _currConfig.maxtlssize * _currConfig.conservative.disklimitfactor; @@ -41,6 +43,7 @@ void MemoryFlushConfigUpdater::considerUseConservativeMemoryMode(const LockGuard &, MemoryFlush::Config &newConfig) { if (shouldUseConservativeMode(_currState.memoryState(), _useConservativeMemoryMode, + _currConfig.conservative.highwatermarkfactor, _currConfig.conservative.lowwatermarkfactor)) { newConfig.maxGlobalMemory = _currConfig.maxmemory * _currConfig.conservative.memorylimitfactor; -- cgit v1.2.3