Use a conservative flush strategy before we reach the proton resource limits for memory and disk.

This should improve protons ability to have enough headroom to handle transient resource usage
(memory index and disk index fusion) before reaching feed blocked inside proton.

3 files changed, 33 insertions, 4 deletions

diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
index 7ff59a9f41a..d544f1cb1ab 100644
--- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
+++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp
@@ -11,6 +11,7 @@ ProtonConfig::Flush::Memory
 getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize,
           double conservativeMemoryLimitFactor = 0.5,
           double conservativeDiskLimitFactor = 0.6,
+          double high_watermark_factor = 0.9,
           double lowWatermarkFactor = 0.8)
 {
     ProtonConfig::Flush::Memory result;
@@ -19,6 +20,7 @@ getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize,
     result.maxtlssize = maxTlsSize;
     result.conservative.memorylimitfactor = conservativeMemoryLimitFactor;
     result.conservative.disklimitfactor = conservativeDiskLimitFactor;
+    result.conservative.highwatermarkfactor = high_watermark_factor;
     result.conservative.lowwatermarkfactor = lowWatermarkFactor;
     return result;
 }
@@ -32,14 +34,16 @@ getDefaultConfig()
 ResourceUsageState
 aboveLimit()
 {
-    return ResourceUsageState(0.7, 0.8);
+    // The high watermark limit is 0.63 (0.7 * 0.9 (factor)).
+    return ResourceUsageState(0.7, 0.64);
 }
 
 ResourceUsageState
 belowLimit()
 {
-    // This is still over default low watermark of 0.56 (0.7 * 0.8)
-    return ResourceUsageState(0.7, 0.6);
+    // The high watermark limit is 0.63 (0.7 * 0.9 (factor)).
+    // This is still over the low watermark limit of 0.56 (0.7 * 0.8 (factor)).
+    return ResourceUsageState(0.7, 0.62);
 }
 
 const HwInfo::Memory defaultMemory(8_Gi);
@@ -175,6 +179,24 @@ TEST_F("require that last config if remembered when setting new disk/memory usag
     TEST_DO(f.assertStrategyConfig(6, 3, 18));
 }
 
+TEST_F("Use conservative settings when above high watermark for disk usage", Fixture)
+{
+    // The high watermark limit is 0.63 (0.7 * 0.9 (factor)).
+    f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.62), belowLimit());
+    TEST_DO(f.assertStrategyConfig(4, 1, 20));
+    f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.64), belowLimit());
+    TEST_DO(f.assertStrategyConfig(4, 1, 12));
+}
+
+TEST_F("Use conservative settings when above high watermark for memory usage", Fixture)
+{
+    // The high watermark limit is 0.54 (0.6 * 0.9 (factor)).
+    f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.53));
+    TEST_DO(f.assertStrategyConfig(4, 1, 20));
+    f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.55));
+    TEST_DO(f.assertStrategyConfig(2, 0, 20));
+}
+
 TEST_F("require that we must go below low watermark for disk usage before using normal tls size value again", Fixture)
 {
     f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.8), belowLimit());
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index be581040af6..81f8b49eb49 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -81,6 +81,10 @@ flush.memory.conservative.memorylimitfactor double default=0.5
 ## In this case this factor is multiplied with 'maxtlssize' to calculate a conservative value to use instead.
 flush.memory.conservative.disklimitfactor double default=0.5
 
+## The factor used to multiply with the resource limits for disk / memory to find the high
+## watermark indicating when to from normal into conservative mode for the flush strategy.
+flush.memory.conservative.highwatermarkfactor double default=0.95
+
 ## The factor used to multiply with the resource limits for disk / memory to find the low
 ## watermark indicating when to go back from conservative to normal mode for the flush strategy.
 flush.memory.conservative.lowwatermarkfactor double default=0.9
diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp
index 51243b718ec..50a499c8a73 100644
--- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp
@@ -13,9 +13,10 @@ namespace {
 bool
 shouldUseConservativeMode(const ResourceUsageState &resourceState,
                           bool currentlyUseConservativeMode,
+                          double high_watermark_factor,
                           double lowWatermarkFactor)
 {
-    return resourceState.aboveLimit() ||
+    return resourceState.aboveLimit(high_watermark_factor) ||
            (currentlyUseConservativeMode && resourceState.aboveLimit(lowWatermarkFactor));
 }
 
@@ -25,6 +26,7 @@ void
 MemoryFlushConfigUpdater::considerUseConservativeDiskMode(const LockGuard &guard, MemoryFlush::Config &newConfig)
 {
     if (shouldUseConservativeMode(_currState.diskState(), _useConservativeDiskMode,
+                                  _currConfig.conservative.highwatermarkfactor,
                                   _currConfig.conservative.lowwatermarkfactor))
     {
         newConfig.maxGlobalTlsSize = _currConfig.maxtlssize * _currConfig.conservative.disklimitfactor;
@@ -41,6 +43,7 @@ void
 MemoryFlushConfigUpdater::considerUseConservativeMemoryMode(const LockGuard &, MemoryFlush::Config &newConfig)
 {
     if (shouldUseConservativeMode(_currState.memoryState(), _useConservativeMemoryMode,
+                                  _currConfig.conservative.highwatermarkfactor,
                                   _currConfig.conservative.lowwatermarkfactor))
     {
         newConfig.maxGlobalMemory = _currConfig.maxmemory * _currConfig.conservative.memorylimitfactor;