From afa904a588acbca603b3f6e958451026902cb5d6 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 15 Nov 2022 16:20:59 +0100 Subject: - If less than 16G memory only use single flush thread. - No more than 1 flushtread per 8 vcpus. --- .../java/com/yahoo/vespa/model/search/NodeResourcesTuning.java | 5 ++++- .../com/yahoo/vespa/model/search/NodeResourcesTuningTest.java | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index bc5de959f3f..35211a6a45d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -17,7 +17,8 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY = 0.02; private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04; private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; - private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 12.0; + private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 16.0; + private final static double MAX_FLUSH_TREAD_RATIO = 1.0/8; private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; @@ -96,6 +97,8 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { if (usableMemoryGb() < MIN_MEMORY_PER_FLUSH_THREAD_GB) { builder.maxconcurrent(1); } + builder.maxconcurrent(Math.min(builder.build().maxconcurrent(), + Math.max(1, (int)Math.ceil(resources.vcpu()*MAX_FLUSH_TREAD_RATIO)))); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 861482ab70c..d0fa205237e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -168,10 +168,15 @@ public class NodeResourcesTuningTest { assertSharedDisk(true, true); } + private static ProtonConfig fromMemAndCpu(int gb, int vcpu) { + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(gb).minCpuCores(vcpu)); + } @Test public void require_that_concurrent_flush_threads_is_1_with_low_memory() { - assertEquals(2, configFromMemorySetting(13, 0).flush().maxconcurrent()); - assertEquals(1, configFromMemorySetting(11, 0).flush().maxconcurrent()); + assertEquals(2, fromMemAndCpu(17, 9).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(17, 8).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) { -- cgit v1.2.3 From a515d38535ac26411f0b947f5eb85cb15759e360 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 15 Nov 2022 17:08:05 +0100 Subject: Update config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java Co-authored-by: Geir Storli --- .../src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 35211a6a45d..a5260303efe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -18,7 +18,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04; private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 16.0; - private final static double MAX_FLUSH_TREAD_RATIO = 1.0/8; + private final static double MAX_FLUSH_THREAD_RATIO = 1.0/8; private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; -- cgit v1.2.3 From 07b0309d59f46590dbce87076e1393aaaa9bab07 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 15 Nov 2022 17:08:13 +0100 Subject: Update config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java Co-authored-by: Geir Storli --- .../src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index a5260303efe..0cf361c10dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -98,7 +98,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { builder.maxconcurrent(1); } builder.maxconcurrent(Math.min(builder.build().maxconcurrent(), - Math.max(1, (int)Math.ceil(resources.vcpu()*MAX_FLUSH_TREAD_RATIO)))); + Math.max(1, (int)Math.ceil(resources.vcpu()*MAX_FLUSH_THREAD_RATIO)))); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { -- cgit v1.2.3