Add feature flag for controlling memory limits for merge operations

7 files changed, 62 insertions, 4 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 3fd5af98f04..833e2f020bc 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -117,6 +117,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; }
         @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "warn"; }
         @ModelFeatureFlag(owners = {"hmusum"}) default int searchHandlerThreadpool() { return 2; }
+        @ModelFeatureFlag(owners = {"vekterli"}) default long mergingMaxMemoryUsagePerNode() { return -1; }
     }
 
     /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 1b35460523e..1bda8a509f1 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -87,6 +87,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private List<DataplaneToken> dataplaneTokens;
     private int contentLayerMetadataFeatureLevel = 0;
     private boolean dynamicHeapSize = false;
+    private long mergingMaxMemoryUsagePerNode = -1;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -146,6 +147,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; }
     @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; }
     @Override public boolean dynamicHeapSize() { return dynamicHeapSize; }
+    @Override public long mergingMaxMemoryUsagePerNode() { return mergingMaxMemoryUsagePerNode; }
 
     public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
         this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim;
@@ -383,6 +385,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
 
     public TestProperties setDynamicHeapSize(boolean b) { this.dynamicHeapSize = b; return this; }
 
+    public TestProperties setMergingMaxMemoryUsagePerNode(long maxUsage) {
+        this.mergingMaxMemoryUsagePerNode = maxUsage;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
index b88ab0c5a45..1865db0ec1c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
@@ -1,6 +1,7 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.content.storagecluster;
 
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.vespa.config.content.core.StorServerConfig;
 import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
@@ -10,10 +11,10 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
  */
 public class StorServerProducer implements StorServerConfig.Producer {
     public static class Builder {
-        StorServerProducer build(ModelElement element) {
+        StorServerProducer build(ModelContext.Properties properties, ModelElement element) {
             ModelElement tuning = element.child("tuning");
 
-            StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element));
+            StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags());
             if (tuning == null) return producer;
 
             ModelElement merges = tuning.child("merges");
@@ -28,6 +29,7 @@ public class StorServerProducer implements StorServerConfig.Producer {
     private final String clusterName;
     private Integer maxMergesPerNode;
     private Integer queueSize;
+    private Long mergingMaxMemoryUsagePerNode;
 
     private StorServerProducer setMaxMergesPerNode(Integer value) {
         if (value != null) {
@@ -42,8 +44,9 @@ public class StorServerProducer implements StorServerConfig.Producer {
         return this;
     }
 
-    StorServerProducer(String clusterName) {
+    StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) {
         this.clusterName = clusterName;
+        this.mergingMaxMemoryUsagePerNode = featureFlags.mergingMaxMemoryUsagePerNode();
     }
 
     @Override
@@ -60,5 +63,10 @@ public class StorServerProducer implements StorServerConfig.Producer {
         if (queueSize != null) {
             builder.max_merge_queue_size(queueSize);
         }
+        if (mergingMaxMemoryUsagePerNode != null) {
+            builder.merge_throttling_memory_limit(
+                    new StorServerConfig.Merge_throttling_memory_limit.Builder()
+                            .max_usage_bytes(mergingMaxMemoryUsagePerNode));
+        }
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java
index ce2899877a7..701da93a329 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java
@@ -35,7 +35,7 @@ public class StorageCluster extends TreeConfigProducer<StorageNode>
             return new StorageCluster(ancestor,
                                       ContentCluster.getClusterId(clusterElem),
                                       new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem),
-                                      new StorServerProducer.Builder().build(clusterElem),
+                                      new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem),
                                       new StorVisitorProducer.Builder().build(clusterElem),
                                       new PersistenceProducer.Builder().build(clusterElem));
         }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index e1c4620e9b7..e7b2c549fa5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -185,6 +185,36 @@ public class StorageClusterTest {
     }
 
     @Test
+    void merge_throttler_memory_limit_config_has_expected_defaults() {
+        var config = configFromProperties(new TestProperties());
+        var limit = config.merge_throttling_memory_limit();
+
+        assertEquals(-1L, limit.max_usage_bytes()); // TODO change default
+        assertMergeAutoScaleConfigHasExpectedValues(limit);
+    }
+
+    void assertMergeAutoScaleConfigHasExpectedValues(StorServerConfig.Merge_throttling_memory_limit limit) {
+        assertEquals(128L*1024*1024,    limit.auto_lower_bound_bytes());
+        assertEquals(2L*1024*1024*1024, limit.auto_upper_bound_bytes());
+        assertEquals(0.03,              limit.auto_phys_mem_scale_factor(), 0.000001);
+    }
+
+    @Test
+    void merge_throttler_memory_limit_is_controlled_by_feature_flag() {
+        var config = configFromProperties(new TestProperties().setMergingMaxMemoryUsagePerNode(-1));
+        assertEquals(-1L, config.merge_throttling_memory_limit().max_usage_bytes());
+
+        config = configFromProperties(new TestProperties().setMergingMaxMemoryUsagePerNode(0));
+        assertEquals(0L, config.merge_throttling_memory_limit().max_usage_bytes());
+
+        config = configFromProperties(new TestProperties().setMergingMaxMemoryUsagePerNode(1_234_456_789));
+        assertEquals(1_234_456_789L, config.merge_throttling_memory_limit().max_usage_bytes());
+
+        // Feature flag should not affect the other config values
+        assertMergeAutoScaleConfigHasExpectedValues(config.merge_throttling_memory_limit());
+    }
+
+    @Test
     void testVisitors() {
         StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder();
         parse(cluster("bees",
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 5ad8eee90e8..2564584a7df 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -209,6 +209,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean dynamicHeapSize;
         private final String unknownConfigDefinition;
         private final int searchHandlerThreadpool;
+        private final long mergingMaxMemoryUsagePerNode;
 
         public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
             this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -253,6 +254,7 @@ public class ModelContextImpl implements ModelContext {
             this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE);
             this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION);
             this.searchHandlerThreadpool = flagValue(source, appId, version, Flags.SEARCH_HANDLER_THREADPOOL);
+            this.mergingMaxMemoryUsagePerNode = flagValue(source, appId, version, Flags.MERGING_MAX_MEMORY_USAGE_PER_NODE);
         }
 
         @Override public int heapSizePercentage() { return heapPercentage; }
@@ -305,6 +307,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean dynamicHeapSize() { return dynamicHeapSize; }
         @Override public String unknownConfigDefinition() { return unknownConfigDefinition; }
         @Override public int searchHandlerThreadpool() { return searchHandlerThreadpool; }
+        @Override public long mergingMaxMemoryUsagePerNode() { return mergingMaxMemoryUsagePerNode; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 9c2669eebe0..d701a35c2ef 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -423,6 +423,15 @@ public class Flags {
             "Whether to send cloud trial email notifications",
             "Takes effect immediately");
 
+    public static final UnboundLongFlag MERGING_MAX_MEMORY_USAGE_PER_NODE = defineLongFlag(
+            "merging-max-memory-usage-per-node", -1,
+            List.of("vekterli"), "2023-11-03", "2024-03-01",
+            "Soft limit of the maximum amount of memory that can be used across merge operations on a content node. " +
+            "Value semantics: < 0: unlimited (legacy behavior), == 0: auto-deduced from node HW and config," +
+            " > 0: explicit memory usage limit in bytes.",
+            "Takes effect at redeployment",
+            INSTANCE_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,