Correctly account for node memory overhead when computing args for startup script

7 files changed, 35 insertions, 28 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
index a8085919a98..f87f1382ffb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
@@ -16,7 +16,7 @@ import java.util.Objects;
 public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> {
 
     // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy).
-    // Keep in sync with node-repository/ClusterModel.
+    // Keep in sync with node-repository/ClusterModel and startup scripts (go and bash).
     public static final double memoryOverheadGb = 0.7;
 
     private ConfigSentinel configSentinel = null;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java
index 9cf5fe84c21..4900b56801c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java
@@ -23,21 +23,22 @@ public class JvmHeapSizeValidator implements Validator {
         context.model().getContainerClusters().forEach((clusterId, appCluster) -> {
             var mp = appCluster.getMemoryPercentage().orElse(null);
             if (mp == null) return;
-            if (mp.availableMemoryGb().isEmpty()) {
+            if (mp.asAbsoluteGb().isEmpty()) {
                 context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'");
                 return;
             }
             long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes();
             if (jvmModelCost > 0) {
-                double availableMemoryGb = mp.availableMemoryGb().getAsDouble();
+                double availableMemoryGb = mp.asAbsoluteGb().getAsDouble();
+                int percentageOfTotal = mp.ofContainerTotal().getAsInt();
                 double modelCostGb = jvmModelCost / (1024D * 1024 * 1024);
                 context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB",
-                        mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb));
-                if (mp.percentage() < percentLimit) {
+                        percentageOfTotal, percentLimit, availableMemoryGb, gbLimit, modelCostGb));
+                if (percentageOfTotal < percentLimit) {
                     context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " +
                                     "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " +
                                     "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).",
-                            clusterId, mp.percentage(), percentLimit, modelCostGb));
+                            clusterId, percentageOfTotal, percentLimit, modelCostGb));
                 }
                 if (availableMemoryGb < gbLimit) {
                     context.illegal(
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java
index 008a3fc5547..e57110e44e5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java
@@ -115,7 +115,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida
         double memoryUsedByModels = currentModelCostInGb + nextModelCostInGb;
         double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - memoryUsedByModels);
 
-        var availableMemoryPercentage = cluster.availableMemoryPercentage();
+        var availableMemoryPercentage = cluster.heapSizePercentageOfAvailable();
         int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage);
 
         var prefix = "Validating Onnx models memory usage for %s".formatted(cluster);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index ed7646b3066..5a86ebfd7eb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -209,22 +209,27 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage));
 
         if (isHostedVespa()) {
-            int availableMemoryPercentage = availableMemoryPercentage();
-            if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known
-
-            // Node memory is known so convert available memory percentage to node memory percentage
-            double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow();
-            double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024);
-            double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb);
-            int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage);
-            logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f"
-                    .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb));
-            return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory));
+            int heapSizePercentageOfAvailable = heapSizePercentageOfAvailable();
+            if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(heapSizePercentageOfAvailable)); // Node memory is not known
+
+            // Node memory is known so convert available memory ofContainerAvailable to node memory ofContainerAvailable
+            double totalMemoryGb = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow();
+            double totalMemoryMinusOverhead = Math.max(0, totalMemoryGb - Host.memoryOverheadGb);
+            double onnxModelCostGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024);
+            double availableMemoryGb = Math.max(0, totalMemoryMinusOverhead - onnxModelCostGb);
+            int memoryPercentageOfAvailable = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryMinusOverhead);
+            int memoryPercentageOfTotal = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryGb);
+            logger.log(FINE, () -> ("cluster id '%s': memoryPercentageOfAvailable=%d, memoryPercentageOfTotal=%d, " +
+                                    "availableMemoryGb=%f, totalMemoryGb=%f, heapSizePercentageOfAvailable=%d, onnxModelCostGb=%f")
+                    .formatted(id(), memoryPercentageOfAvailable, memoryPercentageOfTotal,
+                               availableMemoryGb, totalMemoryGb, heapSizePercentageOfAvailable, onnxModelCostGb));
+            return Optional.of(JvmMemoryPercentage.of(memoryPercentageOfAvailable, memoryPercentageOfTotal,
+                                                      availableMemoryGb * heapSizePercentageOfAvailable * 1e-2));
         }
         return Optional.empty();
     }
 
-    public int availableMemoryPercentage() {
+    public int heapSizePercentageOfAvailable() {
         return getHostClusterId().isPresent() ?
                 heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster :
                 heapSizePercentageOfAvailableMemory;
@@ -310,14 +315,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
     public void getConfig(QrStartConfig.Builder builder) {
         super.getConfig(builder);
         var memoryPct = getMemoryPercentage().orElse(null);
-        int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent()
-                ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536;
+        int heapsize = memoryPct != null && memoryPct.asAbsoluteGb().isPresent()
+                       ? (int) (memoryPct.asAbsoluteGb().getAsDouble() * 1024) : 1536;
         builder.jvm.verbosegc(true)
                 .availableProcessors(0)
                 .compressedClassSpaceSize(0)
                 .minHeapsize(heapsize)
                 .heapsize(heapsize);
-        if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage());
+        if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.ofContainerAvailable());
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index aeb6c030a49..00ab47e8ddb 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -73,6 +73,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalDouble;
+import java.util.OptionalInt;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -721,10 +722,10 @@ public abstract class ContainerCluster<CONTAINER extends Container>
      * Returns the percentage of host physical memory this application has specified for nodes in this cluster,
      * or empty if this is not specified by the application.
      */
-    public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) {
-        static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); }
-        static JvmMemoryPercentage of(int percentage, double availableMemoryGb) {
-            return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb));
+    public record JvmMemoryPercentage(int ofContainerAvailable, OptionalInt ofContainerTotal, OptionalDouble asAbsoluteGb) { // optionalInt pctOfTotal < int pctOfAvailable
+        static JvmMemoryPercentage of(int percentageOfAvailable) { return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.empty(), OptionalDouble.empty()); }
+        static JvmMemoryPercentage of(int percentageOfAvailable, int percentageOfTotal, double absoluteMemoryGb) {
+            return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.of(percentageOfTotal), OptionalDouble.of(absoluteMemoryGb));
         }
     }
     public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 5bcd21a5b9b..9841ffe760c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -1040,7 +1040,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         }
         catch (NumberFormatException e) {
             throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster +
-                                               " must be an integer percentage ending by the '%' sign", e);
+                                               " must be given as an integer followe by '%'", e);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index bac86e37e8f..1ebf27734bd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -247,7 +247,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
             for (ContainerModel containerModel : containers) {
                 Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId();
                 if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) {
-                    return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01;
+                    return containerModel.getCluster().getMemoryPercentage().get().ofContainerAvailable() * 0.01;
                 }
             }
             return 0.0;