diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/vespa/model')
4 files changed, 35 insertions, 30 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 047a6ef9bd5..581f20cbfe9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -8,13 +8,17 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import java.util.Objects; /** - * A physical host, running a set of services. + * A node with an identity, with some dedicated compute resources, running a set of services. * The identity of a host is its hostname. Hosts are comparable on their host name. * * @author gjoranv */ public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> { + // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy). + // Keep in sync with node-repository/ClusterModel. + public static final double memoryOverheadGb = 0.7; + private ConfigSentinel configSentinel = null; private final String hostname; private final boolean runsConfigServer; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 6977a5ca465..3c1c4867f13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; @@ -75,8 +76,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); - public static final int defaultHeapSizePercentageOfTotalNodeMemory = 70; - public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; + public static final int defaultHeapSizePercentageOfAvailableMemory = 85; + public static final int heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster = 24; private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -91,7 +92,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private int zookeeperSessionTimeoutSeconds = 30; private final int transport_events_before_wakeup; private final int transport_connections_per_target; - private final int heapSizePercentageOfTotalNodeMemory; + + /** The heap size % of total memory available to the JVM process. */ + private final int heapSizePercentageOfAvailableMemory; private Integer memoryPercentage = null; @@ -119,9 +122,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addTestrunnerComponentsIfTester(deployState); transport_connections_per_target = deployState.featureFlags().mbusJavaRpcNumTargets(); transport_events_before_wakeup = deployState.featureFlags().mbusJavaEventsBeforeWakeup(); - heapSizePercentageOfTotalNodeMemory = deployState.featureFlags().heapSizePercentage() > 0 + heapSizePercentageOfAvailableMemory = deployState.featureFlags().heapSizePercentage() > 0 ? Math.min(99, deployState.featureFlags().heapSizePercentage()) - : defaultHeapSizePercentageOfTotalNodeMemory; + : defaultHeapSizePercentageOfAvailableMemory; } @Override @@ -178,12 +181,18 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public Optional<Integer> getMemoryPercentage() { - if (memoryPercentage != null) { - return Optional.of(memoryPercentage); - } else if (isHostedVespa()) { - return getHostClusterId().isPresent() ? - Optional.of(heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster) : - Optional.of(heapSizePercentageOfTotalNodeMemory); + if (memoryPercentage != null) return Optional.of(memoryPercentage); + + if (isHostedVespa()) { + int availableMemoryPercentage = getHostClusterId().isPresent() ? + heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : + heapSizePercentageOfAvailableMemory; + if (getContainers().isEmpty()) return Optional.of(availableMemoryPercentage); // Node memory is not known + + // Node memory is known so convert available memory percentage to node memory percentage + double totalMemory = getContainers().get(0).getHostResource().realResources().memoryGb(); + double availableMemory = totalMemory - Host.memoryOverheadGb; + return Optional.of((int) (availableMemory / totalMemory * availableMemoryPercentage)); } return Optional.empty(); } @@ -289,9 +298,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .compressedClassSpaceSize(0) .minHeapsize(1536) .heapsize(1536); - if (getMemoryPercentage().isPresent()) { - builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); - } + getMemoryPercentage().ifPresent(percentage -> builder.jvm.heapSizeAsPercentageOfPhysicalMemory(percentage)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e1d222e0546..b0b9125603e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -905,22 +905,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private static boolean applyMemoryPercentage(ApplicationContainerCluster cluster, String memoryPercentage) { - if (memoryPercentage == null || memoryPercentage.isEmpty()) return false; - memoryPercentage = memoryPercentage.trim(); - - if ( ! memoryPercentage.endsWith("%")) - throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign"); - memoryPercentage = memoryPercentage.substring(0, memoryPercentage.length()-1).trim(); - try { + if (memoryPercentage == null || memoryPercentage.isEmpty()) return false; + memoryPercentage = memoryPercentage.trim(); + if ( ! memoryPercentage.endsWith("%")) + throw new IllegalArgumentException("Missing % sign"); + memoryPercentage = memoryPercentage.substring(0, memoryPercentage.length()-1).trim(); cluster.setMemoryPercentage(Integer.parseInt(memoryPercentage)); + return true; } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign"); + " must be an integer percentage ending by the '%' sign", e); } - return true; } /** Allocate a container cluster without a nodes tag */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index b444de5fb14..1ad99404823 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.Host; import static java.lang.Long.min; import static java.lang.Long.max; @@ -27,10 +28,6 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final int threadsPerSearch; private final double fractionOfMemoryReserved; - // Memory for other processes running on the node (config-proxy, metrics-proxy). - // Keep in sync with node-repository/ClusterModel - public static final double nodeMemoryOverheadGb = 0.7; - public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, double fractionOfMemoryReserved) { @@ -129,7 +126,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { - double usableMemoryGb = resources.memoryGb() - nodeMemoryOverheadGb; + double usableMemoryGb = resources.memoryGb() - Host.memoryOverheadGb; return usableMemoryGb * (1 - fractionOfMemoryReserved); } |