diff options
| -rwxr-xr-x | container-disc/src/main/sh/vespa-start-container-daemon.sh | 23 | ||||
| -rwxr-xr-x | vespabase/src/common-env.sh | 27 | ||||
| -rwxr-xr-x | vespabase/src/start-cbinaries.sh | 18 |
3 files changed, 38 insertions, 30 deletions
diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index a0cbd9d9186..223124c29d6 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -110,8 +110,27 @@ configure_cpu() { } configure_numactl() { - numactlcmd=$(get_numa_ctl_cmd) - log_message debug "starting ${VESPA_SERVICE_NAME} for ${VESPA_CONFIG_ID} with numactl command : $numactlcmd" + log_message debug "starting ${VESPA_SERVICE_NAME} for ${VESPA_CONFIG_ID}" + if numactl --interleave all true &> /dev/null; then + # We are allowed to use numactl + numnodes=$(numactl --hardware | + grep available | + awk '$3 == "nodes" { print $2 }') + if [ "$VESPA_AFFINITY_CPU_SOCKET" ] && + [ "$numnodes" -gt 1 ] + then + node=$(($VESPA_AFFINITY_CPU_SOCKET % $numnodes)) + log_message debug "with affinity to $VESPA_AFFINITY_CPU_SOCKET out of $numnodes cpu sockets" + numactlcmd="numactl --cpunodebind=$node --membind=$node" + else + log_message debug "with memory interleaving on all nodes" + numactlcmd="numactl --interleave all" + fi + else + log_message debug "without numactl (no permission or not available)" + numactlcmd="" + fi + log_message debug "numactlcmd: $numactlcmd" } configure_gcopts() { diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index d5f8381432f..eb60154071c 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -292,30 +292,3 @@ log_debug_message () { log_warning_message () { log_message "warning" "$*" 1>&2 } - -get_numa_ctl_cmd () { - if ! type numactl &> /dev/null; then - echo "FATAL: Could not find required program numactl." - exit 1 - fi - - numnodes=$(numactl --hardware 2>/dev/null | - grep available | - awk '$3 == "nodes" { print $2 }') - - if [ -n "$numanodes" ]; then - # We are allowed to use numactl and have NUMA nodes - if [ "$VESPA_AFFINITY_CPU_SOCKET" ] && - [ "$numnodes" -gt 1 ] - then - node=$(($VESPA_AFFINITY_CPU_SOCKET % $numnodes)) - numactlcmd="numactl --cpunodebind=$node --membind=$node" - else - numactlcmd="numactl --interleave all" - fi - else - numactlcmd="" - fi - - echo $numactlcmd -} diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh index 1ef45a71dec..f17829aa081 100755 --- a/vespabase/src/start-cbinaries.sh +++ b/vespabase/src/start-cbinaries.sh @@ -163,13 +163,29 @@ configure_vespa_malloc () { fi } +configure_numa_ctl () { + numactl="" + if numactl --interleave all true &> /dev/null; then + # We are allowed to use numactl + numactl="numactl --interleave all" + if [ "$VESPA_AFFINITY_CPU_SOCKET" ]; then + numcpu=`numactl --hardware 2>/dev/null | grep available | cut -d' ' -f2` + if [ "$numcpu" ] && [ "$numcpu" -gt 1 ]; then + log_debug_message "Starting $0 with affinity $VESPA_AFFINITY_CPU_SOCKET out of $numcpu" + node=$(($VESPA_AFFINITY_CPU_SOCKET % $numcpu)) + numactl="numactl --cpunodebind=$node --membind=$node" + fi + fi + fi +} + configure_valgrind configure_huge_pages configure_use_madvise configure_vespa_malloc if $no_valgrind ; then - numactl=$(get_numa_ctl_cmd) + configure_numa_ctl ulimit -c unlimited log_debug_message "Starting $0 with : " \ $numactl env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" |