Merge pull request #18885 from vespa-engine/bjorncs/jvm-heap-dump

Add helper script to generate JVM heap dump

3 files changed, 135 insertions, 0 deletions

diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt
index 433e3c799b5..75ec63674c4 100644
--- a/container-core/CMakeLists.txt
+++ b/container-core/CMakeLists.txt
@@ -3,3 +3,4 @@ install_config_definitions()
 install_java_artifact_dependencies(container-core)
 
 vespa_install_script(src/main/sh/vespa-load-balancer-status libexec/vespa)
+vespa_install_script(src/main/sh/vespa-jvm-dumper bin)
diff --git a/container-core/src/main/sh/vespa-jvm-dumper b/container-core/src/main/sh/vespa-jvm-dumper
new file mode 100755
index 00000000000..4710c9c9d9f
--- /dev/null
+++ b/container-core/src/main/sh/vespa-jvm-dumper
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    empty_if_start_slash=${mypath%%/*}
+    if [ "${empty_if_start_slash}" ]; then
+        mypath=$(pwd)/${mypath}
+    fi
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findhost () {
+    if [ "${VESPA_HOSTNAME}" = "" ]; then
+        VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1
+    fi
+    validate="${VESPA_HOME}/bin/vespa-validate-hostname"
+    if [ -f "$validate" ]; then
+        "$validate" "${VESPA_HOSTNAME}" || exit 1
+    fi
+    export VESPA_HOSTNAME
+}
+
+findroot
+findhost
+
+# END environment bootstrap section
+
+set -euo pipefail
+
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <id> <output-directory>"
+  echo ""
+  vespa-sentinel-cmd list 2>/dev/null
+  exit 1
+fi
+
+readonly SERVICE=$1
+readonly OUTPUT_DIRECTORY=$2
+if ! [ -d "${OUTPUT_DIRECTORY}" -a -w "${OUTPUT_DIRECTORY}" ]; then
+  echo "Directory '${OUTPUT_DIRECTORY}' is not writable"
+  exit 1
+fi
+readonly STATUS=$(vespa-sentinel-cmd list | grep "id=\"${SERVICE}\"")
+if [ -z "${STATUS}" ]; then
+  echo "No service named '${SERVICE}'"
+  exit 1
+else
+  echo "Found service: ${STATUS}"
+fi
+readonly JVM_PID=$(echo ${STATUS} | cut -d " "  -f 4 | cut -d "=" -f 2)
+if ! [[ "${JVM_PID}" =~ ^[0-9]+$ ]]; then
+  echo "Could not find valid pid for '${SERVICE}' (pid='${JVM_PID}')"
+  exit 1
+else
+  echo "Pid for '${SERVICE}' is '${JVM_PID}'"
+fi
+if ! [ -n $(ps -p ${JVM_PID} -o pid=) ]; then
+  echo "Could not find process for '${JVM_PID}'"
+  exit 1
+fi
+
+echo "Creating heap dump"
+readonly HEAP_DUMP_FILE=${OUTPUT_DIRECTORY}/heapdump.bin
+if test -f "${HEAP_DUMP_FILE}"; then
+    rm "${HEAP_DUMP_FILE}"
+fi
+jmap -dump:live,format=b,file=${HEAP_DUMP_FILE} ${JVM_PID}
+
+echo "Getting jmap information"
+jhsdb jmap  --heap --pid ${JVM_PID} > ${OUTPUT_DIRECTORY}/jmap-output
+
+echo "Getting jstat information"
+jstat -gcutil ${JVM_PID} > ${OUTPUT_DIRECTORY}/jstat-output
+
+echo "Getting jstack information"
+jstack ${JVM_PID} > ${OUTPUT_DIRECTORY}/jstack-output
+
+echo "Getting pmap information"
+pmap -x ${JVM_PID} | sort -nk3 | tail -10 > ${OUTPUT_DIRECTORY}/pmap-output
+
+echo "Copying vespa logs"
+cp ${VESPA_HOME}/logs/vespa/vespa.log ${OUTPUT_DIRECTORY}/vespa.log
+
+echo "Done!"
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 6bd5fc5c96e..3df5c3744d3 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -686,6 +686,7 @@ fi
 %endif
 %dir %{_prefix}
 %dir %{_prefix}/bin
+%{_prefix}/bin/vespa-jvm-dumper
 %{_prefix}/bin/vespa-logfmt
 %{_prefix}/bin/vespa-security-env
 %dir %{_prefix}/conf