Report jstack for java cores

2 files changed, 75 insertions, 14 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index 2cddee6aa2a..6e5f65a5d48 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -77,10 +77,23 @@ public class CoreCollector {
     List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
         String threads = allThreads ? "thread apply all bt" : "bt";
         String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
+
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
-        if (result.getExitStatus() != 0) {
+        if (result.getExitStatus() != 0)
             throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command));
-        }
+
+        return Arrays.asList(result.getOutput().split("\n"));
+    }
+
+    List<String> readJstack(NodeAgentContext context, Path coredumpPath, Path binPath) {
+        String[] command = isRunningVespa6(context) ?
+                new String[] {"jstack", binPath.toString(), coredumpPath.toString()} :
+                new String[] {"jhsdb", "jstack", "--exe", binPath.toString(), "--core", coredumpPath.toString()};
+
+        ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
+        if (result.getExitStatus() != 0)
+            throw new RuntimeException("Failed to read jstack " + result + ", Command: " + Arrays.toString(command));
+
         return Arrays.asList(result.getOutput().split("\n"));
     }
 
@@ -96,11 +109,19 @@ public class CoreCollector {
             Path binPath = readBinPath(context, coredumpPath);
 
             data.put("bin_path", binPath.toString());
-            data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false));
-            data.put("backtrace_all_threads", readBacktrace(context, coredumpPath, binPath, true));
+            if (binPath.getFileName().toString().equals("java")) {
+                data.put("backtrace_all_threads", readJstack(context, coredumpPath, binPath));
+            } else {
+                data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false));
+                data.put("backtrace_all_threads", readBacktrace(context, coredumpPath, binPath, true));
+            }
         } catch (RuntimeException e) {
             context.log(logger, Level.WARNING, "Failed to extract backtrace", e);
         }
         return data;
     }
+
+    private static boolean isRunningVespa6(NodeAgentContext context) {
+        return context.node().wantedVespaVersion().map(v -> v.getMajor() == 6).orElse(false);
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index d809d9cbf96..37156ade064 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
+import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
@@ -9,9 +10,6 @@ import org.junit.Test;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -25,13 +23,14 @@ import static org.mockito.Mockito.when;
  */
 public class CoreCollectorTest {
     private final String GDB_PATH = "/my/path/to/gdb";
+    private final String JDK_PATH = "/path/to/jdk/java";
     private final DockerOperations docker = mock(DockerOperations.class);
     private final CoreCollector coreCollector = new CoreCollector(docker, Paths.get(GDB_PATH));
     private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
 
     private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234");
     private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program");
-    private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]",
+    private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]",
             "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t    printf(argv[3]);",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4");
@@ -127,10 +126,10 @@ public class CoreCollectorTest {
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
 
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE));
-        expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE));
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", TEST_BIN_PATH.toString(),
+                "backtrace", GDB_BACKTRACE,
+                "backtrace_all_threads", GDB_BACKTRACE);
         assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
@@ -142,16 +141,57 @@ public class CoreCollectorTest {
         mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
                 "", "Failure");
 
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
+        Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString());
         assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
+    @Test
+    public void reportsJstackInsteadOfGdbForJdkCores() {
+        mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
+                "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)");
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+                "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.");
+
+        String jstack = "jstack11";
+        mockExec(new String[]{"jhsdb", "jstack", "--exe", JDK_PATH, "--core", "/tmp/core.1234"},
+                jstack);
+
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", JDK_PATH,
+                "backtrace_all_threads", List.of(jstack));
+        assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
+    }
+
+    @Test
+    public void reportsJstackInsteadOfGdbForJdkCoresVespa6() {
+        NodeAgentContext contextVespa6 = new NodeAgentContextImpl.Builder("container-123.domain.tld")
+                .nodeSpecBuilder(n -> n.wantedVespaVersion(Version.fromString("6.330.51")))
+                .build();
+
+        mockExec(contextVespa6, new String[]{"file", TEST_CORE_PATH.toString()},
+                "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)", "");
+        mockExec(contextVespa6, new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+                "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.", "");
+
+        String jstack = "jstack8";
+        mockExec(contextVespa6, new String[]{"jstack", JDK_PATH, "/tmp/core.1234"},
+                jstack, "");
+
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", JDK_PATH,
+                "backtrace_all_threads", List.of(jstack));
+        assertEquals(expectedData, coreCollector.collect(contextVespa6, TEST_CORE_PATH));
+    }
+
     private void mockExec(String[] cmd, String output) {
         mockExec(cmd, output, "");
     }
 
     private void mockExec(String[] cmd, String output, String error) {
+        mockExec(context, cmd, output, error);
+    }
+
+    private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) {
         when(docker.executeCommandInContainerAsRoot(context, cmd))
                 .thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
     }