Merge pull request #10123 from vespa-engine/freva/jstack

[VESPA-14408] Report jstack for java cores

7 files changed, 82 insertions, 30 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 167ca15bdbf..f4355ed3afa 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -86,7 +86,7 @@ public class StorageMaintainer {
             throw new RuntimeException("Result from disk usage command not as expected: " + output);
         }
 
-        return 1024 * Long.valueOf(results[0]);
+        return 1024 * Long.parseLong(results[0]);
     }
 
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index 2cddee6aa2a..6e5f65a5d48 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -77,10 +77,23 @@ public class CoreCollector {
     List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
         String threads = allThreads ? "thread apply all bt" : "bt";
         String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
+
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
-        if (result.getExitStatus() != 0) {
+        if (result.getExitStatus() != 0)
             throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command));
-        }
+
+        return Arrays.asList(result.getOutput().split("\n"));
+    }
+
+    List<String> readJstack(NodeAgentContext context, Path coredumpPath, Path binPath) {
+        String[] command = isRunningVespa6(context) ?
+                new String[] {"jstack", binPath.toString(), coredumpPath.toString()} :
+                new String[] {"jhsdb", "jstack", "--exe", binPath.toString(), "--core", coredumpPath.toString()};
+
+        ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
+        if (result.getExitStatus() != 0)
+            throw new RuntimeException("Failed to read jstack " + result + ", Command: " + Arrays.toString(command));
+
         return Arrays.asList(result.getOutput().split("\n"));
     }
 
@@ -96,11 +109,19 @@ public class CoreCollector {
             Path binPath = readBinPath(context, coredumpPath);
 
             data.put("bin_path", binPath.toString());
-            data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false));
-            data.put("backtrace_all_threads", readBacktrace(context, coredumpPath, binPath, true));
+            if (binPath.getFileName().toString().equals("java")) {
+                data.put("backtrace_all_threads", readJstack(context, coredumpPath, binPath));
+            } else {
+                data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false));
+                data.put("backtrace_all_threads", readBacktrace(context, coredumpPath, binPath, true));
+            }
         } catch (RuntimeException e) {
             context.log(logger, Level.WARNING, "Failed to extract backtrace", e);
         }
         return data;
     }
+
+    private static boolean isRunningVespa6(NodeAgentContext context) {
+        return context.node().wantedVespaVersion().map(v -> v.getMajor() == 6).orElse(false);
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
index ef8ea60bee3..e20480e14ef 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
@@ -18,6 +18,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -183,8 +184,8 @@ public class NodeAgentContextImpl implements NodeAgentContext {
                     .flavor("d-2-8-50");
         }
 
-        public Builder nodeType(NodeType nodeType) {
-            this.nodeSpecBuilder.type(nodeType);
+        public Builder nodeSpecBuilder(Function<NodeSpec.Builder, NodeSpec.Builder> nodeSpecBuilderModifier) {
+            this.nodeSpecBuilder = nodeSpecBuilderModifier.apply(nodeSpecBuilder);
             return this;
         }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 98583262136..b7e7b97cdd8 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -355,7 +355,7 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     private boolean noCpuCap(ZoneApi zone) {
-        return zone.getEnvironment() == Environment.dev || (zone.getSystemName().isCd());
+        return zone.getEnvironment() == Environment.dev || zone.getSystemName().isCd();
     }
 
     private boolean downloadImageIfNeeded(NodeSpec node, Optional<Container> container) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
index a815515ac83..9151dde19a6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
@@ -34,7 +34,7 @@ public class StoredInteger implements Supplier<OptionalInt> {
         if (!hasBeenRead) {
             try {
                 String value = new String(Files.readAllBytes(path));
-                this.value = OptionalInt.of(Integer.valueOf(value));
+                this.value = OptionalInt.of(Integer.parseInt(value));
             } catch (NoSuchFileException e) {
                 this.value = OptionalInt.empty();
             } catch (IOException e) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index d809d9cbf96..37156ade064 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
+import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
@@ -9,9 +10,6 @@ import org.junit.Test;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -25,13 +23,14 @@ import static org.mockito.Mockito.when;
  */
 public class CoreCollectorTest {
     private final String GDB_PATH = "/my/path/to/gdb";
+    private final String JDK_PATH = "/path/to/jdk/java";
     private final DockerOperations docker = mock(DockerOperations.class);
     private final CoreCollector coreCollector = new CoreCollector(docker, Paths.get(GDB_PATH));
     private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build();
 
     private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234");
     private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program");
-    private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]",
+    private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]",
             "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t    printf(argv[3]);",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4");
@@ -127,10 +126,10 @@ public class CoreCollectorTest {
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
 
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE));
-        expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE));
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", TEST_BIN_PATH.toString(),
+                "backtrace", GDB_BACKTRACE,
+                "backtrace_all_threads", GDB_BACKTRACE);
         assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
@@ -142,16 +141,57 @@ public class CoreCollectorTest {
         mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
                 "", "Failure");
 
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
+        Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString());
         assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
+    @Test
+    public void reportsJstackInsteadOfGdbForJdkCores() {
+        mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
+                "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)");
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+                "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.");
+
+        String jstack = "jstack11";
+        mockExec(new String[]{"jhsdb", "jstack", "--exe", JDK_PATH, "--core", "/tmp/core.1234"},
+                jstack);
+
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", JDK_PATH,
+                "backtrace_all_threads", List.of(jstack));
+        assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
+    }
+
+    @Test
+    public void reportsJstackInsteadOfGdbForJdkCoresVespa6() {
+        NodeAgentContext contextVespa6 = new NodeAgentContextImpl.Builder("container-123.domain.tld")
+                .nodeSpecBuilder(n -> n.wantedVespaVersion(Version.fromString("6.330.51")))
+                .build();
+
+        mockExec(contextVespa6, new String[]{"file", TEST_CORE_PATH.toString()},
+                "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)", "");
+        mockExec(contextVespa6, new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+                "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.", "");
+
+        String jstack = "jstack8";
+        mockExec(contextVespa6, new String[]{"jstack", JDK_PATH, "/tmp/core.1234"},
+                jstack, "");
+
+        Map<String, Object> expectedData = Map.of(
+                "bin_path", JDK_PATH,
+                "backtrace_all_threads", List.of(jstack));
+        assertEquals(expectedData, coreCollector.collect(contextVespa6, TEST_CORE_PATH));
+    }
+
     private void mockExec(String[] cmd, String output) {
         mockExec(cmd, output, "");
     }
 
     private void mockExec(String[] cmd, String output, String error) {
+        mockExec(context, cmd, output, error);
+    }
+
+    private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) {
         when(docker.executeCommandInContainerAsRoot(context, cmd))
                 .thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
     }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index ca9b05a3ff6..b33f52ff629 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -1,11 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
 import org.junit.Test;
@@ -157,14 +154,7 @@ public class NodeAdminImplTest {
     }
 
     private NodeAgentContext createNodeAgentContext(String hostname) {
-        NodeSpec nodeSpec = new NodeSpec.Builder()
-                .hostname(hostname)
-                .state(NodeState.active)
-                .type(NodeType.tenant)
-                .flavor("default")
-                .build();
-
-        return new NodeAgentContextImpl.Builder(nodeSpec).build();
+        return new NodeAgentContextImpl.Builder(hostname).build();
     }
 
     private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) {