diff options
author | Valerij Fredriksen <valerijf@verizonmedia.com> | 2020-10-30 16:14:47 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@verizonmedia.com> | 2020-10-30 16:14:47 +0100 |
commit | dab6ed561cb895ee91b79d5cb9a85f57e7850512 (patch) | |
tree | 02bf54f07bd52dfa23d02bd1af62d50f87f24485 /node-admin | |
parent | 0ed0cb858ea09522f05669b2d9e211c5bf8738fb (diff) |
Report java heap dumps
Diffstat (limited to 'node-admin')
5 files changed, 14 insertions, 17 deletions
diff --git a/node-admin/.gitignore b/node-admin/.gitignore index 5881f4b513b..adbb97d2d31 100644 --- a/node-admin/.gitignore +++ b/node-admin/.gitignore @@ -1 +1 @@ -node-admin-zone-app/components +data/
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index 63bfe87b662..0141e7f0274 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -24,9 +24,11 @@ import java.util.regex.Pattern; public class CoreCollector { private static final Logger logger = Logger.getLogger(CoreCollector.class.getName()); + private static final Pattern JAVA_CORE_PATTERN = Pattern.compile("java_pid.*\\.hprof"); private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$"); private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); + static final String GDB_PATH = "/opt/rh/devtoolset-9/root/bin/gdb"; private final ContainerOperations docker; @@ -35,8 +37,7 @@ public class CoreCollector { } Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) { - String command = GDBPath().toString() - + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; + String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; String[] wrappedCommand = {"/bin/sh", "-c", command}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand); @@ -48,10 +49,6 @@ public class CoreCollector { return Paths.get(matcher.group("path").split(" ")[0]); } - Path GDBPath() { - return Paths.get("/opt/rh/devtoolset-9/root/bin/gdb"); - } - Path readBinPath(NodeAgentContext context, Path coredumpPath) { String[] command = {"file", coredumpPath.toString()}; try { @@ -79,7 +76,7 @@ public class CoreCollector { List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) { String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = {GDBPath().toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; + String[] command = {GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitStatus() != 0) @@ -105,6 +102,9 @@ public class CoreCollector { * @return map of relevant metadata about the core dump */ Map<String, Object> collect(NodeAgentContext context, Path coredumpPath) { + if (JAVA_CORE_PATTERN.matcher(coredumpPath.getFileName().toString()).matches()) + return Map.of("bin_path", "java"); + Map<String, Object> data = new HashMap<>(); try { Path binPath = readBinPath(context, coredumpPath); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 293f2c688d6..20d710cbad8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -37,7 +37,6 @@ import static com.yahoo.yolean.Exceptions.uncheck; */ public class CoredumpHandler { - private static final Pattern JAVA_CORE_PATTERN = Pattern.compile("java_pid.*\\.hprof"); private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; @@ -84,12 +83,6 @@ public class CoredumpHandler { Path containerCrashPathOnHost = context.pathOnHostFromPathInNode(crashPatchInContainer); Path containerProcessingPathOnHost = containerCrashPathOnHost.resolve(PROCESSING_DIRECTORY_NAME); - // Remove java core dumps - FileFinder.files(containerCrashPathOnHost) - .match(nameMatches(JAVA_CORE_PATTERN)) - .maxDepth(1) - .deleteRecursively(context); - updateMetrics(context, containerCrashPathOnHost); // Check if we have already started to process a core dump or we can enqueue a new core one diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index 0be68e905ca..9f8f8953424 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -12,6 +12,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; +import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -21,7 +22,6 @@ import static org.mockito.Mockito.when; * @author freva */ public class CoreCollectorTest { - private final String GDB_PATH = "/opt/rh/devtoolset-9/root/bin/gdb"; private final String JDK_PATH = "/path/to/jdk/java"; private final ContainerOperations docker = mock(ContainerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker); @@ -161,6 +161,11 @@ public class CoreCollectorTest { assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH)); } + @Test + public void metadata_for_java_heap_dump() { + assertEquals(Map.of("bin_path", "java"), coreCollector.collect(context, Paths.get("java_pid123.hprof"))); + } + private void mockExec(String[] cmd, String output) { mockExec(cmd, output, ""); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index e5c2e35f2b2..fe0a7a52a62 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -45,7 +45,6 @@ import static org.mockito.Mockito.when; */ public class CoredumpHandlerTest { private final FileSystem fileSystem = TestFileSystem.create(); - private final Path donePath = fileSystem.getPath("/home/docker/dumps"); private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld") .fileSystem(fileSystem).build(); private final Path crashPathInContainer = fileSystem.getPath("/var/crash"); |