diff options
2 files changed, 35 insertions, 14 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index 511c2205592..b2d52a5cd24 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -30,8 +30,8 @@ public class CoreCollector { private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$"); private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - static final String GDB_PATH = Files.exists(Path.of("/opt/rh/gcc-toolset-10/root/bin/gdb")) ? - "/opt/rh/gcc-toolset-10/root/bin/gdb" : "/opt/rh/devtoolset-10/root/bin/gdb"; + static final String GDB_PATH_RHEL7 = "/opt/rh/devtoolset-10/root/bin/gdb"; + static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-10/root/bin/gdb"; static final Map<String, Object> JAVA_HEAP_DUMP_METADATA = Map.of("bin_path", "java", "backtrace", List.of("Heap dump, no backtrace available")); @@ -42,8 +42,17 @@ public class CoreCollector { this.docker = docker; } + String getGdbPath(NodeAgentContext context) { + String[] command = {"stat", GDB_PATH_RHEL7}; + ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); + if (result.getExitStatus() == 0) { + return GDB_PATH_RHEL7; + } + return GDB_PATH_RHEL8; + } + Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) { - String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; + String command = getGdbPath(context) + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; String[] wrappedCommand = {"/bin/sh", "-c", command}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand); @@ -82,7 +91,7 @@ public class CoreCollector { List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) { String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = {GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; + String[] command = {getGdbPath(context), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitStatus() != 0) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index e8679714c6f..bfa1ea6ca2b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -12,7 +12,8 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH; +import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7; +import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.JAVA_HEAP_DUMP_METADATA; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -60,9 +61,10 @@ public class CoreCollectorTest { "execfn: '/usr/bin/program', platform: 'x86_64"); assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output"); Path fallbackResponse = Paths.get("/response/from/fallback"); - mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, + mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL7 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, "Core was generated by `/response/from/fallback'."); mockExec(cmd, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); @@ -74,8 +76,10 @@ public class CoreCollectorTest { @Test public void extractsBinaryPathUsingGdbTest() { + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output"); + final String[] cmd = new String[]{"/bin/sh", "-c", - GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}; + GDB_PATH_RHEL7 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}; mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); @@ -92,11 +96,13 @@ public class CoreCollectorTest { @Test public void extractsBacktraceUsingGdb() { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output"); + + mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, String.join("\n", GDB_BACKTRACE)); assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false)); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, "", "Failure"); try { coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false); @@ -109,7 +115,9 @@ public class CoreCollectorTest { @Test public void extractsBacktraceFromAllThreadsUsingGdb() { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output"); + + mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "thread apply all bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, String.join("\n", GDB_BACKTRACE)); assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true)); @@ -120,9 +128,10 @@ public class CoreCollectorTest { mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "", "stat: No such file or directory"); + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "thread apply all bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, String.join("\n", GDB_BACKTRACE)); @@ -138,7 +147,7 @@ public class CoreCollectorTest { mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"}, + mockExec(new String[]{GDB_PATH_RHEL7 + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"}, "", "Failure"); Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString()); @@ -149,7 +158,10 @@ public class CoreCollectorTest { public void reportsJstackInsteadOfGdbForJdkCores() { mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)"); - mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, + + mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "", "stat: No such file or directory"); + + mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL8 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'."); String jstack = "jstack11"; |