Probe container to set the correct GDB path.

2 files changed, 35 insertions, 14 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index 511c2205592..b2d52a5cd24 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -30,8 +30,8 @@ public class CoreCollector {
     private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$");
     private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'");
     private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'");
-    static final String GDB_PATH = Files.exists(Path.of("/opt/rh/gcc-toolset-10/root/bin/gdb")) ?
-                                         "/opt/rh/gcc-toolset-10/root/bin/gdb" : "/opt/rh/devtoolset-10/root/bin/gdb";
+    static final String GDB_PATH_RHEL7 = "/opt/rh/devtoolset-10/root/bin/gdb";
+    static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-10/root/bin/gdb";
 
     static final Map<String, Object> JAVA_HEAP_DUMP_METADATA =
             Map.of("bin_path", "java", "backtrace", List.of("Heap dump, no backtrace available"));
@@ -42,8 +42,17 @@ public class CoreCollector {
         this.docker = docker;
     }
 
+    String getGdbPath(NodeAgentContext context) {
+        String[] command = {"stat", GDB_PATH_RHEL7};
+        ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
+        if (result.getExitStatus() == 0) {
+            return GDB_PATH_RHEL7;
+        }
+        return GDB_PATH_RHEL8;
+    }
+
     Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) {
-        String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
+        String command = getGdbPath(context) + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
         String[] wrappedCommand = {"/bin/sh", "-c", command};
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand);
 
@@ -82,7 +91,7 @@ public class CoreCollector {
 
     List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
         String threads = allThreads ? "thread apply all bt" : "bt";
-        String[] command = {GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
+        String[] command = {getGdbPath(context), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
 
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
         if (result.getExitStatus() != 0)
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index e8679714c6f..bfa1ea6ca2b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -12,7 +12,8 @@ import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
-import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8;
 import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.JAVA_HEAP_DUMP_METADATA;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -60,9 +61,10 @@ public class CoreCollectorTest {
                         "execfn: '/usr/bin/program', platform: 'x86_64");
         assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output");
 
         Path fallbackResponse = Paths.get("/response/from/fallback");
-        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL7 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
                 "Core was generated by `/response/from/fallback'.");
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style");
@@ -74,8 +76,10 @@ public class CoreCollectorTest {
 
     @Test
     public void extractsBinaryPathUsingGdbTest() {
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output");
+
         final String[] cmd = new String[]{"/bin/sh", "-c",
-                GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"};
+                GDB_PATH_RHEL7 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"};
 
         mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'.");
         assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(context, TEST_CORE_PATH));
@@ -92,11 +96,13 @@ public class CoreCollectorTest {
 
     @Test
     public void extractsBacktraceUsingGdb() {
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output");
+
+        mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
         assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false));
 
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 "", "Failure");
         try {
             coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false);
@@ -109,7 +115,9 @@ public class CoreCollectorTest {
 
     @Test
     public void extractsBacktraceFromAllThreadsUsingGdb() {
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch",
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "The stat output");
+
+        mockExec(new String[]{GDB_PATH_RHEL7, "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
         assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true));
@@ -120,9 +128,10 @@ public class CoreCollectorTest {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "", "stat: No such file or directory");
+        mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch",
+        mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
 
@@ -138,7 +147,7 @@ public class CoreCollectorTest {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
-        mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
+        mockExec(new String[]{GDB_PATH_RHEL7 + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
                 "", "Failure");
 
         Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString());
@@ -149,7 +158,10 @@ public class CoreCollectorTest {
     public void reportsJstackInsteadOfGdbForJdkCores() {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)");
-        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7}, "", "stat: No such file or directory");
+
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL8 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
                 "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.");
 
         String jstack = "jstack11";