Merge pull request #18308 from vespa-engine/aressem/gdb-path-rhel8

Set the correct path for GDB on RHEL8.

2 files changed, 49 insertions, 14 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index c746b5f3894..ce8fed0aa70 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -29,7 +29,10 @@ public class CoreCollector {
     private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$");
     private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'");
     private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'");
-    static final String GDB_PATH = "/opt/rh/devtoolset-10/root/bin/gdb";
+    static final String GDB_PATH_RHEL7_DT9 = "/opt/rh/devtoolset-9/root/bin/gdb";
+    static final String GDB_PATH_RHEL7_DT10 = "/opt/rh/devtoolset-10/root/bin/gdb";
+    static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-10/root/bin/gdb";
+
     static final Map<String, Object> JAVA_HEAP_DUMP_METADATA =
             Map.of("bin_path", "java", "backtrace", List.of("Heap dump, no backtrace available"));
 
@@ -39,8 +42,23 @@ public class CoreCollector {
         this.docker = docker;
     }
 
+    String getGdbPath(NodeAgentContext context) {
+        // TODO: Remove when we do not have any devtoolset-9 installs left
+        String[] command_rhel7_dt9 = {"stat", GDB_PATH_RHEL7_DT9};
+        if (docker.executeCommandInContainerAsRoot(context, command_rhel7_dt9).getExitStatus() == 0) {
+            return GDB_PATH_RHEL7_DT9;
+        }
+
+        String[] command_rhel7_dt10 = {"stat", GDB_PATH_RHEL7_DT10};
+        if (docker.executeCommandInContainerAsRoot(context, command_rhel7_dt10).getExitStatus() == 0) {
+            return GDB_PATH_RHEL7_DT10;
+        }
+
+        return GDB_PATH_RHEL8;
+    }
+
     Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) {
-        String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
+        String command = getGdbPath(context) + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
         String[] wrappedCommand = {"/bin/sh", "-c", command};
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand);
 
@@ -79,7 +97,7 @@ public class CoreCollector {
 
     List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
         String threads = allThreads ? "thread apply all bt" : "bt";
-        String[] command = {GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
+        String[] command = {getGdbPath(context), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
 
         ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
         if (result.getExitStatus() != 0)
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index e8679714c6f..d61ab9e53b8 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -12,7 +12,9 @@ import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
-import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7_DT9;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL7_DT10;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8;
 import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.JAVA_HEAP_DUMP_METADATA;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -60,9 +62,10 @@ public class CoreCollectorTest {
                         "execfn: '/usr/bin/program', platform: 'x86_64");
         assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output");
 
         Path fallbackResponse = Paths.get("/response/from/fallback");
-        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL7_DT9 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
                 "Core was generated by `/response/from/fallback'.");
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style");
@@ -74,8 +77,11 @@ public class CoreCollectorTest {
 
     @Test
     public void extractsBinaryPathUsingGdbTest() {
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "", "stat: No such file or directory");
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT10}, "The stat output");
+
         final String[] cmd = new String[]{"/bin/sh", "-c",
-                GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"};
+                GDB_PATH_RHEL7_DT10 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"};
 
         mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'.");
         assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(context, TEST_CORE_PATH));
@@ -92,24 +98,28 @@ public class CoreCollectorTest {
 
     @Test
     public void extractsBacktraceUsingGdb() {
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output");
+
+        mockExec(new String[]{GDB_PATH_RHEL7_DT9, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
         assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false));
 
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{GDB_PATH_RHEL7_DT9, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 "", "Failure");
         try {
             coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false);
             fail("Expected not to be able to read backtrace");
         } catch (RuntimeException e) {
             assertEquals("Failed to read backtrace ProcessResult { exitStatus=1 output= errors=Failure }, Command: " +
-                    "[/opt/rh/devtoolset-10/root/bin/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage());
+                    "[" + GDB_PATH_RHEL7_DT9 + ", -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage());
         }
     }
 
     @Test
     public void extractsBacktraceFromAllThreadsUsingGdb() {
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch",
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output");
+
+        mockExec(new String[]{GDB_PATH_RHEL7_DT9, "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
         assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true));
@@ -120,9 +130,11 @@ public class CoreCollectorTest {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "", "stat: No such file or directory");
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT10}, "", "stat: No such file or directory");
+        mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
-        mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch",
+        mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
 
@@ -138,7 +150,8 @@ public class CoreCollectorTest {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
-        mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output");
+        mockExec(new String[]{GDB_PATH_RHEL7_DT9 + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
                 "", "Failure");
 
         Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString());
@@ -149,7 +162,11 @@ public class CoreCollectorTest {
     public void reportsJstackInsteadOfGdbForJdkCores() {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)");
-        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
+
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "", "stat: No such file or directory");
+        mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT10}, "", "stat: No such file or directory");
+
+        mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL8 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"},
                 "Core was generated by `" + JDK_PATH + " -Dconfig.id=default/container.11 -XX:+Pre'.");
 
         String jstack = "jstack11";