Add support for reporting yinst state and rpm packages

5 files changed, 77 insertions, 12 deletions

diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
index dd7d6c5c93e..96691f8402e 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java
@@ -12,6 +12,7 @@ import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
@@ -35,6 +36,24 @@ public class CoreCollector {
     public CoreCollector(ProcessExecuter processExecuter) {
         this.processExecuter = processExecuter;
     }
+
+    List<String> readYinstState(Path yinstStatePath) throws IOException {
+        Pair<Integer, String> result = processExecuter.exec(new String[]{"cat", yinstStatePath.toString()});
+
+        if (result.getFirst() != 0) {
+            throw new RuntimeException("Failed to read yinst state file at: " + yinstStatePath + ", result: " + result);
+        }
+        return Arrays.asList(result.getSecond().split("\n"));
+    }
+
+    List<String> readRpmPackages() throws IOException {
+        Pair<Integer, String> result = processExecuter.exec(new String[]{"rpm", "-qa"});
+
+        if (result.getFirst() != 0) {
+            throw new RuntimeException("Failed to read RPM packages " + result);
+        }
+        return Arrays.asList(result.getSecond().split("\n"));
+    }
     
     Path readBinPathFallback(Path coredumpPath) throws IOException, InterruptedException {
         String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
@@ -77,19 +96,42 @@ public class CoreCollector {
         return Arrays.asList(result.getSecond().split("\n"));
     }
 
-    public Map<String, Object> collect(Path coredumpPath) {
+    Map<String, Object> collect(Path coredumpPath, Optional<Path> yinstStatePath) {
         Map<String, Object> data = new LinkedHashMap<>();
         try {
             coredumpPath = compressCoredump(coredumpPath);
+        } catch (IOException | InterruptedException e) {
+            logger.log(Level.WARNING, "Failed compressing/decompressing core dump", e);
+        }
+
+        try {
             Path binPath = readBinPath(coredumpPath);
 
             data.put("bin_path", binPath.toString());
             data.put("backtrace", readBacktrace(coredumpPath, binPath, false));
             data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true));
+        } catch (Throwable e) {
+            logger.log(Level.WARNING, "Failed to extrect backtrace", e);
+        }
 
+        yinstStatePath.ifPresent(yinstState -> {
+            try {
+                data.put("yinst_state", readYinstState(yinstState));
+            } catch (Exception e) {
+                logger.log(Level.WARNING, "Failed to read yinst state", e);
+            }
+
+            try {
+                data.put("rpm_packages", readRpmPackages());
+            } catch (Exception e) {
+                logger.log(Level.WARNING, "Failed to read RPM packages", e);
+            }
+        });
+
+        try {
             deleteDecompressedCoredump(coredumpPath);
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, "Failed to collect core dump data", e);
+        } catch (IOException e) {
+            logger.log(Level.WARNING, "Failed to deleting compressed core dump", e);
         }
         return data;
     }
@@ -134,6 +176,7 @@ public class CoreCollector {
     }
 
     private boolean diskSpaceAvailable(Path path) throws IOException {
+        // TODO: If running inside container, check against container memory size, not for the enitre host
         String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo")));
         return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo);
     }
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
index 8e50feee506..cfef8da6732 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java
@@ -40,14 +40,16 @@ public class CoredumpHandler {
     private final Path coredumpsPath;
     private final Path doneCoredumpsPath;
     private final Map<String, Object> nodeAttributes;
+    private final Optional<Path> yinstStatePath;
 
     public CoredumpHandler(HttpClient httpClient, CoreCollector coreCollector, Path coredumpsPath, Path doneCoredumpsPath,
-                           Map<String, Object> nodeAttributes) {
+                           Map<String, Object> nodeAttributes, Optional<Path> yinstStatePath) {
         this.httpClient = httpClient;
         this.coreCollector = coreCollector;
         this.coredumpsPath = coredumpsPath;
         this.doneCoredumpsPath = doneCoredumpsPath;
         this.nodeAttributes = nodeAttributes;
+        this.yinstStatePath = yinstStatePath;
     }
 
     public void processAll() throws IOException {
@@ -116,7 +118,7 @@ public class CoredumpHandler {
     }
 
     private Map<String, Object> collectMetadata(Path coredumpPath, Map<String, Object> nodeAttributes) {
-        Map<String, Object> metadata = coreCollector.collect(coredumpPath);
+        Map<String, Object> metadata = coreCollector.collect(coredumpPath, yinstStatePath);
         metadata.putAll(nodeAttributes);
 
         Map<String, Object> fields = new HashMap<>();
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
index 8447e04eb4b..88d9c5312ad 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java
@@ -124,9 +124,11 @@ public class Maintainer {
         Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString());
         Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString());
         Map<String, Object> attributesMap = parseMap(arguments);
+        Optional<Path> yinstStatePath = SlimeUtils.optionalString(arguments.field("yinstStatePath")).map(Paths::get);
 
         try {
-            CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector, coredumpsPath, doneCoredumpsPath, attributesMap);
+            CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector,
+                    coredumpsPath, doneCoredumpsPath, attributesMap, yinstStatePath);
             coredumpHandler.processAll();
         } catch (IOException e) {
             throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() +
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java
index b05f48edad2..ae771504729 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java
@@ -17,6 +17,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -32,6 +33,7 @@ public class CoreCollectorTest {
     private final ProcessExecuter processExecuter = mock(ProcessExecuter.class);
     private final CoreCollector coreCollector = new CoreCollector(processExecuter);
 
+    private final Path YINST_STATE_PATH = Paths.get("/path/to/yinst.state");
     private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234");
     private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program");
     private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]",
@@ -39,6 +41,14 @@ public class CoreCollectorTest {
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t    printf(argv[3]);",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4");
 
+    private final List<String> YINST_STATE = Arrays.asList("package: some_package-0.0.2",
+            "variable 'value'",
+            "ca_file /path/to/ca.pem");
+
+    private final List<String> RPM_PACKAGES = Arrays.asList("some_package-0.0.2",
+            "another_package-1.0.6",
+            "last_pkg-3.10_102");
+
     @Rule
     public TemporaryFolder folder= new TemporaryFolder();
 
@@ -138,18 +148,26 @@ public class CoreCollectorTest {
         mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
+        mockExec(new String[]{"cat", YINST_STATE_PATH.toString()}, String.join("\n", YINST_STATE));
+        mockExec(new String[]{"rpm", "-qa"}, String.join("\n", RPM_PACKAGES));
 
         Map<String, Object> expectedData = new HashMap<>();
         expectedData.put("bin_path", TEST_BIN_PATH.toString());
         expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE));
         expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE));
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
+        expectedData.put("yinst_state", new ArrayList<>(YINST_STATE));
+        expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES));
+        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(YINST_STATE_PATH)));
     }
 
     @Test
-    public void collectsPartialIfUnableToDetermineDumpingProgramTest() {
+    public void collectsPartialIfUnableToDetermineDumpingProgramTest() throws IOException, InterruptedException {
+        // We fail to get backtrace and RPM packages, but yinst state works, make sure it is returned
+        mockExec(new String[]{"cat", YINST_STATE_PATH.toString()}, String.join("\n", YINST_STATE));
+
         Map<String, Object> expectedData = new HashMap<>();
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
+        expectedData.put("yinst_state", new ArrayList<>(YINST_STATE));
+        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(YINST_STATE_PATH)));
     }
 
     @Test
@@ -162,7 +180,7 @@ public class CoreCollectorTest {
 
         Map<String, Object> expectedData = new HashMap<>();
         expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
+        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.empty()));
     }
 
     @Test
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
index 6d55b915fbe..4bd66709ff0 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java
@@ -74,7 +74,7 @@ public class CoredumpHandlerTest {
         crashPath = folder.newFolder("crash").toPath();
         donePath = folder.newFolder("done").toPath();
 
-        coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes);
+        coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes, Optional.empty());
     }
 
     @Test
@@ -108,7 +108,7 @@ public class CoredumpHandlerTest {
 
     @Test
     public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException {
-        when(coreCollector.collect(any())).thenReturn(metadata);
+        when(coreCollector.collect(any(), any())).thenReturn(metadata);
         createCoredump("core.dump");
         Path processingPath = coredumpHandler.processCoredumps();