diff options
author | valerijf <valerijf@yahoo-inc.com> | 2017-03-14 11:53:36 +0100 |
---|---|---|
committer | valerijf <valerijf@yahoo-inc.com> | 2017-03-14 11:53:36 +0100 |
commit | 1335d8dcad989dd4b31cf122262ce34054ed3535 (patch) | |
tree | 4991777f5fed9fbf00a9920293560008183a4a8b /node-maintainer | |
parent | 32e759d5bf25249d82fd1d1c601e72ec813f420b (diff) |
Add support for reporting yinst state and rpm packages
Diffstat (limited to 'node-maintainer')
5 files changed, 77 insertions, 12 deletions
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index dd7d6c5c93e..96691f8402e 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -35,6 +36,24 @@ public class CoreCollector { public CoreCollector(ProcessExecuter processExecuter) { this.processExecuter = processExecuter; } + + List<String> readYinstState(Path yinstStatePath) throws IOException { + Pair<Integer, String> result = processExecuter.exec(new String[]{"cat", yinstStatePath.toString()}); + + if (result.getFirst() != 0) { + throw new RuntimeException("Failed to read yinst state file at: " + yinstStatePath + ", result: " + result); + } + return Arrays.asList(result.getSecond().split("\n")); + } + + List<String> readRpmPackages() throws IOException { + Pair<Integer, String> result = processExecuter.exec(new String[]{"rpm", "-qa"}); + + if (result.getFirst() != 0) { + throw new RuntimeException("Failed to read RPM packages " + result); + } + return Arrays.asList(result.getSecond().split("\n")); + } Path readBinPathFallback(Path coredumpPath) throws IOException, InterruptedException { String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; @@ -77,19 +96,42 @@ public class CoreCollector { return Arrays.asList(result.getSecond().split("\n")); } - public Map<String, Object> collect(Path coredumpPath) { + Map<String, Object> collect(Path coredumpPath, Optional<Path> yinstStatePath) { Map<String, Object> data = new LinkedHashMap<>(); try { coredumpPath = compressCoredump(coredumpPath); + } catch (IOException | InterruptedException e) { + logger.log(Level.WARNING, "Failed compressing/decompressing core dump", e); + } + + try { Path binPath = readBinPath(coredumpPath); data.put("bin_path", binPath.toString()); data.put("backtrace", readBacktrace(coredumpPath, binPath, false)); data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true)); + } catch (Throwable e) { + logger.log(Level.WARNING, "Failed to extrect backtrace", e); + } + yinstStatePath.ifPresent(yinstState -> { + try { + data.put("yinst_state", readYinstState(yinstState)); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to read yinst state", e); + } + + try { + data.put("rpm_packages", readRpmPackages()); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to read RPM packages", e); + } + }); + + try { deleteDecompressedCoredump(coredumpPath); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to collect core dump data", e); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to deleting compressed core dump", e); } return data; } @@ -134,6 +176,7 @@ public class CoreCollector { } private boolean diskSpaceAvailable(Path path) throws IOException { + // TODO: If running inside container, check against container memory size, not for the enitre host String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo"))); return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java index 8e50feee506..cfef8da6732 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java @@ -40,14 +40,16 @@ public class CoredumpHandler { private final Path coredumpsPath; private final Path doneCoredumpsPath; private final Map<String, Object> nodeAttributes; + private final Optional<Path> yinstStatePath; public CoredumpHandler(HttpClient httpClient, CoreCollector coreCollector, Path coredumpsPath, Path doneCoredumpsPath, - Map<String, Object> nodeAttributes) { + Map<String, Object> nodeAttributes, Optional<Path> yinstStatePath) { this.httpClient = httpClient; this.coreCollector = coreCollector; this.coredumpsPath = coredumpsPath; this.doneCoredumpsPath = doneCoredumpsPath; this.nodeAttributes = nodeAttributes; + this.yinstStatePath = yinstStatePath; } public void processAll() throws IOException { @@ -116,7 +118,7 @@ public class CoredumpHandler { } private Map<String, Object> collectMetadata(Path coredumpPath, Map<String, Object> nodeAttributes) { - Map<String, Object> metadata = coreCollector.collect(coredumpPath); + Map<String, Object> metadata = coreCollector.collect(coredumpPath, yinstStatePath); metadata.putAll(nodeAttributes); Map<String, Object> fields = new HashMap<>(); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java index 8447e04eb4b..88d9c5312ad 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java @@ -124,9 +124,11 @@ public class Maintainer { Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); Map<String, Object> attributesMap = parseMap(arguments); + Optional<Path> yinstStatePath = SlimeUtils.optionalString(arguments.field("yinstStatePath")).map(Paths::get); try { - CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector, coredumpsPath, doneCoredumpsPath, attributesMap); + CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector, + coredumpsPath, doneCoredumpsPath, attributesMap, yinstStatePath); coredumpHandler.processAll(); } catch (IOException e) { throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() + diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java index b05f48edad2..ae771504729 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -32,6 +33,7 @@ public class CoreCollectorTest { private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); private final CoreCollector coreCollector = new CoreCollector(processExecuter); + private final Path YINST_STATE_PATH = Paths.get("/path/to/yinst.state"); private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program"); private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]", @@ -39,6 +41,14 @@ public class CoreCollectorTest { "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t printf(argv[3]);", "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4"); + private final List<String> YINST_STATE = Arrays.asList("package: some_package-0.0.2", + "variable 'value'", + "ca_file /path/to/ca.pem"); + + private final List<String> RPM_PACKAGES = Arrays.asList("some_package-0.0.2", + "another_package-1.0.6", + "last_pkg-3.10_102"); + @Rule public TemporaryFolder folder= new TemporaryFolder(); @@ -138,18 +148,26 @@ public class CoreCollectorTest { mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "thread apply all bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, String.join("\n", GDB_BACKTRACE)); + mockExec(new String[]{"cat", YINST_STATE_PATH.toString()}, String.join("\n", YINST_STATE)); + mockExec(new String[]{"rpm", "-qa"}, String.join("\n", RPM_PACKAGES)); Map<String, Object> expectedData = new HashMap<>(); expectedData.put("bin_path", TEST_BIN_PATH.toString()); expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE)); expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE)); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH)); + expectedData.put("yinst_state", new ArrayList<>(YINST_STATE)); + expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES)); + assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(YINST_STATE_PATH))); } @Test - public void collectsPartialIfUnableToDetermineDumpingProgramTest() { + public void collectsPartialIfUnableToDetermineDumpingProgramTest() throws IOException, InterruptedException { + // We fail to get backtrace and RPM packages, but yinst state works, make sure it is returned + mockExec(new String[]{"cat", YINST_STATE_PATH.toString()}, String.join("\n", YINST_STATE)); + Map<String, Object> expectedData = new HashMap<>(); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH)); + expectedData.put("yinst_state", new ArrayList<>(YINST_STATE)); + assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(YINST_STATE_PATH))); } @Test @@ -162,7 +180,7 @@ public class CoreCollectorTest { Map<String, Object> expectedData = new HashMap<>(); expectedData.put("bin_path", TEST_BIN_PATH.toString()); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH)); + assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.empty())); } @Test diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java index 6d55b915fbe..4bd66709ff0 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java @@ -74,7 +74,7 @@ public class CoredumpHandlerTest { crashPath = folder.newFolder("crash").toPath(); donePath = folder.newFolder("done").toPath(); - coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes); + coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes, Optional.empty()); } @Test @@ -108,7 +108,7 @@ public class CoredumpHandlerTest { @Test public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException { - when(coreCollector.collect(any())).thenReturn(metadata); + when(coreCollector.collect(any(), any())).thenReturn(metadata); createCoredump("core.dump"); Path processingPath = coredumpHandler.processCoredumps(); |