diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-06-20 12:22:18 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-06-20 12:22:18 +0200 |
commit | 6fbf962ae13e6eb45a358017229949778ed08a85 (patch) | |
tree | 0bc3d6d076d8a00ba92b45f8abb3a8fe8ebc2518 | |
parent | dba7714be6b2cb68f10bac0ef89d29f885e4c8e0 (diff) | |
parent | dc335daa99d29bf8983cd684253332e1eaaf164e (diff) |
Merge pull request #23166 from vespa-engine/freva/coredump-zstd
Compress coredumps with Zstandard
2 files changed, 21 insertions, 34 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index cad6b7b76ac..6295765a95f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -13,10 +14,11 @@ import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Clock; -import java.time.Duration; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -26,7 +28,6 @@ import java.util.UUID; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith; @@ -42,15 +43,14 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CoredumpHandler { private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); - private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; + private static final String COMPRESSED_EXTENSION = ".zstd"; public static final String COREDUMP_FILENAME_PREFIX = "dump_"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); - private final Terminal terminal; private final CoreCollector coreCollector; private final CoredumpReporter coredumpReporter; private final String crashPatchInContainer; @@ -65,14 +65,13 @@ public class CoredumpHandler { */ public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics) { - this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, + this(coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString()); } - CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, + CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Clock clock, Supplier<String> coredumpIdSupplier) { - this.terminal = terminal; this.coreCollector = coreCollector; this.coredumpReporter = coredumpReporter; this.crashPatchInContainer = crashPathInContainer; @@ -94,7 +93,7 @@ public class CoredumpHandler { .match(fileAttributes -> !isReadyForProcessing(fileAttributes)) .maxDepth(1).stream() .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toUnmodifiableList()); + .toList(); if (!pendingCores.isEmpty()) throw new ConvergenceException(String.format("Cannot process %s coredumps: Still being written", pendingCores.size() < 5 ? pendingCores : pendingCores.size())); @@ -130,7 +129,7 @@ public class CoredumpHandler { .stream() .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) .map(FileFinder.FileAttributes::path) - .collect(Collectors.toList()); + .toList(); int coredumpIndex = IntStream.range(0, toProcess.size()) .filter(i -> !HS_ERR_PATTERN.matcher(toProcess.get(i).getFileName().toString()).matches()) @@ -191,11 +190,14 @@ public class CoredumpHandler { */ private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory) throws IOException { ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory); - ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + ".lz4"); - terminal.newCommandLine(context) - .add(LZ4_PATH, "-f", coreFile.pathOnHost().toString(), compressedCoreFile.pathOnHost().toString()) - .setTimeout(Duration.ofMinutes(30)) - .execute(); + ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + COMPRESSED_EXTENSION); + + try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile)); + OutputStream fos = Files.newOutputStream(compressedCoreFile)) { + zcis.transferTo(fos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } Files.delete(coreFile); Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString()); @@ -206,7 +208,7 @@ public class CoredumpHandler { ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) { return (ContainerPath) FileFinder.files(coredumpProccessingDirectory) - .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(".lz4").negate())) + .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(COMPRESSED_EXTENSION).negate())) .maxDepth(1) .stream() .map(FileFinder.FileAttributes::path) @@ -222,7 +224,7 @@ public class CoredumpHandler { int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPath) .match(nameStartsWith(".").negate()) .match(nameMatches(HS_ERR_PATTERN).negate()) - .match(nameEndsWith(".lz4").negate()) + .match(nameEndsWith(COMPRESSED_EXTENSION).negate()) .match(nameStartsWith("metadata").negate()) .list().size(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 8f671910f93..5c681ddc129 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -8,10 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,14 +48,13 @@ public class CoredumpHandlerTest { private final ContainerPath containerCrashPath = context.paths().of("/var/crash"); private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); - private final TestTerminal terminal = new TestTerminal(); private final CoreCollector coreCollector = mock(CoreCollector.class); private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class); private final Metrics metrics = new Metrics(); private final ManualClock clock = new ManualClock(); @SuppressWarnings("unchecked") private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); - private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter, + private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter, containerCrashPath.pathInContainer(), doneCoredumpsPath, metrics, clock, coredumpIdSupplier); @@ -187,7 +183,7 @@ public class CoredumpHandlerTest { public void fails_to_get_core_file_if_only_compressed() throws IOException { ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); Files.createDirectories(coredumpDirectory); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.lz4")); + Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zstd")); coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); } @@ -199,18 +195,12 @@ public class CoredumpHandlerTest { Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); assertFolderContents(coredumpDirectory, "metadata.json", "dump_bash.core.431"); - terminal.interceptCommand("/usr/bin/lz4 -f /data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431 " + - "/data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1", - commandLine -> { - uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3)))); - return new TestChildProcess2(0, ""); - }); coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Map::of); verify(coreCollector, never()).collect(any(), any()); verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata")); assertFalse(Files.exists(coredumpDirectory)); assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123"); - assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4"); + assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.zstd"); } @Test @@ -240,11 +230,6 @@ public class CoredumpHandlerTest { Files.createDirectories(containerCrashPath.pathOnHost()); } - @After - public void teardown() { - terminal.verifyAllCommandsExecuted(); - } - private static void assertFolderContents(Path pathToFolder, String... filenames) { Set<String> expectedContentsOfFolder = Set.of(filenames); Set<String> actualContentsOfFolder; |