Compress coredumps with Zstandard

2 files changed, 21 insertions, 34 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
index cad6b7b76ac..6295765a95f 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
@@ -13,10 +14,11 @@ import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
 import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Clock;
-import java.time.Duration;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -26,7 +28,6 @@ import java.util.UUID;
 import java.util.function.Supplier;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith;
@@ -42,15 +43,14 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 public class CoredumpHandler {
 
     private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log");
-    private static final String LZ4_PATH = "/usr/bin/lz4";
     private static final String PROCESSING_DIRECTORY_NAME = "processing";
     private static final String METADATA_FILE_NAME = "metadata.json";
+    private static final String COMPRESSED_EXTENSION = ".zstd";
     public static final String COREDUMP_FILENAME_PREFIX = "dump_";
 
     private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName());
     private final ObjectMapper objectMapper = new ObjectMapper();
 
-    private final Terminal terminal;
     private final CoreCollector coreCollector;
     private final CoredumpReporter coredumpReporter;
     private final String crashPatchInContainer;
@@ -65,14 +65,13 @@ public class CoredumpHandler {
      */
     public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
                            String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics) {
-        this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath,
+        this(coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath,
                 metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString());
     }
 
-    CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
+    CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter,
                     String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics,
                     Clock clock, Supplier<String> coredumpIdSupplier) {
-        this.terminal = terminal;
         this.coreCollector = coreCollector;
         this.coredumpReporter = coredumpReporter;
         this.crashPatchInContainer = crashPathInContainer;
@@ -94,7 +93,7 @@ public class CoredumpHandler {
                     .match(fileAttributes -> !isReadyForProcessing(fileAttributes))
                     .maxDepth(1).stream()
                     .map(FileFinder.FileAttributes::filename)
-                    .collect(Collectors.toUnmodifiableList());
+                    .toList();
             if (!pendingCores.isEmpty())
                 throw new ConvergenceException(String.format("Cannot process %s coredumps: Still being written",
                         pendingCores.size() < 5 ? pendingCores : pendingCores.size()));
@@ -130,7 +129,7 @@ public class CoredumpHandler {
                 .stream()
                 .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime))
                 .map(FileFinder.FileAttributes::path)
-                .collect(Collectors.toList());
+                .toList();
 
         int coredumpIndex = IntStream.range(0, toProcess.size())
                 .filter(i -> !HS_ERR_PATTERN.matcher(toProcess.get(i).getFileName().toString()).matches())
@@ -191,11 +190,14 @@ public class CoredumpHandler {
      */
     private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory) throws IOException {
         ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory);
-        ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + ".lz4");
-        terminal.newCommandLine(context)
-                .add(LZ4_PATH, "-f", coreFile.pathOnHost().toString(), compressedCoreFile.pathOnHost().toString())
-                .setTimeout(Duration.ofMinutes(30))
-                .execute();
+        ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + COMPRESSED_EXTENSION);
+
+        try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile));
+             OutputStream fos = Files.newOutputStream(compressedCoreFile)) {
+            zcis.transferTo(fos);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
         Files.delete(coreFile);
 
         Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString());
@@ -206,7 +208,7 @@ public class CoredumpHandler {
 
     ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) {
         return (ContainerPath) FileFinder.files(coredumpProccessingDirectory)
-                .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(".lz4").negate()))
+                .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(COMPRESSED_EXTENSION).negate()))
                 .maxDepth(1)
                 .stream()
                 .map(FileFinder.FileAttributes::path)
@@ -222,7 +224,7 @@ public class CoredumpHandler {
         int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPath)
                 .match(nameStartsWith(".").negate())
                 .match(nameMatches(HS_ERR_PATTERN).negate())
-                .match(nameEndsWith(".lz4").negate())
+                .match(nameEndsWith(COMPRESSED_EXTENSION).negate())
                 .match(nameStartsWith("metadata").negate())
                 .list().size();
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
index 8f671910f93..5c681ddc129 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -8,10 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
 import com.yahoo.vespa.test.file.TestFileSystem;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -51,14 +48,13 @@ public class CoredumpHandlerTest {
     private final ContainerPath containerCrashPath = context.paths().of("/var/crash");
     private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps");
 
-    private final TestTerminal terminal = new TestTerminal();
     private final CoreCollector coreCollector = mock(CoreCollector.class);
     private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class);
     private final Metrics metrics = new Metrics();
     private final ManualClock clock = new ManualClock();
     @SuppressWarnings("unchecked")
     private final Supplier<String> coredumpIdSupplier = mock(Supplier.class);
-    private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter,
+    private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter,
             containerCrashPath.pathInContainer(), doneCoredumpsPath, metrics, clock, coredumpIdSupplier);
 
 
@@ -187,7 +183,7 @@ public class CoredumpHandlerTest {
     public void fails_to_get_core_file_if_only_compressed() throws IOException {
         ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123");
         Files.createDirectories(coredumpDirectory);
-        Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.lz4"));
+        Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zstd"));
         coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory);
     }
 
@@ -199,18 +195,12 @@ public class CoredumpHandlerTest {
         Files.createFile(coredumpDirectory.resolve("dump_bash.core.431"));
         assertFolderContents(coredumpDirectory, "metadata.json", "dump_bash.core.431");
 
-        terminal.interceptCommand("/usr/bin/lz4 -f /data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431 " +
-                "/data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1",
-                commandLine -> {
-                    uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3))));
-                    return new TestChildProcess2(0, "");
-                });
         coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Map::of);
         verify(coreCollector, never()).collect(any(), any());
         verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata"));
         assertFalse(Files.exists(coredumpDirectory));
         assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123");
-        assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4");
+        assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.zstd");
     }
 
     @Test
@@ -240,11 +230,6 @@ public class CoredumpHandlerTest {
         Files.createDirectories(containerCrashPath.pathOnHost());
     }
 
-    @After
-    public void teardown() {
-        terminal.verifyAllCommandsExecuted();
-    }
-
     private static void assertFolderContents(Path pathToFolder, String... filenames) {
         Set<String> expectedContentsOfFolder = Set.of(filenames);
         Set<String> actualContentsOfFolder;