Merge pull request #7290 from vespa-engine/freva/fix-coredump-handler

NodeAdmin: Fix coredump handler

16 files changed, 413 insertions, 470 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index ecdf871a5ff..3e43ed63aea 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
 import com.yahoo.system.ProcessExecuter;
+import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
@@ -19,7 +20,6 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
 
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
@@ -240,17 +240,12 @@ public class StorageMaintainer {
     }
 
     /** Checks if container has any new coredumps, reports and archives them if so */
-    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node) {
-        final Path coredumpsPath = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/crash"));
-        final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node);
-        try {
-            coredumpHandler.processAll(coredumpsPath, nodeAttributes);
-        } catch (IOException e) {
-            throw new UncheckedIOException("Failed to process coredumps", e);
-        }
+    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) {
+        final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node, container);
+        coredumpHandler.converge(context, nodeAttributes);
     }
 
-    private Map<String, Object> getCoredumpNodeAttributes(NodeSpec node) {
+    private Map<String, Object> getCoredumpNodeAttributes(NodeSpec node, Optional<Container> container) {
         Map<String, Object> attributes = new HashMap<>();
         attributes.put("hostname", node.getHostname());
         attributes.put("parent_hostname", environment.getParentHostHostname());
@@ -259,7 +254,7 @@ public class StorageMaintainer {
         attributes.put("flavor", node.getFlavor());
         attributes.put("kernel_version", System.getProperty("os.version"));
 
-        node.getCurrentDockerImage().ifPresent(image -> attributes.put("docker_image", image.asString()));
+        container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString()));
         node.getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version));
         node.getOwner().ifPresent(owner -> {
             attributes.put("tenant", owner.getTenant());
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index 9830d03240a..d1c44a55c1b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -1,15 +1,14 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
-import com.yahoo.collections.Pair;
-import com.yahoo.system.ProcessExecuter;
+import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.LinkedHashMap;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
@@ -18,31 +17,31 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
- * Takes in a compressed (lz4) or uncompressed core dump and collects relevant metadata.
+ * Takes in an uncompressed core dump and collects relevant metadata.
  *
  * @author freva
  */
-class CoreCollector {
-    static final String GDB_PATH = "/usr/bin/gdb";
-    private static final String LZ4_PATH = "/usr/bin/lz4";
+public class CoreCollector {
+    private static final Logger logger = Logger.getLogger(CoreCollector.class.getName());
+
     private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$");
     private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'");
     private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'");
-    private static final Pattern TOTAL_MEMORY_PATTERN = Pattern.compile("^MemTotal:\\s*(?<totalMem>\\d+) kB$", Pattern.MULTILINE);
 
-    private static final Logger logger = Logger.getLogger(CoreCollector.class.getName());
-    private final ProcessExecuter processExecuter;
+    private final DockerOperations docker;
+    private final Path gdb;
 
-    CoreCollector(ProcessExecuter processExecuter) {
-        this.processExecuter = processExecuter;
+    public CoreCollector(DockerOperations docker, Path pathToGdbInContainer) {
+        this.docker = docker;
+        this.gdb = pathToGdbInContainer;
     }
 
-    Path readBinPathFallback(Path coredumpPath) throws IOException {
-        String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
+    Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) {
+        String command = gdb + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
         String[] wrappedCommand = {"/bin/sh", "-c", command};
-        Pair<Integer, String> result = processExecuter.exec(wrappedCommand);
+        ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), wrappedCommand);
 
-        Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getSecond());
+        Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput());
         if (! matcher.find()) {
             throw new RuntimeException(String.format("Failed to extract binary path from GDB, result: %s, command: %s",
                     result, Arrays.toString(wrappedCommand)));
@@ -50,115 +49,58 @@ class CoreCollector {
         return Paths.get(matcher.group("path").split(" ")[0]);
     }
 
-    Path readBinPath(Path coredumpPath) throws IOException {
+    Path readBinPath(NodeAgentContext context, Path coredumpPath) {
         String[] command = {"file", coredumpPath.toString()};
         try {
-            Pair<Integer, String> result = processExecuter.exec(command);
-
-            if (result.getFirst() != 0) {
+            ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command);
+            if (result.getExitStatus() != 0) {
                 throw new RuntimeException("file command failed with " + result);
             }
 
-            Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getSecond());
+            Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getOutput());
             if (execfnMatcher.find()) {
                 return Paths.get(execfnMatcher.group("path").split(" ")[0]);
             }
 
-            Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getSecond());
+            Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getOutput());
             if (fromMatcher.find()) {
                 return Paths.get(fromMatcher.group("path").split(" ")[0]);
             }
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, String.format("Failed getting bin path, command: %s. " +
+        } catch (RuntimeException e) {
+            context.log(logger, Level.WARNING, String.format("Failed getting bin path, command: %s. " +
                     "Trying fallback instead", Arrays.toString(command)), e);
         }
 
-        return readBinPathFallback(coredumpPath);
+        return readBinPathFallback(context, coredumpPath);
     }
 
-    List<String> readBacktrace(Path coredumpPath, Path binPath, boolean allThreads) throws IOException {
+    List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
         String threads = allThreads ? "thread apply all bt" : "bt";
-        String[] command = {GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
-        Pair<Integer, String> result = processExecuter.exec(command);
-        if (result.getFirst() != 0) {
+        String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
+        ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command);
+        if (result.getExitStatus() != 0) {
             throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command));
         }
-        return Arrays.asList(result.getSecond().split("\n"));
+        return Arrays.asList(result.getOutput().split("\n"));
     }
 
-    Map<String, Object> collect(Path coredumpPath) {
-        Map<String, Object> data = new LinkedHashMap<>();
-        try {
-            coredumpPath = compressCoredump(coredumpPath);
-        } catch (IOException e) {
-            logger.log(Level.WARNING, "Failed compressing/decompressing core dump", e);
-        }
-
+    /**
+     * Collects metadata about a given core dump
+     * @param context context of the NodeAgent that owns the core dump
+     * @param coredumpPath path to core dump file inside the container
+     * @return map of relevant metadata about the core dump
+     */
+    Map<String, Object> collect(NodeAgentContext context, Path coredumpPath) {
+        Map<String, Object> data = new HashMap<>();
         try {
-            Path binPath = readBinPath(coredumpPath);
+            Path binPath = readBinPath(context, coredumpPath);
 
             data.put("bin_path", binPath.toString());
-            data.put("backtrace", readBacktrace(coredumpPath, binPath, false));
-            data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true));
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, "Failed to extract backtrace", e);
-        }
-
-        try {
-            deleteDecompressedCoredump(coredumpPath);
-        } catch (IOException e) {
-            logger.log(Level.WARNING, "Failed to delete decompressed core dump", e);
+            data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false));
+            data.put("backtrace_all_threads", readBacktrace(context, coredumpPath, binPath, true));
+        } catch (RuntimeException e) {
+            context.log(logger, Level.WARNING, "Failed to extract backtrace", e);
         }
         return data;
     }
-
-
-    /**
-     * This method will either compress or decompress the core dump if the input path is to a decompressed or
-     * compressed core dump, respectively.
-     *
-     * @return Path to the decompressed core dump
-     */
-    private Path compressCoredump(Path coredumpPath) throws IOException {
-        if (! coredumpPath.toString().endsWith(".lz4")) {
-            processExecuter.exec(
-                    new String[]{LZ4_PATH, "-f", coredumpPath.toString(), coredumpPath.toString() + ".lz4"});
-            return coredumpPath;
-
-        } else {
-            if (!diskSpaceAvailable(coredumpPath)) {
-                throw new RuntimeException("Not decompressing " + coredumpPath + " due to not enough disk space available");
-            }
-
-            Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", ""));
-            Pair<Integer, String> result = processExecuter.exec(
-                    new String[] {LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()});
-            if (result.getFirst() != 0) {
-                throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result);
-            }
-            return decompressedPath;
-        }
-    }
-
-    /**
-     * Delete the core dump unless:
-     * - The file is compressed
-     * - There is no compressed file (i.e. it was not decompressed in the first place)
-     */
-    void deleteDecompressedCoredump(Path coredumpPath) throws IOException {
-        if (! coredumpPath.toString().endsWith(".lz4") && Paths.get(coredumpPath.toString() + ".lz4").toFile().exists()) {
-            Files.delete(coredumpPath);
-        }
-    }
-
-    private boolean diskSpaceAvailable(Path path) throws IOException {
-        String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo")));
-        return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo);
-    }
-
-    int parseTotalMemorySize(String memInfo) {
-        Matcher matcher = TOTAL_MEMORY_PATTERN.matcher(memInfo);
-        if (!matcher.find()) throw new RuntimeException("Could not parse meminfo: " + memInfo);
-        return Integer.valueOf(matcher.group("totalMem"));
-    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
index eb48086eb0f..16037f3233b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
@@ -2,20 +2,26 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.system.ProcessExecuter;
-import com.yahoo.vespa.hosted.node.admin.maintenance.FileHelper;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.time.Duration;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
-import java.util.logging.Level;
+import java.util.function.Supplier;
 import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
 
 /**
  * Finds coredumps, collects metadata and reports them
@@ -24,127 +30,145 @@ import java.util.logging.Logger;
  */
 public class CoredumpHandler {
 
+    private static final Pattern JAVA_CORE_PATTERN = Pattern.compile("java_pid.*\\.hprof");
+    private static final String LZ4_PATH = "/usr/bin/lz4";
     private static final String PROCESSING_DIRECTORY_NAME = "processing";
-    static final String METADATA_FILE_NAME = "metadata.json";
+    private static final String METADATA_FILE_NAME = "metadata.json";
+    private static final String COREDUMP_FILENAME_PREFIX = "dump_";
 
     private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName());
     private final ObjectMapper objectMapper = new ObjectMapper();
 
+    private final Terminal terminal;
     private final CoreCollector coreCollector;
-    private final Path doneCoredumpsPath;
     private final CoredumpReporter coredumpReporter;
+    private final Path crashPatchInContainer;
+    private final Path doneCoredumpsPath;
+    private final Supplier<String> coredumpIdSupplier;
 
-    public CoredumpHandler(CoredumpReporter coredumpReporter, Path doneCoredumpsPath) {
-        this(new CoreCollector(new ProcessExecuter()), coredumpReporter, doneCoredumpsPath);
+    /**
+     * @param crashPathInContainer path inside the container where core dump are dumped
+     * @param doneCoredumpsPath path on host where processed core dumps are stored
+     */
+    public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
+                           Path crashPathInContainer, Path doneCoredumpsPath) {
+        this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, () -> UUID.randomUUID().toString());
     }
 
-    CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter, Path doneCoredumpsPath) {
+    CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
+                           Path crashPathInContainer, Path doneCoredumpsPath, Supplier<String> coredumpIdSupplier) {
+        this.terminal = terminal;
         this.coreCollector = coreCollector;
         this.coredumpReporter = coredumpReporter;
+        this.crashPatchInContainer = crashPathInContainer;
         this.doneCoredumpsPath = doneCoredumpsPath;
+        this.coredumpIdSupplier = coredumpIdSupplier;
     }
 
-    public void processAll(Path coredumpsPath, Map<String, Object> nodeAttributes) throws IOException {
-        removeJavaCoredumps(coredumpsPath);
-        handleNewCoredumps(coredumpsPath, nodeAttributes);
-        removeOldCoredumps();
-    }
 
-    private void removeJavaCoredumps(Path coredumpsPath) throws IOException {
-        if (! coredumpsPath.toFile().isDirectory()) return;
-        FileHelper.deleteFiles(coredumpsPath, Duration.ZERO, Optional.of("^java_pid.*\\.hprof$"), false);
-    }
+    public void converge(NodeAgentContext context, Map<String, Object> nodeAttributes) {
+        Path containerCrashPathOnHost = context.pathOnHostFromPathInNode(crashPatchInContainer);
+        Path containerProcessingPathOnHost = containerCrashPathOnHost.resolve(PROCESSING_DIRECTORY_NAME);
 
-    private void removeOldCoredumps() throws IOException {
-        if (! doneCoredumpsPath.toFile().isDirectory()) return;
-        FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty());
-    }
+        // Remove java core dumps
+        FileFinder.files(containerCrashPathOnHost)
+                .match(nameMatches(JAVA_CORE_PATTERN))
+                .maxDepth(1)
+                .deleteRecursively();
 
-    private void handleNewCoredumps(Path coredumpsPath, Map<String, Object> nodeAttributes) {
-        enqueueCoredumps(coredumpsPath);
-        processAndReportCoredumps(coredumpsPath, nodeAttributes);
+        // Check if we have already started to process a core dump or we can enqueue a new core one
+        getCoredumpToProcess(containerCrashPathOnHost, containerProcessingPathOnHost)
+                .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributes));
     }
 
+    /** @return path to directory inside processing directory that contains a core dump file to process */
+    Optional<Path> getCoredumpToProcess(Path containerCrashPathOnHost, Path containerProcessingPathOnHost) {
+        return FileFinder.directories(containerProcessingPathOnHost).stream()
+                .map(FileFinder.FileAttributes::path)
+                .findAny()
+                .map(Optional::of)
+                .orElseGet(() -> enqueueCoredump(containerCrashPathOnHost, containerProcessingPathOnHost));
+    }
 
     /**
      * Moves a coredump to a new directory under the processing/ directory. Limit to only processing
      * one coredump at the time, starting with the oldest.
+     *
+     * @return path to directory inside processing directory which contains the enqueued core dump file
      */
-    void enqueueCoredumps(Path coredumpsPath) {
-        Path processingCoredumpsPath = getProcessingCoredumpsPath(coredumpsPath);
-        processingCoredumpsPath.toFile().mkdirs();
-        if (!FileHelper.listContentsOfDirectory(processingCoredumpsPath).isEmpty()) return;
-
-        FileHelper.listContentsOfDirectory(coredumpsPath).stream()
-                .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith("."))
-                .min((Comparator.comparingLong(o -> o.toFile().lastModified())))
-                .ifPresent(coredumpPath -> {
-                    try {
-                        enqueueCoredumpForProcessing(coredumpPath, processingCoredumpsPath);
-                    } catch (Throwable e) {
-                        logger.log(Level.WARNING, "Failed to process coredump " + coredumpPath, e);
-                    }
+    Optional<Path> enqueueCoredump(Path containerCrashPathOnHost, Path containerProcessingPathOnHost) {
+        return FileFinder.files(containerCrashPathOnHost)
+                .match(nameStartsWith(".").negate())
+                .maxDepth(1)
+                .stream()
+                .min(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime))
+                .map(FileFinder.FileAttributes::path)
+                .map(coredumpPath -> {
+                    UnixPath coredumpInProcessingDirectory = new UnixPath(
+                            containerProcessingPathOnHost
+                                    .resolve(coredumpIdSupplier.get())
+                                    .resolve(COREDUMP_FILENAME_PREFIX + coredumpPath.getFileName()));
+                    coredumpInProcessingDirectory.createParents();
+                    return uncheck(() -> Files.move(coredumpPath, coredumpInProcessingDirectory.toPath())).getParent();
                 });
     }
 
-    void processAndReportCoredumps(Path coredumpsPath, Map<String, Object> nodeAttributes) {
-        Path processingCoredumpsPath = getProcessingCoredumpsPath(coredumpsPath);
-        doneCoredumpsPath.toFile().mkdirs();
-
-        FileHelper.listContentsOfDirectory(processingCoredumpsPath).stream()
-                .filter(path -> path.toFile().isDirectory())
-                .forEach(coredumpDirectory -> processAndReportSingleCoredump(coredumpDirectory, nodeAttributes));
-    }
-
-    private void processAndReportSingleCoredump(Path coredumpDirectory, Map<String, Object> nodeAttributes) {
+    void processAndReportSingleCoredump(NodeAgentContext context, Path coredumpDirectory, Map<String, Object> nodeAttributes) {
         try {
-            String metadata = collectMetadata(coredumpDirectory, nodeAttributes);
+            String metadata = getMetadata(context, coredumpDirectory, nodeAttributes);
             String coredumpId = coredumpDirectory.getFileName().toString();
             coredumpReporter.reportCoredump(coredumpId, metadata);
-            finishProcessing(coredumpDirectory);
-            logger.info("Successfully reported coredump " + coredumpId);
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, "Failed to report coredump " + coredumpDirectory, e);
+            finishProcessing(context, coredumpDirectory);
+            context.log(logger, "Successfully reported coredump " + coredumpId);
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to process coredump " + coredumpDirectory, e);
         }
     }
 
-    private void enqueueCoredumpForProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException {
-        // Make coredump readable
-        coredumpPath.toFile().setReadable(true, false);
-
-        // Create new directory for this coredump and move it into it
-        Path folder = processingCoredumpsPath.resolve(UUID.randomUUID().toString());
-        folder.toFile().mkdirs();
-        Files.move(coredumpPath, folder.resolve(coredumpPath.getFileName()));
-    }
-
-    String collectMetadata(Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException {
-        Path metadataPath = coredumpDirectory.resolve(METADATA_FILE_NAME);
-        if (!Files.exists(metadataPath)) {
-            Path coredumpPath = FileHelper.listContentsOfDirectory(coredumpDirectory).stream().findFirst()
-                    .orElseThrow(() -> new RuntimeException("No coredump file found in processing directory " + coredumpDirectory));
-            Map<String, Object> metadata = coreCollector.collect(coredumpPath);
+    /**
+     * @return coredump metadata from metadata.json if present, otherwise attempts to get metadata using
+     * {@link CoreCollector} and stores it to metadata.json
+     */
+    String getMetadata(NodeAgentContext context, Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException {
+        UnixPath metadataPath = new UnixPath(coredumpDirectory.resolve(METADATA_FILE_NAME));
+        if (!Files.exists(metadataPath.toPath())) {
+            Path coredumpFilePathOnHost = findCoredumpFileInProcessingDirectory(coredumpDirectory);
+            Path coredumpFilePathInContainer = context.pathInNodeFromPathOnHost(coredumpFilePathOnHost);
+            Map<String, Object> metadata = coreCollector.collect(context, coredumpFilePathInContainer);
             metadata.putAll(nodeAttributes);
 
-            Map<String, Object> fields = new HashMap<>();
-            fields.put("fields", metadata);
-
-            String metadataFields = objectMapper.writeValueAsString(fields);
-            Files.write(metadataPath, metadataFields.getBytes());
+            String metadataFields = objectMapper.writeValueAsString(ImmutableMap.of("fields", metadata));
+            metadataPath.writeUtf8File(metadataFields);
             return metadataFields;
         } else {
-            return new String(Files.readAllBytes(metadataPath));
+            return metadataPath.readUtf8File();
         }
     }
 
-    private void finishProcessing(Path coredumpDirectory) throws IOException {
-        Files.move(coredumpDirectory, doneCoredumpsPath.resolve(coredumpDirectory.getFileName()));
-    }
-
     /**
-     * @return Path to directory where coredumps are temporarily moved while still being processed
+     * Compresses core file (and deletes the uncompressed core), then moves the entire core dump processing
+     * directory to {@link #doneCoredumpsPath} for archive
      */
-    static Path getProcessingCoredumpsPath(Path coredumpsPath) {
-        return coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME);
+    private void finishProcessing(NodeAgentContext context, Path coredumpDirectory) throws IOException {
+        Path coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory);
+        Path compressedCoreFile = coreFile.getParent().resolve(coreFile.getFileName() + ".lz4");
+        terminal.newCommandLine(context)
+                .add(LZ4_PATH, "-f", coreFile.toString(), compressedCoreFile.toString())
+                .execute();
+        Files.delete(coreFile);
+
+        Path newCoredumpDirectory = doneCoredumpsPath.resolve(coredumpDirectory.getFileName());
+        Files.move(coredumpDirectory, newCoredumpDirectory);
+    }
+
+    private Path findCoredumpFileInProcessingDirectory(Path coredumpProccessingDirectory) {
+        return FileFinder.files(coredumpProccessingDirectory)
+                .match(nameStartsWith(COREDUMP_FILENAME_PREFIX))
+                .maxDepth(1)
+                .stream()
+                .map(FileFinder.FileAttributes::path)
+                .findFirst()
+                .orElseThrow(() -> new IllegalStateException(
+                        "No coredump file found in processing directory " + coredumpProccessingDirectory));
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index c130480ff9c..644ac587c34 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
-import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
 import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
 
@@ -56,15 +55,6 @@ public class NodeAdminImpl implements NodeAdmin {
 
     public NodeAdminImpl(DockerOperations dockerOperations,
                          Function<String, NodeAgent> nodeAgentFactory,
-                         StorageMaintainer storageMaintainer,
-                         Runnable aclMaintainer,
-                         MetricReceiverWrapper metricReceiver,
-                         Clock clock) {
-        this(dockerOperations, nodeAgentFactory, aclMaintainer, metricReceiver, clock);
-    }
-
-    public NodeAdminImpl(DockerOperations dockerOperations,
-                         Function<String, NodeAgent> nodeAgentFactory,
                          Runnable aclMaintainer,
                          MetricReceiverWrapper metricReceiver,
                          Clock clock) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
index 63f469635f8..f65f371ff67 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
@@ -19,24 +19,42 @@ public interface NodeAgentContext extends TaskContext {
 
     AthenzService identity();
 
+
     /**
-     * Translates an absolute path in container to an absolute path in host.
+     * This method is the inverse of {@link #pathInNodeFromPathOnHost(Path)}}
      *
      * @param pathInNode absolute path in the container
      * @return the absolute path on host pointing at the same inode
      */
     Path pathOnHostFromPathInNode(Path pathInNode);
 
+    /** @see #pathOnHostFromPathInNode(Path) */
     default Path pathOnHostFromPathInNode(String pathInNode) {
         return pathOnHostFromPathInNode(Paths.get(pathInNode));
     }
 
+
+    /**
+     * This method is the inverse of {@link #pathOnHostFromPathInNode(Path)}
+     *
+     * @param pathOnHost absolute path on host
+     * @return the absolute path in the container pointing at the same inode
+     */
+    Path pathInNodeFromPathOnHost(Path pathOnHost);
+
+    /** @see #pathOnHostFromPathInNode(Path) */
+    default Path pathInNodeFromPathOnHost(String pathOnHost) {
+        return pathInNodeFromPathOnHost(Paths.get(pathOnHost));
+    }
+
+
     /**
      * @param relativePath relative path under Vespa home in container
      * @return the absolute path under Vespa home in the container
      */
     Path pathInNodeUnderVespaHome(Path relativePath);
 
+    /** @see #pathInNodeUnderVespaHome(Path) */
     default Path pathInNodeUnderVespaHome(String relativePath) {
         return pathInNodeUnderVespaHome(Paths.get(relativePath));
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
index 6d7110aeb51..d3c8b145488 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
@@ -59,15 +59,26 @@ public class NodeAgentContextImpl implements NodeAgentContext {
     @Override
     public Path pathOnHostFromPathInNode(Path pathInNode) {
         if (! pathInNode.isAbsolute())
-            throw new IllegalArgumentException("Expected an absolute path in container, got: " + pathInNode);
+            throw new IllegalArgumentException("Expected an absolute path in the container, got: " + pathInNode);
 
         return pathToNodeRootOnHost.resolve(ROOT.relativize(pathInNode).toString());
     }
 
     @Override
+    public Path pathInNodeFromPathOnHost(Path pathOnHost) {
+        if (! pathOnHost.isAbsolute())
+            throw new IllegalArgumentException("Expected an absolute path on the host, got: " + pathOnHost);
+
+        if (!pathOnHost.startsWith(pathToNodeRootOnHost))
+            throw new IllegalArgumentException("Path " + pathOnHost + " does not exist in the container");
+
+        return ROOT.resolve(pathToNodeRootOnHost.relativize(pathOnHost).toString());
+    }
+
+    @Override
     public Path pathInNodeUnderVespaHome(Path relativePath) {
         if (relativePath.isAbsolute())
-            throw new IllegalArgumentException("Expected a relative path to Vespa home, got: " + relativePath);
+            throw new IllegalArgumentException("Expected a relative path to the Vespa home, got: " + relativePath);
 
         return pathToVespaHome.resolve(relativePath);
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 45af3a85810..3af78593a58 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -378,6 +378,7 @@ public class NodeAgentImpl implements NodeAgent {
                 }
             }
             stopFilebeatSchedulerIfNeeded();
+            storageMaintainer.handleCoreDumpsForContainer(context, node, Optional.of(existingContainer));
             dockerOperations.removeContainer(existingContainer);
             containerState = ABSENT;
             context.log(logger, "Container successfully removed, new containerState is " + containerState);
@@ -491,7 +492,7 @@ public class NodeAgentImpl implements NodeAgent {
                 updateNodeRepoWithCurrentAttributes(node);
                 break;
             case active:
-                storageMaintainer.handleCoreDumpsForContainer(context, node);
+                storageMaintainer.handleCoreDumpsForContainer(context, node, container);
 
                 storageMaintainer.getDiskUsageFor(context)
                         .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.getMinDiskAvailableGb())
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
index 172203a281a..4fdca85363f 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
@@ -5,7 +5,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process;
 /**
  * @author hakonhall
  */
-interface ChildProcess2 extends AutoCloseable {
+public interface ChildProcess2 extends AutoCloseable {
     void waitForTermination();
     int exitCode();
     String getOutput();
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
index 8e66c6e5a48..44b7da9367b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
@@ -220,8 +220,10 @@ public class CommandLine {
         this.sigKillGracePeriod = period;
         return this;
     }
+
+    public List<String> getArguments() { return Collections.unmodifiableList(arguments); }
+
     // Accessor fields necessary for classes in this package. Could be public if necessary.
-    List<String> getArguments() { return Collections.unmodifiableList(arguments); }
     boolean getRedirectStderrToStdoutInsteadOfDiscard() { return redirectStderrToStdoutInsteadOfDiscard; }
     Predicate<Integer> getSuccessfulExitCodePredicate() { return successfulExitCodePredicate; }
     Charset getOutputEncoding() { return outputEncoding; }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 522bcc43444..3c316de94eb 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -100,7 +100,7 @@ public class DockerTester implements AutoCloseable {
         Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(
                 NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, hostName), nodeRepositoryMock,
                 orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer);
-        nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC());
+        nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC());
         nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer,
                 nodeAdmin, DOCKER_HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL,
                 Optional.of(new ClassLocking()));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
index 613b3cb5f5c..e91787ca540 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.system.ProcessExecuter;
+import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
@@ -27,7 +28,7 @@ public class StorageMaintainerMock extends StorageMaintainer {
     }
 
     @Override
-    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node) {
+    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) {
     }
 
     @Override
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index ca75a74cfcd..b5950646da9 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -1,26 +1,20 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
-import com.yahoo.collections.Pair;
-import com.yahoo.system.ProcessExecuter;
-import org.junit.Rule;
+import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
 
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -30,8 +24,10 @@ import static org.mockito.Mockito.when;
  * @author freva
  */
 public class CoreCollectorTest {
-    private final ProcessExecuter processExecuter = mock(ProcessExecuter.class);
-    private final CoreCollector coreCollector = new CoreCollector(processExecuter);
+    private final String GDB_PATH = "/my/path/to/gdb";
+    private final DockerOperations docker = mock(DockerOperations.class);
+    private final CoreCollector coreCollector = new CoreCollector(docker, Paths.get(GDB_PATH));
+    private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld");
 
     private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234");
     private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program");
@@ -40,41 +36,30 @@ public class CoreCollectorTest {
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t    printf(argv[3]);",
             "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4");
 
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder();
-
-    private void mockExec(String[] cmd, String output) throws IOException {
-        mockExec(cmd, output, "");
-    }
-
-    private void mockExec(String[] cmd, String output, String error) throws IOException {
-        when(processExecuter.exec(cmd)).thenReturn(new Pair<>(error.isEmpty() ? 0 : 1, output + error));
-    }
-
     @Test
-    public void extractsBinaryPathTest() throws IOException {
+    public void extractsBinaryPathTest() {
         final String[] cmd = {"file", TEST_CORE_PATH.toString()};
 
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program --foo --bar baz'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin//program'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " +
                         "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " +
                         "execfn: '/usr/bin/program', platform: 'x86_64");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
 
         Path fallbackResponse = Paths.get("/response/from/fallback");
@@ -82,57 +67,57 @@ public class CoreCollectorTest {
                 "Core was generated by `/response/from/fallback'.");
         mockExec(cmd,
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style");
-        assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH));
 
         mockExec(cmd, "", "Error code 1234");
-        assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH));
+        assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH));
     }
 
     @Test
-    public void extractsBinaryPathUsingGdbTest() throws IOException {
+    public void extractsBinaryPathUsingGdbTest() {
         final String[] cmd = new String[]{"/bin/sh", "-c",
                 GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"};
 
         mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'.");
-        assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(TEST_CORE_PATH));
+        assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(context, TEST_CORE_PATH));
 
         mockExec(cmd, "", "Error 123");
         try {
-            coreCollector.readBinPathFallback(TEST_CORE_PATH);
+            coreCollector.readBinPathFallback(context, TEST_CORE_PATH);
             fail("Expected not to be able to get bin path");
         } catch (RuntimeException e) {
-            assertEquals("Failed to extract binary path from GDB, result: (1,Error 123), command: " +
-                    "[/bin/sh, -c, /usr/bin/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage());
+            assertEquals("Failed to extract binary path from GDB, result: ProcessResult { exitStatus=1 output= errors=Error 123 }, command: " +
+                    "[/bin/sh, -c, /my/path/to/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage());
         }
     }
 
     @Test
-    public void extractsBacktraceUsingGdb() throws IOException {
+    public void extractsBacktraceUsingGdb() {
         mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
-        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false));
+        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false));
 
         mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
                 "", "Failure");
         try {
-            coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false);
+            coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false);
             fail("Expected not to be able to read backtrace");
         } catch (RuntimeException e) {
-            assertEquals("Failed to read backtrace (1,Failure), Command: " +
-                    "[/usr/bin/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage());
+            assertEquals("Failed to read backtrace ProcessResult { exitStatus=1 output= errors=Failure }, Command: " +
+                    "[/my/path/to/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage());
         }
     }
 
     @Test
-    public void extractsBacktraceFromAllThreadsUsingGdb() throws IOException {
+    public void extractsBacktraceFromAllThreadsUsingGdb() {
         mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch",
                         "/usr/bin/program", "/tmp/core.1234"},
                 String.join("\n", GDB_BACKTRACE));
-        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, true));
+        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true));
     }
 
     @Test
-    public void collectsDataTest() throws IOException {
+    public void collectsDataTest() {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
@@ -146,11 +131,11 @@ public class CoreCollectorTest {
         expectedData.put("bin_path", TEST_BIN_PATH.toString());
         expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE));
         expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE));
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
+        assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
     @Test
-    public void collectsPartialIfBacktraceFailsTest() throws IOException {
+    public void collectsPartialIfBacktraceFailsTest() {
         mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
                 "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
                         "'/usr/bin/program'");
@@ -159,63 +144,15 @@ public class CoreCollectorTest {
 
         Map<String, Object> expectedData = new HashMap<>();
         expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
-    }
-
-    @Test
-    public void parseTotalMemoryTestTest() {
-        String memInfo = "MemTotal:   100000000 kB\nMemUsed:   1000000 kB\n";
-        assertEquals(100000000, coreCollector.parseTotalMemorySize(memInfo));
-
-        String badMemInfo = "This string has no memTotal value";
-        try {
-            coreCollector.parseTotalMemorySize(badMemInfo);
-            fail("Expected to fail on parsing");
-        } catch (RuntimeException e) {
-            assertEquals("Could not parse meminfo: " + badMemInfo, e.getMessage());
-        }
+        assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH));
     }
 
-    @Test
-    public void testDeleteUncompressedFiles() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        final String coreDumpFilename = "core.dump";
-
-        Path coredumpPath = folder.newFolder("crash").toPath().resolve(documentId);
-        coredumpPath.toFile().mkdirs();
-        coredumpPath.resolve(coreDumpFilename).toFile().createNewFile();
-
-        Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList(
-                coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME),
-                coredumpPath.resolve(coreDumpFilename + ".lz4")));
-        expectedContentsOfCoredump.forEach(path -> {
-            try {
-                path.toFile().createNewFile();
-            } catch (IOException e) { e.printStackTrace();}
-        });
-        coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename));
-
-        assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet()));
+    private void mockExec(String[] cmd, String output) {
+        mockExec(cmd, output, "");
     }
 
-    @Test
-    public void testDeleteUncompressedFilesWithoutLz4() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        final String coreDumpFilename = "core.dump";
-
-        Path coredumpPath = folder.newFolder("crash").toPath().resolve(documentId);
-        coredumpPath.toFile().mkdirs();
-
-        Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList(
-                coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME),
-                coredumpPath.resolve(coreDumpFilename)));
-        expectedContentsOfCoredump.forEach(path -> {
-            try {
-                path.toFile().createNewFile();
-            } catch (IOException e) { e.printStackTrace();}
-        });
-        coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename));
-
-        assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet()));
+    private void mockExec(String[] cmd, String output, String error) {
+        when(docker.executeCommandInContainerAsRoot(context.containerName(), cmd))
+                .thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
index 8522112b0af..d567cac9a70 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -1,204 +1,204 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
 
 import java.io.IOException;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 /**
  * @author freva
  */
 public class CoredumpHandlerTest {
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final Path donePath = fileSystem.getPath("/home/docker/dumps");
+    private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, "container-123.domain.tld");
+    private final Path crashPathInContainer = Paths.get("/var/crash");
+    private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps");
 
-    private static final Map<String, Object> attributes = new LinkedHashMap<>();
-    private static final Map<String, Object> metadata = new LinkedHashMap<>();
-    private static final String expectedMetadataFileContents = "{\"fields\":{" +
-            "\"bin_path\":\"/bin/bash\"," +
-            "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," +
-            "\"hostname\":\"host123.yahoo.com\"," +
-            "\"vespa_version\":\"6.48.4\"," +
-            "\"kernel_version\":\"2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64\"," +
-            "\"docker_image\":\"vespa/ci:6.48.4\"}}";
-
-    static {
-        attributes.put("hostname", "host123.yahoo.com");
-        attributes.put("vespa_version", "6.48.4");
-        attributes.put("kernel_version", "2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64");
-        attributes.put("docker_image", "vespa/ci:6.48.4");
-
-        metadata.put("bin_path", "/bin/bash");
-        metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something"));
-    }
-
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder();
-
+    private final TestTerminal terminal = new TestTerminal();
     private final CoreCollector coreCollector = mock(CoreCollector.class);
     private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class);
-    private CoredumpHandler coredumpHandler;
-    private Path crashPath;
-    private Path donePath;
-    private Path processingPath;
+    @SuppressWarnings("unchecked")
+    private final Supplier<String> coredumpIdSupplier = mock(Supplier.class);
+    private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter,
+            crashPathInContainer, doneCoredumpsPath, coredumpIdSupplier);
 
-    @Before
-    public void setup() throws IOException {
-        crashPath = folder.newFolder("crash").toPath();
-        donePath = folder.newFolder("done").toPath();
-        processingPath = CoredumpHandler.getProcessingCoredumpsPath(crashPath);
-
-        coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter, donePath);
-    }
 
     @Test
-    public void ignoresIncompleteCoredumps() throws IOException {
-        Path coredumpPath = createCoredump(".core.dump", Instant.now());
-        coredumpHandler.enqueueCoredumps(crashPath);
-
-        // The 'processing' directory should be empty
-        assertFolderContents(processingPath);
-
-        // The 'crash' directory should have 'processing' and the incomplete core dump in it
-        assertFolderContents(crashPath, processingPath.getFileName().toString(), coredumpPath.getFileName().toString());
+    public void coredump_enqueue_test() throws IOException {
+        final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path");
+        final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing");
+
+        Files.createDirectories(crashPathOnHost);
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve(".bash.core.431")), FileTime.from(Instant.now()));
+
+        assertFolderContents(crashPathOnHost, ".bash.core.431");
+        Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir);
+        assertEquals(Optional.empty(), enqueuedPath);
+
+        // bash.core.431 finished writing... and 2 more have since been written
+        Files.move(crashPathOnHost.resolve(".bash.core.431"), crashPathOnHost.resolve("bash.core.431"));
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-proton.core.119")), FileTime.from(Instant.now().minus(Duration.ofMinutes(10))));
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-slobrok.core.673")), FileTime.from(Instant.now().minus(Duration.ofMinutes(5))));
+
+        when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321");
+        enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir);
+        assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+        assertFolderContents(crashPathOnHost, "bash.core.431", "vespa-slobrok.core.673");
+        assertFolderContents(processingDir, "id-123");
+        assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119");
+        verify(coredumpIdSupplier, times(1)).get();
+
+        // Enqueue another
+        enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir);
+        assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath);
+        assertFolderContents(crashPathOnHost, "bash.core.431");
+        assertFolderContents(processingDir, "id-123", "id-321");
+        assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673");
+        verify(coredumpIdSupplier, times(2)).get();
     }
 
     @Test
-    public void startProcessingTest() throws IOException {
-        Path coredumpPath = createCoredump("core.dump", Instant.now());
-        coredumpHandler.enqueueCoredumps(crashPath);
-
-        // Contents of 'crash' should be only the 'processing' directory
-        assertFolderContents(crashPath, processingPath.getFileName().toString());
-
-        // The 'processing' directory should have 1 directory inside for the core.dump we just created
-        List<Path> processedCoredumps = Files.list(processingPath).collect(Collectors.toList());
-        assertEquals(processedCoredumps.size(), 1);
-
-        // Inside the coredump directory, there should be 1 file: core.dump
-        assertFolderContents(processedCoredumps.get(0), coredumpPath.getFileName().toString());
+    public void coredump_to_process_test() throws IOException {
+        final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path");
+        final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing");
+
+        // Initially there are no core dumps
+        Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir);
+        assertEquals(Optional.empty(), enqueuedPath);
+
+        // 3 core dumps occur
+        Files.createDirectories(crashPathOnHost);
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("bash.core.431")), FileTime.from(Instant.now()));
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-proton.core.119")), FileTime.from(Instant.now().minus(Duration.ofMinutes(10))));
+        Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-slobrok.core.673")), FileTime.from(Instant.now().minus(Duration.ofMinutes(5))));
+
+        when(coredumpIdSupplier.get()).thenReturn("id-123");
+        enqueuedPath = coredumpHandler.getCoredumpToProcess(crashPathOnHost, processingDir);
+        assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+
+        // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously
+        enqueuedPath = coredumpHandler.getCoredumpToProcess(crashPathOnHost, processingDir);
+        assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+        verify(coredumpIdSupplier, times(1)).get();
     }
 
     @Test
-    public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException {
-        final String oldestCoredump = "core.dump0";
-        final Instant startTime = Instant.now();
-        createCoredump(oldestCoredump, startTime.minusSeconds(3600));
-        createCoredump("core.dump1", startTime.minusSeconds(1000));
-        createCoredump("core.dump2", startTime);
-        coredumpHandler.enqueueCoredumps(crashPath);
-
-        List<Path> processingCoredumps = Files.list(processingPath).collect(Collectors.toList());
-        assertEquals(1, processingCoredumps.size());
-
-        // Make sure that the 1 coredump that we are processing is the oldest one
-        Set<String> filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0))
-                .map(file -> file.getFileName().toString())
-                .collect(Collectors.toSet());
-        assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory);
+    public void get_metadata_test() throws IOException {
+        Map<String, Object> metadata = new HashMap<>();
+        metadata.put("bin_path", "/bin/bash");
+        metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something"));
 
-        // Running enqueueCoredumps should not start processing any new coredumps as we already are processing one
-        coredumpHandler.enqueueCoredumps(crashPath);
-        assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList()));
-        filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0))
-                .map(file -> file.getFileName().toString())
-                .collect(Collectors.toSet());
-        assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory);
+        Map<String, Object> attributes = ImmutableMap.of(
+                "hostname", "host123.yahoo.com",
+                "vespa_version", "6.48.4",
+                "kernel_version", "3.10.0-862.9.1.el7.x86_64",
+                "docker_image", "vespa/ci:6.48.4");
+
+        String expectedMetadataStr = "{\"fields\":{" +
+                "\"hostname\":\"host123.yahoo.com\"," +
+                "\"kernel_version\":\"3.10.0-862.9.1.el7.x86_64\"," +
+                "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," +
+                "\"vespa_version\":\"6.48.4\"," +
+                "\"bin_path\":\"/bin/bash\"," +
+                "\"docker_image\":\"vespa/ci:6.48.4\"" +
+                "}}";
+
+
+        Path coredumpDirectoryInContainer = Paths.get("/var/crash/id-123");
+        Path coredumpDirectory = context.pathOnHostFromPathInNode(coredumpDirectoryInContainer);
+        Files.createDirectories(coredumpDirectory);
+        Files.createFile(coredumpDirectory.resolve("dump_core.456"));
+        when(coreCollector.collect(eq(context), eq(coredumpDirectoryInContainer.resolve("dump_core.456"))))
+                .thenReturn(metadata);
+
+        assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, attributes));
+        verify(coreCollector, times(1)).collect(any(), any());
+
+        // Calling it again will simply read the previously generated metadata from disk
+        assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, attributes));
+        verify(coreCollector, times(1)).collect(any(), any());
     }
 
-    @Test
-    public void coredumpMetadataCollectAndWriteTest() throws IOException {
-        createCoredump("core.dump", Instant.now());
-        coredumpHandler.enqueueCoredumps(crashPath);
-        Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() ->
-                new RuntimeException("Expected to find directory with coredump in processing dir"));
-        when(coreCollector.collect(eq(processingCoredumpPath.resolve("core.dump")))).thenReturn(metadata);
-
-        // Inside 'processing' directory, there should be a new directory containing 'core.dump' file
-        String returnedMetadata = coredumpHandler.collectMetadata(processingCoredumpPath, attributes);
-        String metadataFileContents = new String(Files.readAllBytes(
-                processingCoredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME)));
-        assertEquals(expectedMetadataFileContents, metadataFileContents);
-        assertEquals(expectedMetadataFileContents, returnedMetadata);
+    @Test(expected = IllegalStateException.class)
+    public void cant_get_metadata_if_no_core_file() throws IOException {
+        coredumpHandler.getMetadata(context, fileSystem.getPath("/fake/path"), Collections.emptyMap());
     }
 
     @Test
-    public void coredumpMetadataReadIfExistsTest() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        Path metadataPath = createProcessedCoredump(documentId);
-
-        verifyZeroInteractions(coreCollector);
-        String returnedMetadata = coredumpHandler.collectMetadata(metadataPath.getParent(), attributes);
-        assertEquals(expectedMetadataFileContents, returnedMetadata);
+    public void process_single_coredump_test() throws IOException {
+        Path coredumpDirectory = fileSystem.getPath("/path/to/coredump/proccessing/id-123");
+        Files.createDirectories(coredumpDirectory);
+        Files.write(coredumpDirectory.resolve("metadata.json"), "metadata".getBytes());
+        Files.createFile(coredumpDirectory.resolve("dump_bash.core.431"));
+        assertFolderContents(coredumpDirectory, "metadata.json", "dump_bash.core.431");
+
+        terminal.interceptCommand("/usr/bin/lz4 -f /path/to/coredump/proccessing/id-123/dump_bash.core.431 " +
+                "/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1",
+                commandLine -> {
+                    uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3))));
+                    return new TestChildProcess2(0, "");
+                });
+        coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Collections.emptyMap());
+        verify(coreCollector, never()).collect(any(), any());
+        verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata"));
+        assertFalse(Files.exists(coredumpDirectory));
+        assertFolderContents(doneCoredumpsPath, "id-123");
+        assertFolderContents(doneCoredumpsPath.resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4");
     }
 
-    @Test
-    public void reportSuccessCoredumpTest() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        createProcessedCoredump(documentId);
-
-        coredumpHandler.processAndReportCoredumps(crashPath, attributes);
-        verify(coredumpReporter).reportCoredump(eq(documentId), eq(expectedMetadataFileContents));
-
-        // The coredump should not have been moved out of 'processing' and into 'done' as the report failed
-        assertFolderContents(processingPath);
-        assertFolderContents(donePath.resolve(documentId), CoredumpHandler.METADATA_FILE_NAME);
+    @Before
+    public void setup() throws IOException {
+        Files.createDirectories(donePath);
     }
 
-    @Test
-    public void reportFailCoredumpTest() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        Path metadataPath = createProcessedCoredump(documentId);
-
-        doThrow(new RuntimeException()).when(coredumpReporter).reportCoredump(any(), any());
-        coredumpHandler.processAndReportCoredumps(crashPath, attributes);
-        verify(coredumpReporter).reportCoredump(eq(documentId), eq(expectedMetadataFileContents));
-
-        // The coredump should not have been moved out of 'processing' and into 'done' as the report failed
-        assertFolderContents(donePath);
-        assertFolderContents(metadataPath.getParent(), CoredumpHandler.METADATA_FILE_NAME);
+    @After
+    public void teardown() {
+        terminal.verifyAllCommandsExecuted();
     }
 
-    private static void assertFolderContents(Path pathToFolder, String... filenames) throws IOException {
-        Set<Path> expectedContentsOfFolder = Arrays.stream(filenames)
-                .map(pathToFolder::resolve)
+    private static void assertFolderContents(Path pathToFolder, String... filenames) {
+        Set<String> expectedContentsOfFolder = new HashSet<>(Arrays.asList(filenames));
+        Set<String> actualContentsOfFolder = new UnixPath(pathToFolder)
+                .listContentsOfDirectory().stream()
+                .map(unixPath -> unixPath.toPath().getFileName().toString())
                 .collect(Collectors.toSet());
-        Set<Path> actualContentsOfFolder = Files.list(pathToFolder).collect(Collectors.toSet());
         assertEquals(expectedContentsOfFolder, actualContentsOfFolder);
     }
-
-    private Path createCoredump(String coredumpName, Instant lastModified) throws IOException {
-        Path coredumpPath = crashPath.resolve(coredumpName);
-        coredumpPath.toFile().createNewFile();
-        coredumpPath.toFile().setLastModified(lastModified.toEpochMilli());
-        return coredumpPath;
-    }
-
-    private Path createProcessedCoredump(String documentId) throws IOException {
-        Path coredumpPath = processingPath
-                .resolve(documentId)
-                .resolve(CoredumpHandler.METADATA_FILE_NAME);
-        coredumpPath.getParent().toFile().mkdirs();
-        return Files.write(coredumpPath, expectedMetadataFileContents.getBytes());
-    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index efd35cce00e..065b039c7fd 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -5,7 +5,6 @@ import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
-import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
 import org.junit.Test;
@@ -40,11 +39,10 @@ public class NodeAdminImplTest {
     private interface NodeAgentFactory extends Function<String, NodeAgent> {}
     private final DockerOperations dockerOperations = mock(DockerOperations.class);
     private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class);
-    private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
     private final Runnable aclMaintainer = mock(Runnable.class);
     private final ManualClock clock = new ManualClock();
 
-    private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer,
+    private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer,
             new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock);
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
index 2388e1e02b1..31ac8d1c114 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
@@ -15,23 +15,45 @@ import static org.junit.Assert.assertEquals;
  * @author freva
  */
 public class NodeAgentContextImplTest {
-    private final NodeAgentContext context = nodeAgentFromHostname("container-1.domain.tld");
-
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final NodeAgentContext context = nodeAgentFromHostname(fileSystem, "container-1.domain.tld");
 
     @Test
     public void path_on_host_from_path_in_node_test() {
         assertEquals(
                 "/home/docker/container-1",
-                context.pathOnHostFromPathInNode(Paths.get("/")).toString());
+                context.pathOnHostFromPathInNode("/").toString());
 
         assertEquals(
                 "/home/docker/container-1/dev/null",
-                context.pathOnHostFromPathInNode(Paths.get("/dev/null")).toString());
+                context.pathOnHostFromPathInNode("/dev/null").toString());
     }
 
     @Test(expected=IllegalArgumentException.class)
     public void path_in_container_must_be_absolute() {
-        context.pathOnHostFromPathInNode(Paths.get("some/relative/path"));
+        context.pathOnHostFromPathInNode("some/relative/path");
+    }
+
+    @Test
+    public void path_in_node_from_path_on_host_test() {
+        assertEquals(
+                "/dev/null",
+                context.pathInNodeFromPathOnHost(fileSystem.getPath("/home/docker/container-1/dev/null")).toString());
+    }
+
+    @Test(expected=IllegalArgumentException.class)
+    public void path_on_host_must_be_absolute() {
+        context.pathInNodeFromPathOnHost("some/relative/path");
+    }
+
+    @Test(expected=IllegalArgumentException.class)
+    public void path_on_host_must_be_inside_container_storage_of_context() {
+        context.pathInNodeFromPathOnHost(fileSystem.getPath("/home/docker/container-2/dev/null"));
+    }
+
+    @Test(expected=IllegalArgumentException.class)
+    public void path_on_host_must_be_inside_container_storage() {
+        context.pathInNodeFromPathOnHost(fileSystem.getPath("/home"));
     }
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 99936f56596..fbb584ea1d9 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -457,6 +457,8 @@ public class NodeAgentImplTest {
         nodeAgent.converge();
 
         final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository);
+        inOrder.verify(dockerOperations, times(1)).stopServices(eq(context.containerName()));
+        inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), eq(node), any());
         inOrder.verify(dockerOperations, times(1)).removeContainer(any());
         inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context));
         inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready));