Merge pull request #1624 from yahoo/freva/move-node-admin-maintenenace-to-own-module

Freva/move node admin maintenenace to own module

22 files changed, 292 insertions, 1590 deletions

diff --git a/node-admin/pom.xml b/node-admin/pom.xml
index 159124f6657..6216381bd7c 100644
--- a/node-admin/pom.xml
+++ b/node-admin/pom.xml
@@ -35,12 +35,6 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest-junit</artifactId>
-            <version>2.0.0.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
             <groupId>com.yahoo.vespa</groupId>
             <artifactId>container-dev</artifactId>
             <version>${project.version}</version>
@@ -79,6 +73,13 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-junit</artifactId>
+            <version>2.0.0.0</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
@@ -95,18 +96,6 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>airline</artifactId>
-            <version>0.7</version>
-        </dependency>
-
-        <!-- JSON parser for Maintenance JVM -->
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.6.2</version>
-        </dependency>
     </dependencies>
 
     <build>
@@ -129,25 +118,6 @@
                     </compilerArgs>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <finalName>node-admin-maintenance</finalName>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
index 336e20253b6..0785e292a79 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
@@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
+import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 
 import java.util.List;
@@ -25,7 +26,7 @@ public interface DockerOperations {
 
     void removeContainer(ContainerNodeSpec nodeSpec, Container existingContainer);
 
-    void executeCommandInContainer(ContainerName containerName, String[] command);
+    ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String[] command);
 
     String executeCommandInNetworkNamespace(ContainerName containerName, String[] command);
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index f405ade4081..810d4536d65 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -142,24 +142,6 @@ public class DockerOperationsImpl implements DockerOperations {
     }
 
     /**
-     * Executes a program and returns its result, or if it doesn't exist, return a result
-     * as-if the program executed with exit status 0 and no output.
-     */
-    Optional<ProcessResult> executeOptionalProgramInContainer(ContainerName containerName, String... args) {
-        assert args.length > 0;
-        String[] nodeProgramExistsCommand = programExistsCommand(args[0]);
-        if (!docker.executeInContainer(containerName, nodeProgramExistsCommand).isSuccess()) {
-            return Optional.empty();
-        }
-
-        return Optional.of(docker.executeInContainer(containerName, args));
-    }
-
-    String[] programExistsCommand(String programPath) {
-        return new String[]{ "/usr/bin/env", "test", "-x", programPath };
-    }
-
-    /**
      * Try to suspend node. Suspending a node means the node should be taken offline,
      * such that maintenance can be done of the node (upgrading, rebooting, etc),
      * and such that we will start serving again as soon as possible afterwards.
@@ -168,23 +150,15 @@ public class DockerOperationsImpl implements DockerOperations {
      */
     @Override
     public void trySuspendNode(ContainerName containerName) {
-        PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
-        Optional<ProcessResult> result;
-
         try {
             // TODO: Change to waiting w/o timeout (need separate thread that we can stop).
-            result = executeOptionalProgramInContainer(containerName, SUSPEND_NODE_COMMAND);
+            executeCommandInContainer(containerName, SUSPEND_NODE_COMMAND);
         } catch (RuntimeException e) {
+            PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
             // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to
             // remove container, we will not be able to upgrade to fix any problems in the suspend logic!
             logger.warning("Failed trying to suspend container " + containerName.asString() + "  with "
                    + Arrays.toString(SUSPEND_NODE_COMMAND), e);
-            return;
-        }
-
-        if (result.isPresent() && !result.get().isSuccess()) {
-            logger.warning("The suspend program " + Arrays.toString(SUSPEND_NODE_COMMAND)
-                    + " failed: " + result.get().getOutput() + " for container " + containerName.asString());
         }
     }
 
@@ -242,7 +216,7 @@ public class DockerOperationsImpl implements DockerOperations {
             }
 
             DIRECTORIES_TO_MOUNT.entrySet().stream().filter(Map.Entry::getValue).forEach(entry ->
-                    docker.executeInContainer(nodeSpec.containerName, "sudo", "chmod", "-R", "a+w", entry.getKey()));
+                    docker.executeInContainerAsRoot(nodeSpec.containerName, "chmod", "-R", "a+w", entry.getKey()));
         } catch (IOException e) {
             throw new RuntimeException("Failed to create container " + nodeSpec.containerName.asString(), e);
         }
@@ -288,14 +262,19 @@ public class DockerOperationsImpl implements DockerOperations {
         numberOfRunningContainersGauge.sample(getAllManagedContainers().size());
     }
 
-    @Override
-    public void executeCommandInContainer(ContainerName containerName, String[] command) {
-        Optional<ProcessResult> result = executeOptionalProgramInContainer(containerName, command);
+    ProcessResult executeCommandInContainer(ContainerName containerName, String[] command) {
+        ProcessResult result = docker.executeInContainerAsRoot(containerName, command);
 
-        if (result.isPresent() && !result.get().isSuccess()) {
-            throw new RuntimeException("Container " + containerName.asString()
-                                               + ": command " + Arrays.toString(command) + " failed: " + result.get());
+        if (! result.isSuccess()) {
+            throw new RuntimeException("Container " + containerName.asString() +
+                    ": command " + Arrays.toString(command) + " failed: " + result);
         }
+        return result;
+    }
+
+    @Override
+    public ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String[] command) {
+        return docker.executeInContainerAsRoot(containerName, command);
     }
 
     @Override
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index a9048b06cf5..92d33c49f74 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -1,39 +1,65 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.collections.Pair;
 import com.yahoo.io.IOUtils;
+import com.yahoo.net.HostName;
+import com.yahoo.system.ProcessExecuter;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.Docker;
+import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
+import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
 import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
-import com.yahoo.vespa.hosted.node.maintenance.DeleteOldAppData;
-import com.yahoo.vespa.hosted.node.maintenance.Maintainer;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
 
 /**
  * @author freva
  */
 public class StorageMaintainer {
-    private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(StorageMaintainer.class);
+    private static final ContainerName NODE_ADMIN = new ContainerName("node-admin");
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static Optional<String> kernelVersion = Optional.empty();
+
     private static final long intervalSec = 1000;
 
     private final Object monitor = new Object();
+    private final CounterWrapper numberOfNodeAdminMaintenanceFails;
+    private final Docker docker;
     private final Environment environment;
 
     private Map<ContainerName, MetricsCache> metricsCacheByContainerName = new ConcurrentHashMap<>();
 
 
-    public StorageMaintainer(Environment environment) {
+    public StorageMaintainer(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) {
+        this.docker = docker;
         this.environment = environment;
+
+        Dimensions dimensions = new Dimensions.Builder()
+                .add("host", HostName.getLocalhost())
+                .add("role", "docker").build();
+
+        numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(dimensions, "nodes.maintenance.fails");
     }
 
     public Map<String, Number> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) {
@@ -48,7 +74,7 @@ public class StorageMaintainer {
             // Throttle to one disk usage calculation at a time.
             synchronized (monitor) {
                 PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName);
-                File containerDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/").toFile();
+                Path containerDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/");
                 try {
                     long used = getDiscUsedInBytes(containerDir);
                     metricsCache.metrics.put("node.disk.used", used);
@@ -64,7 +90,7 @@ public class StorageMaintainer {
     }
 
     // Public for testing
-    long getDiscUsedInBytes(File path) throws IOException, InterruptedException {
+    long getDiscUsedInBytes(Path path) throws IOException, InterruptedException {
         final String[] command = {"du", "-xsk", path.toString()};
 
         Process duCommand = new ProcessBuilder().command(command).start();
@@ -83,51 +109,207 @@ public class StorageMaintainer {
         if (results.length != 2) {
             throw new RuntimeException("Result from disk usage command not as expected: " + output);
         }
-        long diskUsageKB = Long.valueOf(results[0]);
 
+        long diskUsageKB = Long.valueOf(results[0]);
         return diskUsageKB * 1024;
     }
 
+
+    /**
+     * Deletes old log files for vespa, nginx, logstash, etc.
+     */
     public void removeOldFilesFromNode(ContainerName containerName) {
+        MaintainerExecutor maintainerExecutor = new MaintainerExecutor();
         String[] pathsToClean = {"/home/y/logs/elasticsearch2", "/home/y/logs/logstash2",
                 "/home/y/logs/daemontools_y", "/home/y/logs/nginx", "/home/y/logs/vespa"};
+
         for (String pathToClean : pathsToClean) {
-            File path = environment.pathInNodeAdminFromPathInNode(containerName, pathToClean).toFile();
-            if (path.exists()) {
-                DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*\\.log\\..+", false);
-                DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*QueryAccessLog.*", false);
+            Path path = environment.pathInNodeAdminFromPathInNode(containerName, pathToClean);
+            if (Files.exists(path)) {
+                maintainerExecutor.addJob("delete-files")
+                        .withArgument("basePath", path)
+                        .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds())
+                        .withArgument("fileNameRegex", ".*\\.log\\..+")
+                        .withArgument("recursive", false);
+
+                maintainerExecutor.addJob("delete-files")
+                        .withArgument("basePath", path)
+                        .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds())
+                        .withArgument("fileNameRegex", ".*QueryAccessLog.*")
+                        .withArgument("recursive", false);
             }
         }
 
-        File logArchiveDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/logs/vespa/logarchive").toFile();
-        if (logArchiveDir.exists()) {
-            DeleteOldAppData.deleteFiles(logArchiveDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false);
-        }
+        Path logArchiveDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/logs/vespa/logarchive");
+        maintainerExecutor.addJob("delete-files")
+                .withArgument("basePath", logArchiveDir)
+                .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds())
+                .withArgument("recursive", false);
 
-        File fileDistrDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/var/db/vespa/filedistribution").toFile();
-        if (fileDistrDir.exists()) {
-            DeleteOldAppData.deleteFiles(fileDistrDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false);
-        }
+        Path fileDistrDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/var/db/vespa/filedistribution");
+        maintainerExecutor.addJob("delete-files")
+                .withArgument("basePath", fileDistrDir)
+                .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds())
+                .withArgument("recursive", false);
+
+        maintainerExecutor.execute();
     }
 
+    /**
+     * Checks if container has any new coredumps, reports and archives them if so
+     */
     public void handleCoreDumpsForContainer(ContainerNodeSpec nodeSpec, Environment environment) {
-        PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, nodeSpec.containerName);
+        Map<String, Object> attributes = new HashMap<>();
+        attributes.put("hostname", nodeSpec.hostname);
+        attributes.put("parent_hostname", HostName.getLocalhost());
+        attributes.put("region", environment.getRegion());
+        attributes.put("environment", environment.getEnvironment());
+        attributes.put("flavor", nodeSpec.nodeFlavor);
+        try {
+            attributes.put("kernel_version", getKernelVersion());
+        } catch (Throwable ignored) {
+            attributes.put("kernel_version", "unknown");
+        }
 
-        Maintainer.handleCoreDumpsForContainer(logger, nodeSpec, environment);
+        nodeSpec.wantedDockerImage.ifPresent(image -> attributes.put("docker_image", image.asString()));
+        nodeSpec.vespaVersion.ifPresent(version -> attributes.put("vespa_version", version));
+        nodeSpec.owner.ifPresent(owner -> {
+            attributes.put("tenant", owner.tenant);
+            attributes.put("application", owner.application);
+            attributes.put("instance", owner.instance);
+        });
+
+        MaintainerExecutor maintainerExecutor = new MaintainerExecutor(true);
+        maintainerExecutor.addJob("handle-core-dumps")
+                .withArgument("doneCoredumpsPath", environment.pathInNodeAdminToDoneCoredumps())
+                .withArgument("containerCoredumpsPath", environment.pathInNodeAdminFromPathInNode(nodeSpec.containerName, "/home/y/var/crash"))
+                .withArgument("attributes", attributes);
+        maintainerExecutor.execute();
     }
 
+    /**
+     * Deletes old
+     *  * archived app data
+     *  * archived and reported coredumps
+     *  * JDisc logs
+     */
     public void cleanNodeAdmin() {
-        Maintainer.deleteOldAppData(NODE_ADMIN_LOGGER);
-        Maintainer.cleanCoreDumps(NODE_ADMIN_LOGGER);
+        MaintainerExecutor maintainerExecutor = new MaintainerExecutor(true);
+        maintainerExecutor.addJob("delete-directories")
+                .withArgument("basePath", environment.getPathResolver().getApplicationStoragePathForNodeAdmin())
+                .withArgument("maxAgeSeconds", Duration.ofDays(7).getSeconds())
+                .withArgument("dirNameRegex", "^" + Pattern.quote(Environment.APPLICATION_STORAGE_CLEANUP_PATH_PREFIX));
+
+        maintainerExecutor.addJob("delete-directories")
+                .withArgument("basePath", environment.pathInNodeAdminToDoneCoredumps())
+                .withArgument("maxAgeSeconds", Duration.ofDays(10).getSeconds());
 
-        File nodeAdminJDiskLogsPath = environment.pathInNodeAdminFromPathInNode(new ContainerName("node-admin"),
-                "/home/y/logs/jdisc_core/").toFile();
-        DeleteOldAppData.deleteFiles(nodeAdminJDiskLogsPath.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false);
+        Path nodeAdminJDiskLogsPath = environment.pathInNodeAdminFromPathInNode(NODE_ADMIN, "/home/y/logs/jdisc_core/");
+        maintainerExecutor.addJob("delete-files")
+                .withArgument("basePath", nodeAdminJDiskLogsPath)
+                .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds())
+                .withArgument("recursive", false);
+        maintainerExecutor.execute();
     }
 
+    /**
+     * Archives container data, runs when container enters state "dirty"
+     */
     public void archiveNodeData(ContainerName containerName) {
-        PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName);
-        Maintainer.archiveAppData(logger, containerName);
+        MaintainerExecutor maintainerExecutor = new MaintainerExecutor(true);
+        maintainerExecutor.addJob("recursive-delete")
+                .withArgument("path", environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/var"));
+
+        maintainerExecutor.addJob("move-files")
+                .withArgument("from", environment.pathInNodeAdminFromPathInNode(containerName, "/"))
+                .withArgument("to", environment.pathInNodeAdminToNodeCleanup(containerName));
+
+        maintainerExecutor.execute();
+    }
+
+
+
+    private String getKernelVersion() throws IOException, InterruptedException {
+        if (! kernelVersion.isPresent()) {
+            Pair<Integer, String> result = new ProcessExecuter().exec(new String[]{"uname", "-r"});
+            if (result.getFirst() == 0) {
+                kernelVersion = Optional.of(result.getSecond().trim());
+            } else {
+                throw new RuntimeException("Failed to get kernel version\n" + result);
+            }
+        }
+
+        return kernelVersion.orElse("unknown");
+    }
+
+    /**
+     * Wrapper for node-admin-maintenance, queues up maintenances jobs and sends a single request to maintenance JVM
+     */
+    private class MaintainerExecutor {
+        private final List<MaintainerExecutorJob> jobs = new ArrayList<>();
+        private final ContainerName executeIn;
+        private final boolean runAsRoot;
+
+        MaintainerExecutor(ContainerName executeIn, boolean runAsRoot) {
+            this.executeIn = executeIn;
+            this.runAsRoot = runAsRoot;
+        }
+
+        MaintainerExecutor(boolean runAsRoot) {
+            this(NODE_ADMIN, runAsRoot);
+        }
+
+        MaintainerExecutor() {
+            this(false);
+        }
+
+        MaintainerExecutorJob addJob(String jobName) {
+            MaintainerExecutorJob job = new MaintainerExecutorJob(jobName);
+            jobs.add(job);
+            return job;
+        }
+
+        ProcessResult execute() {
+            String classPath = String.join(":",
+                    "/home/y/lib/jars/node-admin-maintenance-jar-with-dependencies.jar",
+                    "/home/y/lib/jars/vespajlib.jar");
+
+            String args;
+            try {
+                args = objectMapper.writeValueAsString(jobs);
+            } catch (JsonProcessingException e) {
+                throw new RuntimeException("Failed trasform list of maintenance jobs to JSON");
+            }
+
+            String[] command = {"java", "-cp", classPath, "com.yahoo.vespa.hosted.node.maintenance.Maintainer", args};
+            ProcessResult result = runAsRoot ?  docker.executeInContainerAsRoot(executeIn, command) :
+                                                docker.executeInContainer(executeIn, command);
+
+            if (! result.isSuccess()) {
+                PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, executeIn);
+                logger.warning("Failed to run maintenance jobs: " + args + result);
+                numberOfNodeAdminMaintenanceFails.add();
+            }
+            return result;
+        }
+    }
+
+    private class MaintainerExecutorJob {
+        @JsonProperty(value="jobName")
+        private final String jobName;
+
+        @JsonProperty(value="arguments")
+        private final Map<String, Object> arguments = new HashMap<>();
+
+        MaintainerExecutorJob(String jobName) {
+            this.jobName = jobName;
+        }
+
+        MaintainerExecutorJob withArgument(String argument, Object value) {
+            // Transform Path to String, otherwise ObjectMapper wont encode/decode it properly on the other end
+            arguments.put(argument, (value instanceof Path) ? value.toString() : value);
+            return this;
+        }
     }
 
     private static class MetricsCache {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 7f1ce37c1d2..0c3f0f4a139 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -579,7 +579,7 @@ public class NodeAgentImpl implements NodeAgent {
         try {
             scheduleMaker.writeTo(yamasAgentFolder);
             final String[] restartYamasAgent = new String[] {"service" , "yamas-agent", "restart"};
-            dockerOperations.executeCommandInContainer(nodeSpec.containerName, restartYamasAgent);
+            dockerOperations.executeCommandInContainerAsRoot(nodeSpec.containerName, restartYamasAgent);
         } catch (IOException e) {
             throw new RuntimeException("Failed to write secret-agent schedules for " + nodeSpec.containerName, e);
         }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
index 67d4b28472d..1f64c02e6a4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
@@ -77,7 +77,8 @@ public class ComponentsProviderImpl implements ComponentsProvider {
                 docker,
                 metricReceiver,
                 new Environment(),
-                config.isRunningLocally() ? Optional.empty() : Optional.of(new StorageMaintainer(new Environment())));
+                config.isRunningLocally() ? Optional.empty() :
+                                            Optional.of(new StorageMaintainer(docker, metricReceiver, new Environment())));
 
         if (! config.isRunningLocally()) {
             setCorePattern(docker);
@@ -98,12 +99,12 @@ public class ComponentsProviderImpl implements ComponentsProvider {
 
     private void setCorePattern(Docker docker) {
         final String[] sysctlCorePattern = {"sysctl", "-w", "kernel.core_pattern=/home/y/var/crash/%e.core.%p"};
-        docker.executeInContainer(NODE_ADMIN_CONTAINER_NAME, sysctlCorePattern);
+        docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, sysctlCorePattern);
     }
 
     private void initializeNodeAgentSecretAgent(Docker docker) {
         final Path yamasAgentFolder = Paths.get("/etc/yamas-agent/");
-        docker.executeInContainer(NODE_ADMIN_CONTAINER_NAME, "sudo", "chmod", "a+w", yamasAgentFolder.toString());
+        docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "chmod", "a+w", yamasAgentFolder.toString());
 
         Path nodeAdminCheckPath = Paths.get("/usr/bin/curl");
         SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("node-admin", 60, nodeAdminCheckPath,
@@ -111,7 +112,7 @@ public class ComponentsProviderImpl implements ComponentsProvider {
 
         try {
             scheduleMaker.writeTo(yamasAgentFolder);
-            docker.executeInContainer(NODE_ADMIN_CONTAINER_NAME, "service", "yamas-agent", "restart");
+            docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "service", "yamas-agent", "restart");
         } catch (IOException e) {
             throw new RuntimeException("Failed to write secret-agent schedules for node-admin", e);
         }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollector.java
deleted file mode 100644
index 3f0f504bf65..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollector.java
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Takes in a compressed (lz4) or uncompressed core dump and collects relevant metadata.
- *
- * @author freva
- */
-public class CoreCollector {
-    private static final String GDB_PATH = "/home/y/bin64/gdb";
-    private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$");
-    private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'");
-    private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'");
-    private static final Pattern TOTAL_MEMORY_PATTERN = Pattern.compile("^MemTotal:\\s*(?<totalMem>\\d+) kB$", Pattern.MULTILINE);
-
-    private static final Logger logger = Logger.getLogger(CoreCollector.class.getName());
-
-    private final Maintainer maintainer;
-
-    public CoreCollector(Maintainer maintainer) {
-        this.maintainer = maintainer;
-    }
-
-    Path readBinPathFallback(Path coredumpPath) throws IOException, InterruptedException {
-        String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
-        ProcessResult result = maintainer.exec("sh", "-c", "\"" + command + "\"");
-
-        Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput());
-        if (! matcher.find()) {
-            throw new RuntimeException("Failed to extract binary path from " + result);
-        }
-        return Paths.get(matcher.group("path").split(" ")[0]);
-    }
-
-    Path readBinPath(Path coredumpPath) throws IOException, InterruptedException {
-        try {
-            ProcessResult result = maintainer.exec("file", coredumpPath.toString());
-
-            Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getOutput());
-            if (execfnMatcher.find()) {
-                return Paths.get(execfnMatcher.group("path").split(" ")[0]);
-            }
-
-            Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getOutput());
-            if (fromMatcher.find()) {
-                return Paths.get(fromMatcher.group("path").split(" ")[0]);
-            }
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, "Failed getting bin path, trying fallback instead", e);
-        }
-
-        return readBinPathFallback(coredumpPath);
-    }
-
-    List<String> readBacktrace(Path coredumpPath, Path binPath, boolean allThreads) throws IOException, InterruptedException {
-        String threads = allThreads ? "thread apply all bt" : "bt";
-        ProcessResult result = maintainer.exec(GDB_PATH, "-n", "-ex", threads, "-batch",
-                binPath.toString(), coredumpPath.toString());
-        if (! result.isSuccess()) {
-            throw new RuntimeException("Failed to read backtrace " + result);
-        }
-        return Arrays.asList(result.getOutput().split("\n"));
-    }
-
-    public Map<String, Object> collect(Path coredumpPath) {
-        Map<String, Object> data = new LinkedHashMap<>();
-        try {
-            coredumpPath = compressCoredump(coredumpPath);
-            Path binPath = readBinPath(coredumpPath);
-
-            data.put("bin_path", binPath.toString()); // Gson can't deal with Path
-            data.put("backtrace", readBacktrace(coredumpPath, binPath, false));
-            data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true));
-
-            deleteDecompressedCoredump(coredumpPath);
-        } catch (Throwable e) {
-            logger.log(Level.WARNING, "Failed to collect core dump data", e);
-        }
-        return data;
-    }
-
-
-    /**
-     * This method will either compress or decompress the core dump if the input path is to a decompressed or
-     * compressed core dump, respectively.
-     *
-     * @return Path to the decompressed core dump
-     */
-    private Path compressCoredump(Path coredumpPath) throws IOException, InterruptedException {
-        if (! coredumpPath.toString().endsWith(".lz4")) {
-            maintainer.exec("/home/y/bin64/lz4", coredumpPath.toString(), coredumpPath.toString() + ".lz4");
-            return coredumpPath;
-
-        } else {
-            if (!diskSpaceAvailable(coredumpPath)) {
-                throw new RuntimeException("Not decompressing " + coredumpPath + " due to not enough disk space available");
-            }
-
-            Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", ""));
-            ProcessResult result = maintainer.exec("/home/y/bin64/lz4", "-d", coredumpPath.toString(), decompressedPath.toString());
-            if (!result.isSuccess()) {
-                throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result);
-            }
-            return decompressedPath;
-        }
-    }
-
-    /**
-     * Delete the core dump unless:
-     * - The file is compressed
-     * - There is no compressed file (i.e. it was not decompressed in the first place)
-     */
-    void deleteDecompressedCoredump(Path coredumpPath) throws IOException {
-        if (! coredumpPath.toString().endsWith(".lz4") && Paths.get(coredumpPath.toString() + ".lz4").toFile().exists()) {
-            Files.delete(coredumpPath);
-        }
-    }
-
-    private boolean diskSpaceAvailable(Path path) throws IOException {
-        String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo")));
-        return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo);
-    }
-
-    int parseTotalMemorySize(String memInfo) {
-        Matcher matcher = TOTAL_MEMORY_PATTERN.matcher(memInfo);
-        if (!matcher.find()) throw new RuntimeException("Could not parse meminfo: " + memInfo);
-        return Integer.valueOf(matcher.group("totalMem"));
-    }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandler.java
deleted file mode 100644
index 2004a126d81..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandler.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import com.google.gson.Gson;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Finds coredumps, collects metadata and reports them
- *
- * @author freva
- */
-public class CoredumpHandler {
-    public static final String FEED_ENDPOINT = "http://panic.vespa.us-west-1.prod.vespa.yahooapis.com:4080/document/v1/panic/core_dump/docid";
-    public static final String PROCESSING_DIRECTORY_NAME = "processing";
-    public static final String METADATA_FILE_NAME = "metadata.json";
-
-    private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName());
-    private final Gson gson = new Gson();
-
-    private final HttpClient httpClient;
-    private final CoreCollector coreCollector;
-
-    public CoredumpHandler(HttpClient httpClient, CoreCollector coreCollector) {
-        this.httpClient = httpClient;
-        this.coreCollector = coreCollector;
-    }
-
-    public void processAndReportCoredumps(Path coredumpsPath, Path doneCoredumpPath, Map<String, Object> nodeAttributes) throws IOException {
-        Path processingCoredumps = processCoredumps(coredumpsPath, nodeAttributes);
-        reportCoredumps(processingCoredumps, doneCoredumpPath);
-    }
-
-    public void removeJavaCoredumps(Path javaCoredumpsPath) {
-        if (! javaCoredumpsPath.toFile().isDirectory()) return;
-        DeleteOldAppData.deleteFiles(javaCoredumpsPath.toString(), 0, "^java_pid.*\\.hprof$", false);
-    }
-
-    Path processCoredumps(Path coredumpsPath, Map<String, Object> nodeAttributes) throws IOException {
-        Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME);
-        processingCoredumpsPath.toFile().mkdirs();
-
-        Files.list(coredumpsPath)
-                .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith("."))
-                .forEach(coredumpPath -> {
-                    try {
-                        coredumpPath.toFile().setReadable(true, false);
-                        coredumpPath = startProcessing(coredumpPath, processingCoredumpsPath);
-
-                        Path metadataPath = coredumpPath.getParent().resolve(METADATA_FILE_NAME);
-                        Map<String, Object> metadata = collectMetadata(coredumpPath, nodeAttributes);
-                        writeMetadata(metadataPath, metadata);
-                    } catch (Throwable e) {
-                        logger.log(Level.WARNING, "Failed to process coredump " + coredumpPath, e);
-                    }
-                });
-
-        return processingCoredumpsPath;
-    }
-
-    void reportCoredumps(Path processingCoredumpsPath, Path doneCoredumpsPath) throws IOException {
-        doneCoredumpsPath.toFile().mkdirs();
-
-        Files.list(processingCoredumpsPath)
-                .filter(path -> path.toFile().isDirectory())
-                .forEach(coredumpDirectory -> {
-                    try {
-                        report(coredumpDirectory);
-                        finishProcessing(coredumpDirectory, doneCoredumpsPath);
-                    } catch (Throwable e) {
-                        logger.log(Level.WARNING, "Failed to report coredump " + coredumpDirectory, e);
-                    }
-                });
-    }
-
-    public void removeOldCoredumps(Path doneCoredumpsPath) {
-        DeleteOldAppData.deleteDirectories(doneCoredumpsPath.toString(), Duration.ofDays(10).getSeconds(), null);
-    }
-
-    Path startProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException {
-        Path folder = processingCoredumpsPath.resolve(UUID.randomUUID().toString());
-        folder.toFile().mkdirs();
-        return Files.move(coredumpPath, folder.resolve(coredumpPath.getFileName()));
-    }
-
-    private Map<String, Object> collectMetadata(Path coredumpPath, Map<String, Object> nodeAttributes) {
-        Map<String, Object> metadata = coreCollector.collect(coredumpPath);
-        metadata.putAll(nodeAttributes);
-
-        Map<String, Object> fields = new HashMap<>();
-        fields.put("fields", metadata);
-        return fields;
-    }
-
-    private void writeMetadata(Path metadataPath, Map<String, Object> metadata) throws IOException {
-        Files.write(metadataPath, gson.toJson(metadata).getBytes());
-    }
-
-    void report(Path coredumpDirectory) throws IOException {
-        // Use core dump UUID as document ID
-        String documentId = coredumpDirectory.getFileName().toString();
-        String metadata = new String(Files.readAllBytes(coredumpDirectory.resolve(METADATA_FILE_NAME)));
-
-        HttpPost post = new HttpPost(FEED_ENDPOINT + "/" + documentId);
-        post.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
-        post.setEntity(new StringEntity(metadata));
-
-        HttpResponse response = httpClient.execute(post);
-        if (response.getStatusLine().getStatusCode() / 100 != 2) {
-            String result = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))
-                    .lines().collect(Collectors.joining("\n"));
-            throw new RuntimeException("POST to " + post.getURI() + " failed with HTTP: " +
-                    response.getStatusLine().getStatusCode() + " [" + result + "]");
-        }
-        logger.info("Successfully reported coredump " + documentId);
-    }
-
-    void finishProcessing(Path coredumpDirectory, Path doneCoredumpsPath) throws IOException {
-        Files.move(coredumpDirectory, doneCoredumpsPath.resolve(coredumpDirectory.getFileName()));
-    }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppData.java
deleted file mode 100644
index ed3f3653fee..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppData.java
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-/**
- * @author freva
- */
-
-public class DeleteOldAppData {
-    private static final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(DeleteOldAppData.class);
-
-    /**
-     * (Recursively) deletes files if they match all the criteria, also deletes empty directories.
-     *
-     * @param basePath      Base path from where to start the search
-     * @param maxAgeSeconds Delete files older (last modified date) than maxAgeSeconds
-     * @param fileNameRegex Delete files where filename matches fileNameRegex
-     * @param recursive     Delete files in sub-directories (with the same criteria)
-     */
-    public static void deleteFiles(String basePath, long maxAgeSeconds, String fileNameRegex, boolean recursive) {
-        Pattern fileNamePattern = fileNameRegex != null ? Pattern.compile(fileNameRegex) : null;
-        File[] filesInDeleteDirectory = getContentsOfDirectory(basePath);
-
-        for (File file : filesInDeleteDirectory) {
-            if (file.isDirectory()) {
-                if (recursive) {
-                    deleteFiles(file.getAbsolutePath(), maxAgeSeconds, fileNameRegex, true);
-                    if (file.list().length == 0 && !file.delete()) {
-                        logger.warning("Could not delete directory: " + file.getAbsolutePath());
-                    }
-                }
-            } else if (isPatternMatchingFilename(fileNamePattern, file) &&
-                    isTimeSinceLastModifiedMoreThan(file, Duration.ofSeconds(maxAgeSeconds))) {
-                if (!file.delete()) {
-                    logger.warning("Could not delete file: " + file.getAbsolutePath());
-                }
-            }
-        }
-    }
-
-    /**
-     * Deletes all files in target directory except the n most recent (by modified date)
-     *
-     * @param basePath          Base path to delete from
-     * @param nMostRecentToKeep Number of most recent files to keep
-     */
-    public static void deleteFilesExceptNMostRecent(String basePath, int nMostRecentToKeep) {
-        File[] deleteDirContents = getContentsOfDirectory(basePath);
-
-        if (nMostRecentToKeep < 1) {
-            throw new IllegalArgumentException("Number of files to keep must be a positive number");
-        }
-
-        List<File> filesInDeleteDir = Arrays.stream(deleteDirContents).filter(File::isFile).collect(Collectors.toList());
-        if (filesInDeleteDir.size() <= nMostRecentToKeep) return;
-
-        Collections.sort(filesInDeleteDir, (f1, f2) -> Long.signum(f1.lastModified() - f2.lastModified()));
-
-        for (int i = nMostRecentToKeep; i < filesInDeleteDir.size(); i++) {
-            if (!filesInDeleteDir.get(i).delete()) {
-                logger.warning("Could not delete file: " + filesInDeleteDir.get(i).getAbsolutePath());
-            }
-        }
-    }
-
-    public static void deleteFilesLargerThan(File baseDirectory, long sizeInBytes) {
-        File[] filesInBaseDirectory = getContentsOfDirectory(baseDirectory.getAbsolutePath());
-
-        for (File file : filesInBaseDirectory) {
-            if (file.isDirectory()) {
-                deleteFilesLargerThan(file, sizeInBytes);
-            } else {
-                if (file.length() > sizeInBytes && !file.delete()) {
-                    logger.warning("Could not delete file: " + file.getAbsolutePath());
-                }
-            }
-        }
-    }
-
-    /**
-     * Deletes directories and their contents if they match all the criteria
-     *
-     * @param basePath      Base path to delete the directories from
-     * @param maxAgeSeconds Delete directories older (last modified date) than maxAgeSeconds
-     * @param dirNameRegex  Delete directories where directory name matches dirNameRegex
-     */
-    public static void deleteDirectories(String basePath, long maxAgeSeconds, String dirNameRegex) {
-        Pattern dirNamePattern = dirNameRegex != null ? Pattern.compile(dirNameRegex) : null;
-        File[] filesInDeleteDirectory = getContentsOfDirectory(basePath);
-
-        for (File file : filesInDeleteDirectory) {
-            if (file.isDirectory() &&
-                    isPatternMatchingFilename(dirNamePattern, file) &&
-                    isTimeSinceLastModifiedMoreThan(getMostRecentlyModifiedFileIn(file), Duration.ofSeconds(maxAgeSeconds))) {
-                deleteFiles(file.getPath(), 0, null, true);
-                if (file.list().length == 0 && !file.delete()) {
-                    logger.warning("Could not delete directory: " + file.getAbsolutePath());
-                }
-            }
-        }
-    }
-
-    /**
-     * Similar to rm -rf file:
-     *   - It's not an error if file doesn't exist
-     *   - If file is a directory, it and all content is removed
-     *   - For symlinks: Only the symlink is removed, not what the symlink points to
-     */
-    public static void recursiveDelete(File file) throws IOException {
-        if (file.isDirectory()) {
-            for (File childFile : file.listFiles()) {
-                recursiveDelete(childFile);
-            }
-        }
-
-        Files.deleteIfExists(file.toPath());
-    }
-
-    static File[] getContentsOfDirectory(String directoryPath) {
-        File directory = new File(directoryPath);
-        File[] directoryContents = directory.listFiles();
-
-        return directoryContents == null ? new File[0] : directoryContents;
-    }
-
-    private static File getMostRecentlyModifiedFileIn(File baseFile) {
-        File mostRecent = baseFile;
-        File[] filesInDirectory = getContentsOfDirectory(baseFile.getAbsolutePath());
-
-        for (File file : filesInDirectory) {
-            if (file.isDirectory()) {
-                file = getMostRecentlyModifiedFileIn(file);
-            }
-
-            if (file.lastModified() > mostRecent.lastModified()) {
-                mostRecent = file;
-            }
-        }
-        return mostRecent;
-    }
-
-    private static boolean isTimeSinceLastModifiedMoreThan(File file, Duration duration) {
-        return System.currentTimeMillis() - file.lastModified() > duration.toMillis();
-    }
-
-    private static boolean isPatternMatchingFilename(Pattern pattern, File file) {
-        return pattern == null || pattern.matcher(file.getName()).find();
-    }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java
deleted file mode 100644
index c36aafd54cb..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java
+++ /dev/null
@@ -1,256 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import com.google.gson.Gson;
-import com.yahoo.io.IOUtils;
-import com.yahoo.log.LogSetup;
-import com.yahoo.net.HostName;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
-import com.yahoo.vespa.hosted.node.admin.util.Environment;
-import com.yahoo.vespa.hosted.node.admin.util.PathResolver;
-import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
-import io.airlift.airline.Arguments;
-import io.airlift.airline.Cli;
-import io.airlift.airline.Command;
-import io.airlift.airline.Help;
-import io.airlift.airline.Option;
-import io.airlift.airline.ParseArgumentsUnexpectedException;
-import io.airlift.airline.ParseOptionMissingException;
-import org.apache.http.impl.client.HttpClientBuilder;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Pattern;
-
-/**
- * @author freva
- */
-public class Maintainer {
-    private static final Environment environment = new Environment.Builder().pathResolver(new PathResolver()).build();
-    private static final Maintainer maintainer = new Maintainer();
-    private static final CoredumpHandler COREDUMP_HANDLER =
-            new CoredumpHandler(HttpClientBuilder.create().build(), new CoreCollector(maintainer));
-    private static final Gson gson = new Gson();
-
-    private static final String JOB_DELETE_OLD_APP_DATA = "delete-old-app-data";
-    private static final String JOB_ARCHIVE_APP_DATA = "archive-app-data";
-    private static final String JOB_CLEAN_CORE_DUMPS = "clean-core-dumps";
-    private static final String JOB_HANDLE_CORE_DUMPS = "handle-core-dumps";
-
-    private static Optional<String> kernelVersion = Optional.empty();
-
-    @SuppressWarnings("unchecked")
-    public static void main(String[] args) {
-        LogSetup.initVespaLogging(Maintainer.class.getSimpleName().toLowerCase());
-
-        Cli.CliBuilder<Runnable> builder = Cli.<Runnable>builder("maintainer.jar")
-                .withDescription("This tool makes it easy to delete old log files and other node-admin app data.")
-                .withDefaultCommand(Help.class)
-                .withCommands(Help.class,
-                        DeleteOldAppDataArguments.class,
-                        CleanCoreDumpsArguments.class,
-                        ArchiveApplicationData.class,
-                        HandleCoreDumpsForContainer.class);
-
-        Cli<Runnable> gitParser = builder.build();
-        try {
-            gitParser.parse(args).run();
-        } catch (ParseArgumentsUnexpectedException | ParseOptionMissingException e) {
-            System.err.println(e.getMessage());
-            gitParser.parse("help").run();
-        }
-    }
-
-    public static void cleanCoreDumps(PrefixLogger logger) {
-        executeMaintainer(logger, JOB_CLEAN_CORE_DUMPS);
-    }
-
-    public static void deleteOldAppData(PrefixLogger logger) {
-        executeMaintainer(logger, JOB_DELETE_OLD_APP_DATA);
-    }
-
-    public static void archiveAppData(PrefixLogger logger, ContainerName containerName) {
-        executeMaintainer(logger, JOB_ARCHIVE_APP_DATA, containerName.asString());
-    }
-
-    public static void handleCoreDumpsForContainer(PrefixLogger logger, ContainerNodeSpec nodeSpec, Environment environment) {
-        Map<String, Object> attributes = new HashMap<>();
-        attributes.put("hostname", nodeSpec.hostname);
-        attributes.put("parent_hostname", HostName.getLocalhost());
-        attributes.put("region", environment.getRegion());
-        attributes.put("environment", environment.getEnvironment());
-        attributes.put("flavor", nodeSpec.nodeFlavor);
-        try {
-            attributes.put("kernel_version", getKernelVersion());
-        } catch (Throwable ignored) {
-            attributes.put("kernel_version", "unknown");
-        }
-
-        nodeSpec.wantedDockerImage.ifPresent(image -> attributes.put("docker_image", image.asString()));
-        nodeSpec.vespaVersion.ifPresent(version -> attributes.put("vespa_version", version));
-        nodeSpec.owner.ifPresent(owner -> {
-            attributes.put("tenant", owner.tenant);
-            attributes.put("application", owner.application);
-            attributes.put("instance", owner.instance);
-        });
-
-        executeMaintainer(logger, JOB_HANDLE_CORE_DUMPS,
-                "--container", nodeSpec.containerName.asString(),
-                "--attributes", gson.toJson(attributes));
-    }
-
-    private static void executeMaintainer(PrefixLogger logger, String... params) {
-        String[] baseArguments = {"sudo", "/home/y/libexec/vespa/node-admin/maintenance.sh"};
-        String[] args = concatenateArrays(baseArguments, params);
-        ProcessBuilder processBuilder = new ProcessBuilder(args);
-        Map<String, String> env = processBuilder.environment();
-        env.put("VESPA_SERVICE_NAME", "maintainer");
-
-        try {
-            ProcessResult result = maintainer.exec(args);
-
-            if (! result.getOutput().isEmpty()) logger.info(result.getOutput());
-            if (! result.getErrors().isEmpty()) logger.error(result.getErrors());
-        } catch (IOException | InterruptedException e) {
-            logger.warning("Failed to execute command " + Arrays.toString(args), e);
-        }
-    }
-
-    public ProcessResult exec(String... args) throws IOException, InterruptedException {
-        ProcessBuilder processBuilder = new ProcessBuilder(args);
-        Process process = processBuilder.start();
-        String output = IOUtils.readAll(new InputStreamReader(process.getInputStream()));
-        String errors = IOUtils.readAll(new InputStreamReader(process.getErrorStream()));
-
-        return new ProcessResult(process.waitFor(), output, errors);
-    }
-
-    private static String[] concatenateArrays(String[] ar1, String... ar2) {
-        String[] concatenated = new String[ar1.length + ar2.length];
-        System.arraycopy(ar1, 0, concatenated, 0, ar1.length);
-        System.arraycopy(ar2, 0, concatenated, ar1.length, ar2.length);
-        return concatenated;
-    }
-
-    @Command(name = JOB_DELETE_OLD_APP_DATA, description = "Deletes old app data")
-    public static class DeleteOldAppDataArguments implements Runnable {
-        @Override
-        public void run() {
-            String path = environment.getPathResolver().getApplicationStoragePathForNodeAdmin().toString();
-            String regex = "^" + Pattern.quote(Environment.APPLICATION_STORAGE_CLEANUP_PATH_PREFIX);
-
-            DeleteOldAppData.deleteDirectories(path, Duration.ofDays(7).getSeconds(), regex);
-        }
-    }
-
-    @Command(name = JOB_CLEAN_CORE_DUMPS, description = "Clean core dumps")
-    public static class CleanCoreDumpsArguments implements Runnable {
-        @Override
-        public void run() {
-            Path doneCoredumps = environment.pathInNodeAdminToDoneCoredumps();
-
-            if (doneCoredumps.toFile().exists()) {
-                COREDUMP_HANDLER.removeOldCoredumps(doneCoredumps);
-            }
-        }
-    }
-
-    @Command(name = JOB_ARCHIVE_APP_DATA, description = "Move container's container-storage to cleanup")
-    public static class ArchiveApplicationData implements Runnable {
-        @Arguments(description = "Name of container to archive (required)")
-        public String container;
-
-        @Override
-        public void run() {
-            if (container == null) {
-                throw new IllegalArgumentException("<container> is required");
-            }
-            // Note that ContainerName verifies the name, so it cannot
-            // contain / or be equal to "." or "..".
-            ContainerName containerName = new ContainerName(container);
-
-            Logger logger = Logger.getLogger(ArchiveApplicationData.class.getName());
-            File yVarDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/var").toFile();
-            if (yVarDir.exists()) {
-                logger.info("Recursively deleting " + yVarDir);
-                try {
-                    DeleteOldAppData.recursiveDelete(yVarDir);
-                } catch (IOException e) {
-                    throw new RuntimeException("Failed to delete " + yVarDir, e);
-                }
-            }
-
-            Path from = environment.pathInNodeAdminFromPathInNode(containerName, "/");
-            if (!Files.exists(from)) {
-                logger.info("The container storage at " + from + " doesn't exist");
-                return;
-            }
-
-            Path to = environment.pathInNodeAdminToNodeCleanup(containerName);
-            logger.info("Moving container storage from " + from + " to " + to);
-            try {
-                Files.move(from, to);
-            } catch (IOException e) {
-                throw new RuntimeException("Failed to move " + from + " to " + to, e);
-            }
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    @Command(name = JOB_HANDLE_CORE_DUMPS, description = "Finds container's coredumps, collects metadata and reports them")
-    public static class HandleCoreDumpsForContainer implements Runnable {
-        @Option(name = "--container", description = "Name of the container")
-        public String container;
-
-        @Option(name = "--attributes", description = "Comma separated key=value pairs")
-        public String attributes;
-
-        @Override
-        public void run() {
-            Logger logger = Logger.getLogger(HandleCoreDumpsForContainer.class.getName());
-
-            if (container == null) {
-                throw new IllegalArgumentException("<container> is required");
-            }
-
-            try {
-                Map<String, Object> attributesMap = (Map<String, Object>) gson.fromJson(attributes, Map.class);
-
-                Path path = environment.pathInNodeAdminFromPathInNode(new ContainerName(container), "/home/y/var/crash");
-                Path doneCoredumps = environment.pathInNodeAdminToDoneCoredumps();
-
-                COREDUMP_HANDLER.removeJavaCoredumps(path);
-                COREDUMP_HANDLER.processAndReportCoredumps(path, doneCoredumps, attributesMap);
-            } catch (Throwable e) {
-                logger.log(Level.WARNING, "Could not process coredumps", e);
-            }
-        }
-    }
-
-
-
-    public static String getKernelVersion() throws IOException, InterruptedException {
-        if (! kernelVersion.isPresent()) {
-            ProcessResult result = maintainer.exec("uname", "-r");
-            if (result.isSuccess()) {
-                kernelVersion = Optional.of(result.getOutput().trim());
-            } else {
-                throw new RuntimeException("Failed to get kernel version\n" + result);
-            }
-        }
-
-        return kernelVersion.get();
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
index ed6808e1eac..519f2c49374 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
@@ -25,7 +25,6 @@ import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class DockerOperationsImplTest {
@@ -35,64 +34,37 @@ public class DockerOperationsImplTest {
             new MetricReceiverWrapper(MetricReceiver.nullImplementation));
 
     @Test
-    public void absenceOfNodeProgramIsSuccess() throws Exception {
+    public void processResultFromNodeProgramWhenSuccess() throws Exception {
         final ContainerName containerName = new ContainerName("container-name");
+        final ProcessResult actualResult = new ProcessResult(0, "output", "errors");
         final String programPath = "/bin/command";
+        final String[] command = new String[] {programPath, "arg"};
 
-        when(docker.executeInContainer(any(), anyVararg())).thenReturn(new ProcessResult(3, "output", "errors"));
-
-        Optional<ProcessResult> result = dockerOperations.executeOptionalProgramInContainer(
-                containerName,
-                programPath,
-                "arg1",
-                "arg2");
+        when(docker.executeInContainerAsRoot(any(), anyVararg()))
+                .thenReturn(actualResult); // output from node program
 
-        String[] nodeProgramExistsCommand = dockerOperations.programExistsCommand(programPath);
-        assertThat(nodeProgramExistsCommand.length, is(4));
+        ProcessResult result = dockerOperations.executeCommandInContainer(containerName, command);
 
-        verify(docker, times(1)).executeInContainer(
+        final InOrder inOrder = inOrder(docker);
+        inOrder.verify(docker, times(1)).executeInContainerAsRoot(
                 eq(containerName),
-                // Mockito fails if we put the array here instead...
-                eq(nodeProgramExistsCommand[0]),
-                eq(nodeProgramExistsCommand[1]),
-                eq(nodeProgramExistsCommand[2]),
-                eq(nodeProgramExistsCommand[3]));
-        assertThat(result.isPresent(), is(false));
+                eq(command[0]),
+                eq(command[1]));
+
+        assertThat(result, is(actualResult));
     }
 
-    @Test
-    public void processResultFromNodeProgramWhenPresent() throws Exception {
+    @Test(expected=RuntimeException.class)
+    public void processResultFromNodeProgramWhenNonZeroExitCode() throws Exception {
         final ContainerName containerName = new ContainerName("container-name");
         final ProcessResult actualResult = new ProcessResult(3, "output", "errors");
         final String programPath = "/bin/command";
         final String[] command = new String[] {programPath, "arg"};
 
-        when(docker.executeInContainer(any(), anyVararg()))
-                .thenReturn(new ProcessResult(0, "", "")) // node program exists
+        when(docker.executeInContainerAsRoot(any(), anyVararg()))
                 .thenReturn(actualResult); // output from node program
 
-        Optional<ProcessResult> result = dockerOperations.executeOptionalProgramInContainer(
-                containerName,
-                command);
-
-        String[] nodeProgramExistsCommand = dockerOperations.programExistsCommand(programPath);
-        assertThat(nodeProgramExistsCommand.length, is(4));
-
-        final InOrder inOrder = inOrder(docker);
-        inOrder.verify(docker, times(1)).executeInContainer(
-                eq(containerName),
-                // Mockito fails if we put the array here instead...
-                eq(nodeProgramExistsCommand[0]),
-                eq(nodeProgramExistsCommand[1]),
-                eq(nodeProgramExistsCommand[2]),
-                eq(nodeProgramExistsCommand[3]));
-        inOrder.verify(docker, times(1)).executeInContainer(
-                eq(containerName),
-                eq(command[0]),
-                eq(command[1]));
-
-        assertThat(result.isPresent(), is(true));
-        assertThat(result.get(), is(actualResult));
+        dockerOperations.executeCommandInContainer(containerName, command);
     }
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
index 9ad61baf585..06f8079172d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
@@ -36,16 +36,14 @@ public class DockerFailTest {
             CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier();
             callOrderVerifier.assertInOrder(
                     "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: hostName, ContainerName { name=container }",
-                    "executeInContainer with ContainerName { name=container }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                    "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                    "executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
 
             dockerTester.deleteContainer(containerNodeSpec.containerName);
 
             callOrderVerifier.assertInOrder(
                     "deleteContainer with ContainerName { name=container }",
                     "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: hostName, ContainerName { name=container }",
-                    "executeInContainer with ContainerName { name=container }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                    "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                    "executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index fa4b235066b..2be286eca7b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -156,6 +156,14 @@ public class DockerMock implements Docker {
         return new ProcessResult(0, null, "");
     }
 
+    @Override
+    public ProcessResult executeInContainerAsRoot(ContainerName containerName, String... args) {
+        synchronized (monitor) {
+            callOrderVerifier.add("executeInContainerAsRoot with " + containerName + ", args: " + Arrays.toString(args));
+        }
+        return new ProcessResult(0, null, "");
+    }
+
 
     public static class StartContainerCommandMock implements CreateContainerCommand {
         @Override
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 0912ccff814..aef03327645 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -49,10 +49,10 @@ public class DockerTester implements AutoCloseable {
         Environment environment = new Environment.Builder().inetAddressResolver(inetAddressResolver).build();
 
         callOrderVerifier = new CallOrderVerifier();
-        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(environment, callOrderVerifier);
         orchestratorMock = new OrchestratorMock(callOrderVerifier);
         nodeRepositoryMock = new NodeRepoMock(callOrderVerifier);
         dockerMock = new DockerMock(callOrderVerifier);
+        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerMock, environment, callOrderVerifier);
 
 
         MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index 2f88efd8620..582df7d22f6 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -38,19 +38,16 @@ public class MultiDockerTest {
             CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier();
             callOrderVerifier.assertInOrder(
                     "createContainerCommand with DockerImage { imageId=image1 }, HostName: host1, ContainerName { name=container1 }",
-                    "executeInContainer with ContainerName { name=container1 }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                    "executeInContainer with ContainerName { name=container1 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]",
+                    "executeInContainerAsRoot with ContainerName { name=container1 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]",
 
                     "createContainerCommand with DockerImage { imageId=image2 }, HostName: host2, ContainerName { name=container2 }",
-                    "executeInContainer with ContainerName { name=container2 }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                    "executeInContainer with ContainerName { name=container2 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]",
+                    "executeInContainerAsRoot with ContainerName { name=container2 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]",
 
                     "stopContainer with ContainerName { name=container2 }",
                     "deleteContainer with ContainerName { name=container2 }",
 
                     "createContainerCommand with DockerImage { imageId=image1 }, HostName: host3, ContainerName { name=container3 }",
-                    "executeInContainer with ContainerName { name=container3 }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                    "executeInContainer with ContainerName { name=container3 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                    "executeInContainerAsRoot with ContainerName { name=container3 }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
 
             callOrderVerifier.assertInOrderWithAssertMessage("Maintainer did not receive call to delete application storage",
                                                              "deleteContainer with ContainerName { name=container2 }",
@@ -85,8 +82,7 @@ public class MultiDockerTest {
 
         tester.getCallOrderVerifier().assertInOrder(
                 "createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName,
-                "executeInContainer with " + containerName + ", args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                "executeInContainer with " + containerName + ", args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                "executeInContainerAsRoot with " + containerName + ", args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
 
         return containerNodeSpec;
     }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
index 53d3422e8d5..222d46658eb 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
@@ -37,8 +37,7 @@ public class NodeStateTest {
 
         tester.getCallOrderVerifier().assertInOrder(
                 "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1, ContainerName { name=container }",
-                "executeInContainer with ContainerName { name=container }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                "executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
     }
 
 
@@ -58,8 +57,7 @@ public class NodeStateTest {
             }
 
             dockerTester.getCallOrderVerifier()
-                        .assertInOrder("executeInContainer with ContainerName { name=container }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                                       "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", stop]",
+                        .assertInOrder("executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", stop]",
                                        "stopContainer with ContainerName { name=container }",
                                        "deleteContainer with ContainerName { name=container }");
         }
@@ -95,8 +93,7 @@ public class NodeStateTest {
             callOrderVerifier.assertInOrderWithAssertMessage("Node not started again after being put to active state",
                                                              "deleteContainer with ContainerName { name=container }",
                                                              "createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1, ContainerName { name=container }",
-                                                             "executeInContainer with ContainerName { name=container }, args: [/usr/bin/env, test, -x, " + DockerOperationsImpl.NODE_PROGRAM + "]",
-                                                             "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
+                                                             "executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", resume]");
         }
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index adeac210325..9c2adca3bb7 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -40,7 +40,7 @@ public class RestartTest {
             dockerTester.updateContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration));
 
             callOrderVerifier.assertInOrder("Suspend for host1",
-                                            "executeInContainer with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", restart]");
+                                            "executeInContainerAsRoot with ContainerName { name=container }, args: [" + DockerOperationsImpl.NODE_PROGRAM + ", restart]");
         }
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
index a2b3d655b6c..6107ca74752 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
@@ -1,7 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
+import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.Docker;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
@@ -15,8 +18,8 @@ import java.util.Map;
 public class StorageMaintainerMock extends StorageMaintainer {
     private final CallOrderVerifier callOrderVerifier;
 
-    public StorageMaintainerMock(Environment environment, CallOrderVerifier callOrderVerifier) {
-        super(environment);
+    public StorageMaintainerMock(Docker docker, Environment environment, CallOrderVerifier callOrderVerifier) {
+        super(docker, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment);
         this.callOrderVerifier = callOrderVerifier;
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
index 886536a5555..25d267eef28 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -1,12 +1,15 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
-import com.yahoo.vespa.hosted.node.maintenance.DeleteOldAppDataTest;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 
 import static org.junit.Assert.*;
 
@@ -21,12 +24,17 @@ public class StorageMaintainerTest {
     @Test
     public void testDiskUsed() throws IOException, InterruptedException {
         int writeSize = 10000;
-        DeleteOldAppDataTest.writeNBytesToFile(folder.newFile(), writeSize);
+        writeNBytesToFile(folder.newFile(), writeSize);
 
         Environment environment = new Environment.Builder().build();
-        StorageMaintainer storageMaintainer = new StorageMaintainer(environment);
-        long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot());
+        StorageMaintainer storageMaintainer = new StorageMaintainer(null,
+                new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment);
+        long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot().toPath());
         if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4)
             fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close.");
     }
+
+    private static void writeNBytesToFile(File file, int nBytes) throws IOException {
+        Files.write(file.toPath(), new byte[nBytes]);
+    }
 }
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollectorTest.java
deleted file mode 100644
index 1f9b480b0c1..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoreCollectorTest.java
+++ /dev/null
@@ -1,227 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * @author freva
- */
-public class CoreCollectorTest {
-    private final Maintainer maintainer = mock(Maintainer.class);
-    private final CoreCollector coreCollector = new CoreCollector(maintainer);
-
-    private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234");
-    private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program");
-    private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]",
-            "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.",
-            "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t    printf(argv[3]);",
-            "#0  0x00000000004004d8 in main (argv=0x1) at main.c:4");
-
-    @Rule
-    public TemporaryFolder folder= new TemporaryFolder();
-
-    private void mockExec(String[] cmd, String output) throws IOException, InterruptedException {
-        mockExec(cmd, output, "");
-    }
-
-    private void mockExec(String[] cmd, String output, String error) throws IOException, InterruptedException {
-        when(maintainer.exec(cmd)).thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
-    }
-
-    @Test
-    public void extractsBinaryPathTest() throws IOException, InterruptedException {
-        final String[] cmd = {"file", TEST_CORE_PATH.toString()};
-
-        mockExec(cmd,
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
-                        "'/usr/bin/program'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
-
-        mockExec(cmd,
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
-                        "'/usr/bin/program --foo --bar baz'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
-
-        mockExec(cmd,
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
-                        "'/usr/bin//program'");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
-
-        mockExec(cmd,
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " +
-                        "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " +
-                        "execfn: '/usr/bin/program', platform: 'x86_64");
-        assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH));
-
-
-        Path fallbackResponse = Paths.get("/response/from/fallback");
-        mockExec(new String[]{"sh", "-c", "\"/home/y/bin64/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by'\""},
-                "Core was generated by `/response/from/fallback'.");
-        mockExec(cmd,
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style");
-        assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH));
-
-        mockExec(cmd, "", "Error code 1234");
-        assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH));
-    }
-
-    @Test
-    public void extractsBinaryPathUsingGdbTest() throws IOException, InterruptedException {
-        final String[] cmd = new String[]{"sh", "-c",
-                "\"/home/y/bin64/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by'\""};
-
-        mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'.");
-        assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(TEST_CORE_PATH));
-
-        mockExec(cmd, "", "Error 123");
-        try {
-            coreCollector.readBinPathFallback(TEST_CORE_PATH);
-            fail("Expected not to be able to get bin path");
-        } catch (RuntimeException e) {
-            assertEquals(e.getMessage(), "Failed to extract binary path from ProcessResult { exitStatus=1 output= errors=Error 123 }");
-        }
-    }
-
-    @Test
-    public void extractsBacktraceUsingGdb() throws IOException, InterruptedException {
-        mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
-                String.join("\n", GDB_BACKTRACE));
-        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false));
-
-        mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
-                "", "Failure");
-        try {
-            coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false);
-            fail("Expected not to be able to read backtrace");
-        } catch (RuntimeException e) {
-            assertEquals("Failed to read backtrace ProcessResult { exitStatus=1 output= errors=Failure }", e.getMessage());
-        }
-    }
-
-    @Test
-    public void extractsBacktraceFromAllThreadsUsingGdb() throws IOException, InterruptedException {
-        mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "thread apply all bt", "-batch",
-                        "/usr/bin/program", "/tmp/core.1234"},
-                String.join("\n", GDB_BACKTRACE));
-        assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, true));
-    }
-
-    @Test
-    public void collectsDataTest() throws IOException, InterruptedException {
-        mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
-                        "'/usr/bin/program'");
-        mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
-                String.join("\n", GDB_BACKTRACE));
-        mockExec(new String[]{"/home/y/bin64/gdb", "-n", "-ex", "thread apply all bt", "-batch",
-                        "/usr/bin/program", "/tmp/core.1234"},
-                String.join("\n", GDB_BACKTRACE));
-
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE));
-        expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE));
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
-    }
-
-    @Test
-    public void collectsPartialIfUnableToDetermineDumpingProgramTest() {
-        Map<String, Object> expectedData = new HashMap<>();
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
-    }
-
-    @Test
-    public void collectsPartialIfBacktraceFailsTest() throws IOException, InterruptedException {
-        mockExec(new String[]{"file", TEST_CORE_PATH.toString()},
-                "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
-                        "'/usr/bin/program'");
-        mockExec(new String[]{"/home/y/bin64/gdb -n -ex bt -batch /usr/bin/program /tmp/core.1234"},
-                "", "Failure");
-
-        Map<String, Object> expectedData = new HashMap<>();
-        expectedData.put("bin_path", TEST_BIN_PATH.toString());
-        assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH));
-    }
-
-    @Test
-    public void parseTotalMemoryTestTest() throws IOException {
-        String memInfo = "MemTotal:   100000000 kB\nMemUsed:   1000000 kB\n";
-        assertEquals(100000000, coreCollector.parseTotalMemorySize(memInfo));
-
-        String badMemInfo = "This string has no memTotal value";
-        try {
-            coreCollector.parseTotalMemorySize(badMemInfo);
-            fail("Expected to fail on parsing");
-        } catch (RuntimeException e) {
-            assertEquals("Could not parse meminfo: " + badMemInfo, e.getMessage());
-        }
-    }
-
-    @Test
-    public void testDeleteUncompressedFiles() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        final String coreDumpFilename = "core.dump";
-
-        Path coredumpPath = folder.newFolder("crash").toPath()
-                .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)
-                .resolve(documentId);
-        coredumpPath.toFile().mkdirs();
-        coredumpPath.resolve(coreDumpFilename).toFile().createNewFile();
-
-        Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList(
-                coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME),
-                coredumpPath.resolve(coreDumpFilename + ".lz4")));
-        expectedContentsOfCoredump.forEach(path -> {
-            try {
-                path.toFile().createNewFile();
-            } catch (IOException ignored) { ignored.printStackTrace();}
-        });
-        coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename));
-
-        assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet()));
-    }
-
-    @Test
-    public void testDeleteUncompressedFilesWithoutLz4() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        final String coreDumpFilename = "core.dump";
-
-        Path coredumpPath = folder.newFolder("crash").toPath()
-                .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)
-                .resolve(documentId);
-        coredumpPath.toFile().mkdirs();
-
-        Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList(
-                coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME),
-                coredumpPath.resolve(coreDumpFilename)));
-        expectedContentsOfCoredump.forEach(path -> {
-            try {
-                path.toFile().createNewFile();
-            } catch (IOException ignored) { ignored.printStackTrace();}
-        });
-        coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename));
-
-        assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet()));
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandlerTest.java
deleted file mode 100644
index 1dfc6abeb6b..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/CoredumpHandlerTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpVersion;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.DefaultHttpResponseFactory;
-import org.apache.http.message.BasicStatusLine;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.ArgumentCaptor;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * @author freva
- */
-public class CoredumpHandlerTest {
-    private final HttpClient httpClient = mock(HttpClient.class);
-    private final CoreCollector coreCollector = mock(CoreCollector.class);
-    private static final Map<String, Object> attributes = new LinkedHashMap<>();
-    private static final Map<String, Object> metadata = new LinkedHashMap<>();
-    private static final String expectedMetadataFileContents = "{\"fields\":{" +
-            "\"bin_path\":\"/bin/bash\"," +
-            "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," +
-            "\"hostname\":\"host123.yahoo.com\"," +
-            "\"vespa_version\":\"6.48.4\"," +
-            "\"kernel_version\":\"2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64\"," +
-            "\"docker_image\":\"vespa/ci:6.48.4\"}}";
-
-    static {
-        attributes.put("hostname", "host123.yahoo.com");
-        attributes.put("vespa_version", "6.48.4");
-        attributes.put("kernel_version", "2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64");
-        attributes.put("docker_image", "vespa/ci:6.48.4");
-
-        metadata.put("bin_path", "/bin/bash");
-        metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something"));
-    }
-
-    private final CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector);
-
-
-    @Rule
-    public TemporaryFolder folder= new TemporaryFolder();
-
-    @Test
-    public void ignoresIncompleteCoredumps() throws IOException {
-        Path coredumpPath = createCoredump(".core.dump");
-        Path crashPath = coredumpPath.getParent();
-        Path processingPath = coredumpHandler.processCoredumps(crashPath, attributes);
-
-        // The 'processing' directory should be empty
-        assertFolderContents(processingPath);
-
-        // The 'crash' directory should have 'processing' and the incomplete core dump in it
-        assertFolderContents(crashPath, processingPath.getFileName().toString(), coredumpPath.getFileName().toString());
-    }
-
-    @Test
-    public void startProcessingTest() throws IOException {
-        Path coredumpPath = createCoredump("core.dump");
-        Path crashPath = coredumpPath.getParent();
-        Path processingPath = crashPath.resolve("processing_dir");
-        coredumpHandler.startProcessing(coredumpPath, crashPath.resolve("processing_dir"));
-
-        // Contents of 'crash' should be only the 'processing' directory
-        assertFolderContents(crashPath, processingPath.getFileName().toString());
-
-        // The 'processing' directory should have 1 directory inside for the core.dump we just created
-        List<Path> processedCoredumps = Files.list(processingPath).collect(Collectors.toList());
-        assertEquals(processedCoredumps.size(), 1);
-
-        // Inside the coredump directory, there should be 1 file: core.dump
-        assertFolderContents(processedCoredumps.get(0), coredumpPath.getFileName().toString());
-    }
-
-    @Test
-    public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException {
-        when(coreCollector.collect(any())).thenReturn(metadata);
-        Path coredumpPath = createCoredump("core.dump");
-        Path crashPath = coredumpPath.getParent();
-        Path processingPath = coredumpHandler.processCoredumps(crashPath, attributes);
-
-        // Inside 'processing' directory, there should be a new directory containing 'metadata.json' file
-        List<Path> processedCoredumps = Files.list(processingPath).collect(Collectors.toList());
-        String metadataFileContents = new String(Files.readAllBytes(
-                processedCoredumps.get(0).resolve(CoredumpHandler.METADATA_FILE_NAME)));
-        assertEquals(expectedMetadataFileContents, metadataFileContents);
-    }
-
-    @Test
-    public void reportSuccessCoredumpTest() throws IOException, URISyntaxException, InterruptedException {
-        final String documentId = "UIDD-ABCD-EFGH";
-        Path coredumpPath = createProcessedCoredump(documentId);
-
-        setNextHttpResponse(200, Optional.empty());
-        coredumpHandler.report(coredumpPath.getParent());
-        validateNextHttpPost(documentId, expectedMetadataFileContents);
-    }
-
-    @Test
-    public void reportFailCoredumpTest() throws IOException, URISyntaxException {
-        final String documentId = "UIDD-ABCD-EFGH";
-
-        Path metadataPath = createProcessedCoredump(documentId);
-        Path crashPath = metadataPath.getParent().getParent().getParent();
-        Path donePath = folder.newFolder("done").toPath();
-
-        setNextHttpResponse(500, Optional.of("Internal server error"));
-        coredumpHandler.reportCoredumps(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME), donePath);
-        validateNextHttpPost(documentId, expectedMetadataFileContents);
-
-        // The coredump should not have been moved out of 'processing' and into 'done' as the report failed
-        assertFolderContents(donePath);
-        assertFolderContents(metadataPath.getParent(), CoredumpHandler.METADATA_FILE_NAME);
-    }
-
-    @Test
-    public void finishProcessingTest() throws IOException {
-        final String documentId = "UIDD-ABCD-EFGH";
-
-        Path coredumpPath = createProcessedCoredump(documentId);
-        Path crashPath = coredumpPath.getParent().getParent().getParent();
-        Path donePath = folder.newFolder("done").toPath();
-
-        coredumpHandler.finishProcessing(coredumpPath.getParent(), donePath);
-
-        // The coredump should've been moved out of 'processing' and into 'done'
-        assertFolderContents(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME));
-        assertFolderContents(donePath.resolve(documentId), CoredumpHandler.METADATA_FILE_NAME);
-    }
-
-
-    private static void assertFolderContents(Path pathToFolder, String... filenames) throws IOException {
-        Set<Path> expectedContentsOfFolder = Arrays.stream(filenames)
-                .map(pathToFolder::resolve)
-                .collect(Collectors.toSet());
-        Set<Path> actualContentsOfFolder = Files.list(pathToFolder).collect(Collectors.toSet());
-        assertEquals(expectedContentsOfFolder, actualContentsOfFolder);
-    }
-
-    private Path createCoredump(String coredumpName) throws IOException {
-        Path crashPath = folder.newFolder("crash").toPath();
-        Path coredumpPath = crashPath.resolve(coredumpName);
-        coredumpPath.toFile().createNewFile();
-        return coredumpPath;
-    }
-
-    private Path createProcessedCoredump(String documentId) throws IOException {
-        Path crashPath = folder.newFolder("crash").toPath();
-        Path coredumpPath = crashPath
-                .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)
-                .resolve(documentId)
-                .resolve(CoredumpHandler.METADATA_FILE_NAME);
-        coredumpPath.getParent().toFile().mkdirs();
-        return Files.write(coredumpPath, expectedMetadataFileContents.getBytes());
-    }
-
-    private void setNextHttpResponse(int code, Optional<String> message) throws IOException {
-        DefaultHttpResponseFactory responseFactory = new DefaultHttpResponseFactory();
-        HttpResponse httpResponse = responseFactory.newHttpResponse(
-                new BasicStatusLine(HttpVersion.HTTP_1_1, code, null), null);
-        if (message.isPresent()) httpResponse.setEntity(new StringEntity(message.get()));
-
-        when(httpClient.execute(any())).thenReturn(httpResponse);
-    }
-
-    private void validateNextHttpPost(String documentId, String expectedBody) throws IOException, URISyntaxException {
-        ArgumentCaptor<HttpPost> capturedPost = ArgumentCaptor.forClass(HttpPost.class);
-        verify(httpClient).execute(capturedPost.capture());
-
-        URI expectedURI = new URI(CoredumpHandler.FEED_ENDPOINT + "/" + documentId);
-        assertEquals(expectedURI, capturedPost.getValue().getURI());
-        assertEquals("application/json", capturedPost.getValue().getHeaders(HttpHeaders.CONTENT_TYPE)[0].getValue());
-        assertEquals(expectedBody,
-                new BufferedReader(new InputStreamReader(capturedPost.getValue().getEntity().getContent())).readLine());
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java
deleted file mode 100644
index 41a42095b1d..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.maintenance;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.Arrays;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author freva
- */
-public class DeleteOldAppDataTest {
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder();
-
-    @Before
-    public void initFiles() throws IOException {
-        for (int i=0; i<10; i++) {
-            File temp = folder.newFile("test_" + i + ".json");
-            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis());
-        }
-
-        for (int i=0; i<7; i++) {
-            File temp = folder.newFile("test_" + i + "_file.test");
-            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis());
-        }
-
-        for (int i=0; i<5; i++) {
-            File temp = folder.newFile(i + "-abc" + ".json");
-            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis());
-        }
-
-        File temp = folder.newFile("week_old_file.json");
-        temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis());
-    }
-
-    @Test
-    public void testDeleteAll() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, null, false);
-
-        assertThat(folder.getRoot().listFiles().length, is(0));
-    }
-
-    @Test
-    public void testDeletePrefix() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, "^test_", false);
-
-        assertThat(folder.getRoot().listFiles().length, is(6)); // 5 abc files + 1 week_old_file
-    }
-
-    @Test
-    public void testDeleteSuffix() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, ".json$", false);
-
-        assertThat(folder.getRoot().listFiles().length, is(7));
-    }
-
-    @Test
-    public void testDeletePrefixAndSuffix() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, "^test_.*\\.json$", false);
-
-        assertThat(folder.getRoot().listFiles().length, is(13)); // 5 abc files + 7 test_*_file.test files + week_old_file
-    }
-
-    @Test
-    public void testDeleteOld() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 600, null, false);
-
-        assertThat(folder.getRoot().listFiles().length, is(13)); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file
-    }
-
-    @Test
-    public void testDeleteWithAllParameters() {
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 200, "^test_.*\\.json$", false);
-
-        assertThat(folder.getRoot().listFiles().length, is(15)); // All 23 - 8 (from test_*_.json)
-    }
-
-    @Test
-    public void testDeleteWithSubDirectoriesNoRecursive() throws IOException {
-        initSubDirectories();
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, "^test_.*\\.json$", false);
-
-        // 6 test_*.json from test_folder1/
-        // + 9 test_*.json and 4 abc_*.json from test_folder2/
-        // + 13 test_*.json from test_folder2/subSubFolder2/
-        // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
-        // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves
-        assertThat(getNumberOfFilesAndDirectoriesIn(folder.getRoot()), is(48));
-    }
-
-    @Test
-    public void testDeleteWithSubDirectoriesRecursive() throws IOException {
-        initSubDirectories();
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, "^test_.*\\.json$", true);
-
-        // 4 abc_*.json from test_folder2/
-        // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
-        // + test_folder2/ itself
-        assertThat(getNumberOfFilesAndDirectoriesIn(folder.getRoot()), is(18));
-    }
-
-    @Test
-    public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException {
-        initSubDirectories();
-
-        DeleteOldAppData.deleteFiles(folder.getRoot().getAbsolutePath(), 0, "^test_", false);
-
-        assertThat(folder.getRoot().listFiles().length, is(8)); // 5 abc files + 1 week_old_file + 2 directories
-    }
-
-    @Test
-    public void testGetContentsOfNonExistingDirectory() throws IOException {
-        assertArrayEquals(new File[0], DeleteOldAppData.getContentsOfDirectory("/some/made/up/dir/"));
-    }
-
-    @Test(expected=IllegalArgumentException.class)
-    public void testDeleteFilesExceptNMostRecentWithNegativeN() {
-        DeleteOldAppData.deleteFilesExceptNMostRecent(folder.getRoot().getAbsolutePath(), -5);
-    }
-
-    @Test
-    public void testDeleteFilesExceptFiveMostRecent() {
-        DeleteOldAppData.deleteFilesExceptNMostRecent(folder.getRoot().getAbsolutePath(), 5);
-
-        assertThat(folder.getRoot().listFiles().length, is(5));
-
-        String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"};
-        String[] remainingFiles = folder.getRoot().list();
-        Arrays.sort(remainingFiles);
-
-        assertArrayEquals(oldestFiles, remainingFiles);
-    }
-
-    @Test
-    public void testDeleteFilesExceptNMostRecentWithLargeN() {
-        String[] filesPreDelete = folder.getRoot().list();
-
-        DeleteOldAppData.deleteFilesExceptNMostRecent(folder.getRoot().getAbsolutePath(), 50);
-
-        assertArrayEquals(filesPreDelete, folder.getRoot().list());
-    }
-
-    @Test
-    public void testDeleteFilesLargerThan10B() throws IOException {
-        initSubDirectories();
-
-        File temp1 = new File(folder.getRoot(), "small_file");
-        writeNBytesToFile(temp1, 50);
-
-        File temp2 = new File(folder.getRoot(), "some_file");
-        writeNBytesToFile(temp2, 20);
-
-        File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file");
-        writeNBytesToFile(temp3, 75);
-
-        DeleteOldAppData.deleteFilesLargerThan(folder.getRoot(), 10);
-
-        assertThat(getNumberOfFilesAndDirectoriesIn(folder.getRoot()), is(58));
-        assertThat(temp1.exists() || temp2.exists() || temp3.exists(), is(false));
-    }
-
-    @Test
-    public void testDeleteDirectories() throws IOException {
-        initSubDirectories();
-
-        DeleteOldAppData.deleteDirectories(folder.getRoot().getAbsolutePath(), 0, ".*folder2");
-
-        //23 files in root
-        // + 6 in test_folder1 + test_folder1 itself
-        assertThat(getNumberOfFilesAndDirectoriesIn(folder.getRoot()), is(30));
-    }
-
-    @Test
-    public void testDeleteDirectoriesBasedOnAge() throws IOException {
-        initSubDirectories();
-
-        DeleteOldAppData.deleteDirectories(folder.getRoot().getAbsolutePath(), 50, ".*folder.*");
-
-        //23 files in root
-        // + 13 in test_folder2
-        // + 13 in subSubFolder2
-        // + test_folder2 + subSubFolder2 itself
-        assertThat(getNumberOfFilesAndDirectoriesIn(folder.getRoot()), is(51));
-    }
-
-    @Test
-    public void testRecursivelyDeleteDirectory() throws IOException {
-        initSubDirectories();
-        DeleteOldAppData.recursiveDelete(folder.getRoot());
-        assertTrue(!folder.getRoot().exists());
-    }
-
-    @Test
-    public void testRecursivelyDeleteRegularFile() throws IOException {
-        File file = folder.newFile();
-        assertTrue(file.exists());
-        assertTrue(file.isFile());
-        DeleteOldAppData.recursiveDelete(file);
-        assertTrue(!file.exists());
-    }
-
-    @Test
-    public void testRecursivelyDeleteNonExistingFile() throws IOException {
-        File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile();
-        assertTrue(!file.exists());
-        DeleteOldAppData.recursiveDelete(file);
-        assertTrue(!file.exists());
-    }
-
-    @Test
-    public void testInitSubDirectories() throws IOException {
-        initSubDirectories();
-        assertTrue(folder.getRoot().exists());
-        assertTrue(folder.getRoot().isDirectory());
-
-        Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1");
-        assertTrue(test_folder1.toFile().exists());
-        assertTrue(test_folder1.toFile().isDirectory());
-
-        Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2");
-        assertTrue(test_folder2.toFile().exists());
-        assertTrue(test_folder2.toFile().isDirectory());
-
-        Path subSubFolder2 = test_folder2.resolve("subSubFolder2");
-        assertTrue(subSubFolder2.toFile().exists());
-        assertTrue(subSubFolder2.toFile().isDirectory());
-    }
-
-    private void initSubDirectories() throws IOException {
-        File subFolder1 = folder.newFolder("test_folder1");
-        File subFolder2 = folder.newFolder("test_folder2");
-        File subSubFolder2 = folder.newFolder("test_folder2/subSubFolder2");
-
-
-        for (int j=0; j<6; j++) {
-            File temp = File.createTempFile("test_", ".json", subFolder1);
-            temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis());
-        }
-
-        for (int j=0; j<9; j++) {
-            File.createTempFile("test_", ".json", subFolder2);
-        }
-
-        for (int j=0; j<4; j++) {
-            File.createTempFile("abc_", ".txt", subFolder2);
-        }
-
-        for (int j=0; j<13; j++) {
-            File temp = File.createTempFile("test_", ".json", subSubFolder2);
-            temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis());
-        }
-
-        //Must be after all the files have been created
-        subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis());
-        subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis());
-        subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis());
-    }
-
-    private static int getNumberOfFilesAndDirectoriesIn(File folder) {
-        int total = 0;
-        for (File file : folder.listFiles()) {
-            if (file.isDirectory()) {
-                total += getNumberOfFilesAndDirectoriesIn(file);
-            }
-            total++;
-        }
-
-        return total;
-    }
-
-    public static void writeNBytesToFile(File file, int nBytes) throws IOException {
-        Files.write(file.toPath(), new byte[nBytes]);
-    }
-}