Merge pull request #7277 from vespa-engine/freva/use-node-agent-context-in-storage-maintainer

NodeAdmin: Use NodeAgentContext in StorageMaintainer

11 files changed, 212 insertions, 422 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
index 416b844b8d0..b860863c08d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
@@ -11,9 +11,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl;
 
 import java.net.URI;
 import java.nio.file.Path;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -26,10 +23,6 @@ import java.util.Optional;
  * @author hmusum
  */
 public class Environment {
-    private static final DateTimeFormatter filenameFormatter = DateTimeFormatter
-            .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").withZone(ZoneOffset.UTC);
-    public static final String APPLICATION_STORAGE_CLEANUP_PATH_PREFIX = "cleanup_";
-
     private final ConfigServerInfo configServerInfo;
     private final String environment;
     private final String region;
@@ -117,22 +110,6 @@ public class Environment {
     }
 
     /**
-     * Absolute path in node admin to directory with processed and reported core dumps
-     */
-    public Path pathInNodeAdminToDoneCoredumps() {
-        return pathResolver.getApplicationStoragePathForNodeAdmin().resolve("processed-coredumps");
-    }
-
-    /**
-     * Absolute path in node admin container to the node cleanup directory.
-     */
-    public Path pathInNodeAdminToNodeCleanup(ContainerName containerName) {
-        return pathResolver.getApplicationStoragePathForNodeAdmin()
-                .resolve("archive")
-                .resolve(containerName.asString() + "_" + filenameFormatter.format(Instant.now()));
-    }
-
-    /**
      * Translates an absolute path in node agent container to an absolute path in node admin container.
      * @param containerName name of the node agent container
      * @param pathInNode absolute path in that container
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
index dd052e992f7..ce751548f75 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.logging;
 
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
 
@@ -30,7 +31,7 @@ public class FilebeatConfigProvider {
         this.environment = environment;
     }
 
-    public Optional<String> getConfig(NodeSpec node) {
+    public Optional<String> getConfig(NodeAgentContext context, NodeSpec node) {
 
         if (environment.getLogstashNodes().size() == 0 || !node.getOwner().isPresent()) {
             return Optional.empty();
@@ -40,7 +41,7 @@ public class FilebeatConfigProvider {
         String logstashNodeString = environment.getLogstashNodes().stream()
                 .map(this::addQuotes)
                 .collect(Collectors.joining(","));
-        return Optional.of(getTemplate()
+        return Optional.of(getTemplate(context)
                 .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment())
                 .replaceAll(REGION_FIELD, environment.getRegion())
                 .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize))
@@ -58,7 +59,7 @@ public class FilebeatConfigProvider {
                 : String.format("\"%s\"", logstashNode);
     }
 
-    private String getTemplate() {
+    private String getTemplate(NodeAgentContext context) {
         return "################### Filebeat Configuration Example #########################\n" +
                 "\n" +
                 "############################# Filebeat ######################################\n" +
@@ -68,7 +69,7 @@ public class FilebeatConfigProvider {
                 "\n" +
                 "    # vespa\n" +
                 "    - paths:\n" +
-                "        - " + environment.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" +
+                "        - " + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" +
                 "      exclude_files: [\".gz$\"]\n" +
                 "      document_type: vespa\n" +
                 "      fields:\n" +
@@ -84,7 +85,7 @@ public class FilebeatConfigProvider {
                 "\n" +
                 "    # vespa qrs\n" +
                 "    - paths:\n" +
-                "        - " + environment.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" +
+                "        - " + context.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" +
                 "      exclude_files: [\".gz$\"]\n" +
                 "      exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" +
                 "      document_type: vespa-qrs\n" +
@@ -198,7 +199,7 @@ public class FilebeatConfigProvider {
                 "  # To enable logging to files, to_files option has to be set to true\n" +
                 "  files:\n" +
                 "    # The directory where the log files will written to.\n" +
-                "    path: " + environment.pathInNodeUnderVespaHome("logs/filebeat") + "\n" +
+                "    path: " + context.pathInNodeUnderVespaHome("logs/filebeat") + "\n" +
                 "\n" +
                 "    # The name of the files where the logs are written to.\n" +
                 "    name: filebeat\n" +
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 28327acb7c9..ecdf871a5ff 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -1,22 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.collections.Pair;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.io.IOUtils;
+import com.yahoo.log.LogLevel;
 import com.yahoo.system.ProcessExecuter;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
-import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil;
-import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
 import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
 
@@ -25,10 +22,10 @@ import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -36,121 +33,112 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck;
 
 /**
  * @author freva
  */
 public class StorageMaintainer {
-    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final Logger logger = Logger.getLogger(StorageMaintainer.class.getName());
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter
+            .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC);
 
     private final DockerOperations dockerOperations;
     private final ProcessExecuter processExecuter;
     private final Environment environment;
     private final CoredumpHandler coredumpHandler;
-    private final Clock clock;
-
-    private final Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>();
-
-    public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter,
-                             MetricReceiverWrapper metricReceiver, Environment environment,
-                             CoredumpHandler coredumpHandler, Clock clock) {
-        this(dockerOperations, processExecuter, environment, coredumpHandler, clock);
-    }
-
-    public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter,
-                             Environment environment, CoredumpHandler coredumpHandler, Path pathToContainerArchive) {
-        this(dockerOperations, processExecuter, environment, coredumpHandler, Clock.systemUTC());
-    }
+    private final Path archiveContainerStoragePath;
 
     public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter,
-                             Environment environment,
-                             CoredumpHandler coredumpHandler, Clock clock) {
+                             Environment environment, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) {
         this.dockerOperations = dockerOperations;
         this.processExecuter = processExecuter;
         this.environment = environment;
         this.coredumpHandler = coredumpHandler;
-        this.clock = clock;
+        this.archiveContainerStoragePath = archiveContainerStoragePath;
     }
 
-    public void writeMetricsConfig(ContainerName containerName, NodeSpec node) {
+    public void writeMetricsConfig(NodeAgentContext context, NodeSpec node) {
         List<SecretAgentCheckConfig> configs = new ArrayList<>();
 
         // host-life
-        Path hostLifeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life");
+        Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life");
         SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath);
         configs.add(annotatedCheck(node, hostLifeSchedule));
 
         // ntp
-        Path ntpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp");
+        Path ntpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp");
         SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath);
         configs.add(annotatedCheck(node, ntpSchedule));
 
         // coredumps (except for the done coredumps which is handled by the host)
-        Path coredumpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps");
+        Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps");
         SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300,
                 coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin",
-                "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString());
+                "129600", "--crit", "1", "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString());
         configs.add(annotatedCheck(node, coredumpSchedule));
 
         // athenz certificate check
-        Path athenzCertExpiryCheckPath = environment.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs");
+        Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs");
         SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60,
                  athenzCertExpiryCheckPath, "--threshold", "20")
                 .withRunAsUser("root");
         configs.add(annotatedCheck(node, athenzCertExpirySchedule));
 
-        if (node.getNodeType() != NodeType.config) {
+        if (context.nodeType() != NodeType.config) {
             // vespa-health
-            Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health");
+            Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health");
             SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all");
             configs.add(annotatedCheck(node, vespaHealthSchedule));
 
             // vespa
-            Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa");
+            Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa");
             SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all");
             configs.add(annotatedCheck(node, vespaSchedule));
         }
 
-        if (node.getNodeType() == NodeType.config) {
+        if (context.nodeType() == NodeType.config) {
             // configserver
-            Path configServerCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2");
+            Path configServerCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2");
             SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60,
                     configServerCheckPath, "-zero", "configserver");
             configs.add(annotatedCheck(node, configServerSchedule));
 
             //zkbackupage
-            Path zkbackupCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
+            Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
             SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300,
-                    zkbackupCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(),
+                    zkbackupCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(),
                     "-m", "150", "-a", "config-zkbackupage");
             configs.add(annotatedCheck(node, zkbackupSchedule));
         }
 
-        if (node.getNodeType() == NodeType.proxy) {
+        if (context.nodeType() == NodeType.proxy) {
             //routing-configage
-            Path routingAgeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
+            Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
             SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60,
-                    routingAgeCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(),
+                    routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(),
                     "-m", "90", "-a", "routing-configage");
             configs.add(annotatedCheck(node, routingAgeSchedule));
 
             //ssl-check
-            Path sslCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status");
+            Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status");
             SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300,
                     sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30");
             configs.add(annotatedCheck(node, sslSchedule));
         }
 
         // Write config and restart yamas-agent
-        Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/etc/yamas-agent/"));
-        configs.forEach(s -> IOExceptionUtil.uncheck(() -> s.writeTo(yamasAgentFolder)));
-        final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"};
-        dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent);
+        Path yamasAgentFolder = context.pathOnHostFromPathInNode("/etc/yamas-agent");
+        configs.forEach(s -> uncheck(() -> s.writeTo(yamasAgentFolder)));
+        dockerOperations.executeCommandInContainerAsRoot(context.containerName(), "service", "yamas-agent", "restart");
     }
 
     private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) {
@@ -175,42 +163,35 @@ public class StorageMaintainer {
         return check;
     }
 
-    public void writeFilebeatConfig(ContainerName containerName, NodeSpec node) {
-        PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName);
+    public void writeFilebeatConfig(NodeAgentContext context, NodeSpec node) {
         try {
             FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment);
-            Optional<String> config = filebeatConfigProvider.getConfig(node);
+            Optional<String> config = filebeatConfigProvider.getConfig(context, node);
             if (!config.isPresent()) return;
 
-            Path filebeatPath = environment.pathInNodeAdminFromPathInNode(
-                    containerName, Paths.get("/etc/filebeat/filebeat.yml"));
+            Path filebeatPath = context.pathOnHostFromPathInNode("/etc/filebeat/filebeat.yml");
             Files.write(filebeatPath, config.get().getBytes());
-            logger.info("Wrote filebeat config.");
+            context.log(logger, "Wrote filebeat config");
         } catch (Throwable t) {
-            logger.error("Failed writing filebeat config; " + node, t);
+            context.log(logger, LogLevel.ERROR, "Failed writing filebeat config", t);
         }
     }
 
-    public Optional<Long> getDiskUsageFor(ContainerName containerName) {
-        Path containerDir = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/home/"));
+    public Optional<Long> getDiskUsageFor(NodeAgentContext context) {
+        Path containerDir = context.pathOnHostFromPathInNode("/");
         try {
             return Optional.of(getDiskUsedInBytes(containerDir));
         } catch (Throwable e) {
-            PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName);
-            logger.error("Problems during disk usage calculations in " + containerDir.toAbsolutePath(), e);
+            context.log(logger, LogLevel.WARNING, "Problems during disk usage calculations in " + containerDir.toAbsolutePath(), e);
             return Optional.empty();
         }
     }
 
     // Public for testing
     long getDiskUsedInBytes(Path path) throws IOException, InterruptedException {
-        if (!Files.exists(path)) {
-            return 0;
-        }
-
-        final String[] command = {"du", "-xsk", path.toString()};
+        if (!Files.exists(path)) return 0;
 
-        Process duCommand = new ProcessBuilder().command(command).start();
+        Process duCommand = new ProcessBuilder().command("du", "-xsk", path.toString()).start();
         if (!duCommand.waitFor(60, TimeUnit.SECONDS)) {
             duCommand.destroy();
             duCommand.waitFor();
@@ -227,77 +208,40 @@ public class StorageMaintainer {
     }
 
 
-    /**
-     * Deletes old log files for vespa, nginx, logstash, etc.
-     */
-    public void removeOldFilesFromNode(ContainerName containerName) {
-        if (! getMaintenanceThrottlerFor(containerName).shouldRemoveOldFilesNow()) return;
-
-        MaintainerExecutor maintainerExecutor = new MaintainerExecutor();
-        addRemoveOldFilesCommand(maintainerExecutor, containerName);
-
-        maintainerExecutor.execute();
-        getMaintenanceThrottlerFor(containerName).updateNextRemoveOldFilesTime();
-    }
-
-    private void addRemoveOldFilesCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName) {
-        Path[] pathsToClean = {
-                environment.pathInNodeUnderVespaHome("logs/elasticsearch2"),
-                environment.pathInNodeUnderVespaHome("logs/logstash2"),
-                environment.pathInNodeUnderVespaHome("logs/daemontools_y"),
-                environment.pathInNodeUnderVespaHome("logs/nginx"),
-                environment.pathInNodeUnderVespaHome("logs/vespa")
+    /** Deletes old log files for vespa, nginx, logstash, etc. */
+    public void removeOldFilesFromNode(NodeAgentContext context) {
+        Path[] logPaths = {
+                context.pathInNodeUnderVespaHome("logs/elasticsearch2"),
+                context.pathInNodeUnderVespaHome("logs/logstash2"),
+                context.pathInNodeUnderVespaHome("logs/daemontools_y"),
+                context.pathInNodeUnderVespaHome("logs/nginx"),
+                context.pathInNodeUnderVespaHome("logs/vespa")
         };
 
-        for (Path pathToClean : pathsToClean) {
-            Path path = environment.pathInNodeAdminFromPathInNode(containerName, pathToClean);
-            if (Files.exists(path)) {
-                maintainerExecutor.addJob("delete-files")
-                        .withArgument("basePath", path)
-                        .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds())
-                        .withArgument("fileNameRegex", ".*\\.log.+")
-                        .withArgument("recursive", false);
-            }
+        for (Path pathToClean : logPaths) {
+            Path path = context.pathOnHostFromPathInNode(pathToClean);
+            FileFinder.files(path)
+                    .match(olderThan(Duration.ofDays(3)).and(nameMatches(Pattern.compile(".*\\.log.+"))))
+                    .maxDepth(1)
+                    .deleteRecursively();
         }
 
-        Path qrsDir = environment.pathInNodeAdminFromPathInNode(
-                containerName, environment.pathInNodeUnderVespaHome("logs/vespa/qrs"));
-        maintainerExecutor.addJob("delete-files")
-                .withArgument("basePath", qrsDir)
-                .withArgument("maxAgeSeconds", Duration.ofDays(3).getSeconds())
-                .withArgument("recursive", false);
-
-        Path logArchiveDir = environment.pathInNodeAdminFromPathInNode(
-                containerName, environment.pathInNodeUnderVespaHome("logs/vespa/logarchive"));
-        maintainerExecutor.addJob("delete-files")
-                .withArgument("basePath", logArchiveDir)
-                .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds())
-                .withArgument("recursive", false);
-
-        Path fileDistrDir = environment.pathInNodeAdminFromPathInNode(
-                containerName, environment.pathInNodeUnderVespaHome("var/db/vespa/filedistribution"));
-        maintainerExecutor.addJob("delete-files")
-                .withArgument("basePath", fileDistrDir)
-                .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds())
-                .withArgument("recursive", true);
-    }
+        FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/qrs")))
+                .match(olderThan(Duration.ofDays(3)))
+                .deleteRecursively();
 
-    /**
-     * Checks if container has any new coredumps, reports and archives them if so
-     */
-    public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node) {
-        MaintainerExecutor maintainerExecutor = new MaintainerExecutor();
-        addHandleCoredumpsCommand(maintainerExecutor, containerName, node);
-        maintainerExecutor.execute();
+        FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/logarchive")))
+                .match(olderThan(Duration.ofDays(31)))
+                .deleteRecursively();
+
+        FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/filedistribution")))
+                .match(olderThan(Duration.ofDays(31)))
+                .deleteRecursively();
     }
 
-    /**
-     * Will either schedule coredump execution in the given maintainerExecutor or run coredump handling
-     * directly if {@link #coredumpHandler} is set.
-     */
-    private void addHandleCoredumpsCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName, NodeSpec node) {
-        final Path coredumpsPath = environment.pathInNodeAdminFromPathInNode(
-                containerName, environment.pathInNodeUnderVespaHome("var/crash"));
+    /** Checks if container has any new coredumps, reports and archives them if so */
+    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node) {
+        final Path coredumpsPath = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/crash"));
         final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node);
         try {
             coredumpHandler.processAll(coredumpsPath, nodeAttributes);
@@ -329,24 +273,15 @@ public class StorageMaintainer {
      * Prepares the container-storage for the next container by deleting/archiving all the data of the current container.
      * Removes old files, reports coredumps and archives container data, runs when container enters state "dirty"
      */
-    public void cleanupNodeStorage(ContainerName containerName, NodeSpec node) {
-        MaintainerExecutor maintainerExecutor = new MaintainerExecutor();
-        addRemoveOldFilesCommand(maintainerExecutor, containerName);
-        addHandleCoredumpsCommand(maintainerExecutor, containerName, node);
-        addArchiveNodeData(maintainerExecutor, containerName);
-
-        maintainerExecutor.execute();
-        getMaintenanceThrottlerFor(containerName).reset();
-    }
-
-    private void addArchiveNodeData(MaintainerExecutor maintainerExecutor, ContainerName containerName) {
-        maintainerExecutor.addJob("recursive-delete")
-                .withArgument("path", environment.pathInNodeAdminFromPathInNode(
-                        containerName, environment.pathInNodeUnderVespaHome("var")));
-
-        maintainerExecutor.addJob("move-files")
-                .withArgument("from", environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/")))
-                .withArgument("to", environment.pathInNodeAdminToNodeCleanup(containerName));
+    public void archiveNodeStorage(NodeAgentContext context) {
+        Path logsDirInContainer = context.pathInNodeUnderVespaHome("logs");
+        Path containerLogsOnHost = context.pathOnHostFromPathInNode(logsDirInContainer);
+        Path containerLogsInArchiveDir = archiveContainerStoragePath
+                .resolve(context.containerName().asString() + "_" + DATE_TIME_FORMATTER.format(Instant.now()) + logsDirInContainer);
+
+        new UnixPath(containerLogsInArchiveDir).createParents();
+        new UnixPath(containerLogsOnHost).moveIfExists(containerLogsInArchiveDir);
+        new UnixPath(context.pathOnHostFromPathInNode("/")).deleteRecursively();
     }
 
     /**
@@ -400,69 +335,4 @@ public class StorageMaintainer {
             throw new RuntimeException("Failed to execute maintainer", e);
         }
     }
-
-    /**
-     * Wrapper for node-admin-maintenance, queues up maintenances jobs and sends a single request to maintenance JVM
-     */
-    private class MaintainerExecutor {
-        private final List<MaintainerExecutorJob> jobs = new ArrayList<>();
-
-        MaintainerExecutorJob addJob(String jobName) {
-            MaintainerExecutorJob job = new MaintainerExecutorJob(jobName);
-            jobs.add(job);
-            return job;
-        }
-
-        void execute() {
-            if (jobs.isEmpty()) return;
-
-            String args;
-            try {
-                args = objectMapper.writeValueAsString(jobs);
-            } catch (JsonProcessingException e) {
-                throw new RuntimeException("Failed transform list of maintenance jobs to JSON");
-            }
-
-            executeMaintainer("com.yahoo.vespa.hosted.node.maintainer.Maintainer", args);
-        }
-    }
-
-    private class MaintainerExecutorJob {
-        @JsonProperty(value="type")
-        private final String type;
-
-        @JsonProperty(value="arguments")
-        private final Map<String, Object> arguments = new HashMap<>();
-
-        MaintainerExecutorJob(String type) {
-            this.type = type;
-        }
-
-        MaintainerExecutorJob withArgument(String argument, Object value) {
-            // Transform Path to String, otherwise ObjectMapper wont encode/decode it properly on the other end
-            arguments.put(argument, (value instanceof Path) ? value.toString() : value);
-            return this;
-        }
-    }
-
-    private MaintenanceThrottler getMaintenanceThrottlerFor(ContainerName containerName) {
-        maintenanceThrottlerByContainerName.putIfAbsent(containerName, new MaintenanceThrottler());
-        return maintenanceThrottlerByContainerName.get(containerName);
-    }
-
-    private class MaintenanceThrottler {
-        private Instant nextRemoveOldFilesAt = Instant.EPOCH;
-
-        void updateNextRemoveOldFilesTime() {
-            nextRemoveOldFilesAt = clock.instant().plus(Duration.ofHours(1));
-        }
-
-        boolean shouldRemoveOldFilesNow() {
-            return !nextRemoveOldFilesAt.isAfter(clock.instant());
-        }
-
-        void reset() {
-            nextRemoveOldFilesAt = Instant.EPOCH;
-        }
-    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index d58f6e1cb2f..45af3a85810 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -227,8 +227,8 @@ public class NodeAgentImpl implements NodeAgent {
     void resumeNodeIfNeeded(NodeSpec node) {
         if (!hasResumedNode) {
             if (!currentFilebeatRestarter.isPresent()) {
-                storageMaintainer.writeMetricsConfig(context.containerName(), node);
-                storageMaintainer.writeFilebeatConfig(context.containerName(), node);
+                storageMaintainer.writeMetricsConfig(context, node);
+                storageMaintainer.writeFilebeatConfig(context, node);
                 currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay(
                         () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS));
             }
@@ -475,7 +475,7 @@ public class NodeAgentImpl implements NodeAgent {
             // Every time the node spec changes, we should clear the metrics for this container as the dimensions
             // will change and we will be reporting duplicate metrics.
             if (container.map(c -> c.state.isRunning()).orElse(false)) {
-                storageMaintainer.writeMetricsConfig(context.containerName(), node);
+                storageMaintainer.writeMetricsConfig(context, node);
             }
 
             context.log(logger, LogLevel.DEBUG, "Loading new node spec: " + node.toString());
@@ -491,12 +491,12 @@ public class NodeAgentImpl implements NodeAgent {
                 updateNodeRepoWithCurrentAttributes(node);
                 break;
             case active:
-                storageMaintainer.handleCoreDumpsForContainer(context.containerName(), node);
+                storageMaintainer.handleCoreDumpsForContainer(context, node);
 
-                storageMaintainer.getDiskUsageFor(context.containerName())
+                storageMaintainer.getDiskUsageFor(context)
                         .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.getMinDiskAvailableGb())
                         .filter(diskUtil -> diskUtil >= 0.8)
-                        .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context.containerName()));
+                        .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context));
 
                 scheduleDownLoadIfNeeded(node);
                 if (isDownloadingImage()) {
@@ -544,7 +544,7 @@ public class NodeAgentImpl implements NodeAgent {
                 removeContainerIfNeededUpdateContainerState(node, container);
                 context.log(logger, "State is " + node.getState() + ", will delete application storage and mark node as ready");
                 athenzCredentialsMaintainer.clearCredentials();
-                storageMaintainer.cleanupNodeStorage(context.containerName(), node);
+                storageMaintainer.archiveNodeStorage(context);
                 updateNodeRepoWithCurrentAttributes(node);
                 nodeRepository.setNodeState(context.hostname().value(), Node.State.ready);
                 expectNodeNotInNodeRepo = true;
@@ -621,7 +621,7 @@ public class NodeAgentImpl implements NodeAgent {
         final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue();
         final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue();
         final long diskTotalBytes = (long) (node.getMinDiskAvailableGb() * BYTES_IN_GB);
-        final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context.containerName());
+        final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context);
 
         lastCpuMetric.updateCpuDeltas(cpuSystemTotalTime, cpuContainerTotalTime, cpuContainerKernelTime);
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java
index 59dbb6fb5d1..6e679af4449 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java
@@ -46,10 +46,9 @@ public class SecretAgentCheckConfig {
     }
 
     public void writeTo(Path yamasAgentDirectory) throws IOException {
-        if (! Files.exists(yamasAgentDirectory)) yamasAgentDirectory.toFile().mkdirs();
+        Files.createDirectories(yamasAgentDirectory);
         Path scheduleFilePath = yamasAgentDirectory.resolve(id + ".yaml");
         Files.write(scheduleFilePath, render().getBytes());
-        scheduleFilePath.toFile().setReadable(true, false); // Give everyone read access to the schedule file
     }
 
     public FileWriter getFileWriterTo(Path destinationPath) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index da119b756b8..522bcc43444 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -91,7 +91,7 @@ public class DockerTester implements AutoCloseable {
 
         Clock clock = Clock.systemUTC();
         DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, processExecuter);
-        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier, clock);
+        StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier);
         AclMaintainer aclMaintainer = mock(AclMaintainer.class);
         AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class);
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
index 9ba0984c93d..613b3cb5f5c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java
@@ -2,44 +2,40 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.system.ProcessExecuter;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
-import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
 
-import java.time.Clock;
 import java.util.Optional;
 
-import static org.mockito.Mockito.mock;
-
 /**
  * @author freva
  */
 public class StorageMaintainerMock extends StorageMaintainer {
     private final CallOrderVerifier callOrderVerifier;
 
-    public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier, Clock clock) {
-        super(dockerOperations, processExecuter, environment, mock(CoredumpHandler.class), clock);
+    public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier) {
+        super(dockerOperations, processExecuter, environment, null, null);
         this.callOrderVerifier = callOrderVerifier;
     }
 
     @Override
-    public Optional<Long> getDiskUsageFor(ContainerName containerName) {
+    public Optional<Long> getDiskUsageFor(NodeAgentContext context) {
         return Optional.empty();
     }
 
     @Override
-    public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node) {
+    public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node) {
     }
 
     @Override
-    public void removeOldFilesFromNode(ContainerName containerName) {
+    public void removeOldFilesFromNode(NodeAgentContext context) {
     }
 
     @Override
-    public void cleanupNodeStorage(ContainerName containerName, NodeSpec node) {
-        callOrderVerifier.add("DeleteContainerStorage with " + containerName);
+    public void archiveNodeStorage(NodeAgentContext context) {
+        callOrderVerifier.add("DeleteContainerStorage with " + context.containerName());
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
index f418552553e..f7e10c3aa13 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
@@ -7,6 +7,8 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
 import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
@@ -30,12 +32,13 @@ public class FilebeatConfigProviderTest {
     private static final String region = "us-north-1";
     private static final String system = "main";
     private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2");
+    private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("node-123.hostname.tld");
 
     @Test
     public void it_replaces_all_fields_correctly() {
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes));
 
-        Optional<String> config = filebeatConfigProvider.getConfig(createNodeRepositoryNode(tenant, application, instance));
+        Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
 
         assertTrue(config.isPresent());
         String configString = config.get();
@@ -47,7 +50,7 @@ public class FilebeatConfigProviderTest {
         Environment env = getEnvironment(Collections.emptyList());
 
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env);
-        Optional<String> config = filebeatConfigProvider.getConfig(createNodeRepositoryNode(tenant, application, instance));
+        Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
         assertFalse(config.isPresent());
     }
 
@@ -63,7 +66,7 @@ public class FilebeatConfigProviderTest {
                 .minMainMemoryAvailableGb(1)
                 .minDiskAvailableGb(1)
                 .build();
-        Optional<String> config = filebeatConfigProvider.getConfig(node);
+        Optional<String> config = filebeatConfigProvider.getConfig(context, node);
         assertFalse(config.isPresent());
     }
 
@@ -81,7 +84,7 @@ public class FilebeatConfigProviderTest {
     public void it_does_not_add_double_quotes() {
         Environment environment = getEnvironment(ImmutableList.of("unquoted", "\"quoted\""));
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment);
-        Optional<String> config = filebeatConfigProvider.getConfig(createNodeRepositoryNode(tenant, application, instance));
+        Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
         assertThat(config.get(), containsString("hosts: [\"unquoted\",\"quoted\"]"));
     }
 
@@ -94,7 +97,7 @@ public class FilebeatConfigProviderTest {
     private String getConfigString() {
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes));
         NodeSpec node = createNodeRepositoryNode(tenant, application, instance);
-        return filebeatConfigProvider.getConfig(node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config"));
+        return filebeatConfigProvider.getConfig(context, node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config"));
     }
 
     private Environment getEnvironment(List<String> logstashNodes) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
index ed70741c09a..7c10e703bb5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -1,41 +1,40 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance;
 
-import com.yahoo.collections.Pair;
-import com.yahoo.config.provision.NodeType;
+import com.google.common.collect.ImmutableSet;
 import com.yahoo.system.ProcessExecuter;
-import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
 import com.yahoo.vespa.hosted.node.admin.component.PathResolver;
-import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
-import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
-import java.time.Duration;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 /**
  * @author dybis
  */
 public class StorageMaintainerTest {
-    private final ManualClock clock = new ManualClock();
     private final Environment environment = new Environment.Builder()
             .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder()))
             .region("us-east-1")
@@ -47,16 +46,15 @@ public class StorageMaintainerTest {
             .build();
     private final DockerOperations docker = mock(DockerOperations.class);
     private final ProcessExecuter processExecuter = mock(ProcessExecuter.class);
-    private final StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter,
-            environment, mock(CoredumpHandler.class), clock);
 
     @Rule
     public TemporaryFolder folder = new TemporaryFolder();
 
     @Test
     public void testDiskUsed() throws IOException, InterruptedException {
+        StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, null);
         int writeSize = 10000;
-        writeNBytesToFile(folder.newFile(), writeSize);
+        Files.write(folder.newFile().toPath(), new byte[writeSize]);
 
         long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath());
         if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4)
@@ -65,73 +63,79 @@ public class StorageMaintainerTest {
 
     @Test
     public void testNonExistingDiskUsed() throws IOException, InterruptedException {
+        StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, null);
         long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath().resolve("doesn't exist"));
         assertEquals(0L, usedBytes);
     }
 
     @Test
-    public void testMaintenanceThrottlingAfterSuccessfulMaintenance() {
-        String hostname = "node-123.us-north-3.test.yahoo.com";
-        ContainerName containerName = ContainerName.fromHostname(hostname);
-        NodeSpec node = new NodeSpec.Builder()
-                .hostname(hostname)
-                .state(Node.State.ready)
-                .nodeType(NodeType.tenant)
-                .flavor("docker")
-                .minCpuCores(1)
-                .minMainMemoryAvailableGb(1)
-                .minDiskAvailableGb(1)
-                .build();
-
-        try {
-            when(processExecuter.exec(any(String[].class))).thenReturn(new Pair<>(0, ""));
-        } catch (IOException ignored) { }
-        storageMaintainer.removeOldFilesFromNode(containerName);
-        verifyProcessExecuterCalled(1);
-        // Will not actually run maintenance job until an hour passes
-        storageMaintainer.removeOldFilesFromNode(containerName);
-        verifyProcessExecuterCalled(1);
-
-        clock.advance(Duration.ofMinutes(61));
-        storageMaintainer.removeOldFilesFromNode(containerName);
-        verifyProcessExecuterCalled(2);
-
-        // cleanupNodeStorage is unthrottled and it should reset previous times
-        storageMaintainer.cleanupNodeStorage(containerName, node);
-        verifyProcessExecuterCalled(3);
-        storageMaintainer.cleanupNodeStorage(containerName, node);
-        verifyProcessExecuterCalled(4);
+    public void archive_container_data_test() throws IOException {
+        // Create some files in containers
+        FileSystem fileSystem = TestFileSystem.create();
+        NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1");
+        createNodeAgentContextAndContainerStorage(fileSystem, "container-2");
+
+        Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-archive");
+        Files.createDirectories(pathToArchiveDir);
+
+        Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent();
+        Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot)
+                .maxDepth(1)
+                .stream()
+                .map(FileFinder.FileAttributes::filename)
+                .collect(Collectors.toSet());
+        assertEquals(ImmutableSet.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive);
+
+
+        // Archive container-1
+        StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, pathToArchiveDir);
+        storageMaintainer.archiveNodeStorage(context1);
+
+        // container-1 should be gone from container-storage
+        Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot)
+                .maxDepth(1)
+                .stream()
+                .map(FileFinder.FileAttributes::filename)
+                .collect(Collectors.toSet());
+        assertEquals(ImmutableSet.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive);
+
+        // container archive directory should contain exactly 1 directory - the one we just archived
+        List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list();
+        assertEquals(1, containerArchiveContentsAfterArchive.size());
+        Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path();
+        assertTrue(archivedContainerStoragePath.getFileName().toString().matches("container-1_[0-9]{14}"));
+        Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath)
+                .stream()
+                .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString())
+                .collect(Collectors.toSet());
+        assertEquals(ImmutableSet.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents);
     }
 
-    @Test
-    public void testMaintenanceThrottlingAfterFailedMaintenance() {
-        String hostname = "node-123.us-north-3.test.yahoo.com";
-        ContainerName containerName = ContainerName.fromHostname(hostname);
-
-        try {
-            when(processExecuter.exec(any(String[].class)))
-                    .thenThrow(new RuntimeException("Something went wrong"))
-                    .thenReturn(new Pair<>(0, ""));
-        } catch (IOException ignored) { }
-
-        try {
-            storageMaintainer.removeOldFilesFromNode(containerName);
-            fail("Maintenance job should've failed!");
-        } catch (RuntimeException ignored) { }
-        verifyProcessExecuterCalled(1);
-
-        // Maintenance job failed, we should be able to immediately re-run it
-        storageMaintainer.removeOldFilesFromNode(containerName);
-        verifyProcessExecuterCalled(2);
-    }
-
-    private static void writeNBytesToFile(File file, int nBytes) throws IOException {
-        Files.write(file.toPath(), new byte[nBytes]);
-    }
-
-    private void verifyProcessExecuterCalled(int times) {
-        try {
-            verify(processExecuter, times(times)).exec(any(String[].class));
-        } catch (IOException ignored) { }
+    private NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException {
+        NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, containerName + ".domain.tld");
+
+        Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome(""));
+        Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something"));
+        Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf"));
+
+        Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa"));
+        Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log"));
+        Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log"));
+
+        Files.createDirectories(containerVespaHomeOnHost.resolve("var/db"));
+        Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file"));
+
+        Path containerRootOnHost = context.pathOnHostFromPathInNode("/");
+        Set<String> actualContents = FileFinder.files(containerRootOnHost)
+                .stream()
+                .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString())
+                .collect(Collectors.toSet());
+        Set<String> expectedContents = new HashSet<>(Arrays.asList(
+                "etc/something/conf",
+                "opt/vespa/logs/vespa/vespa.log",
+                "opt/vespa/logs/vespa/zookeeper.log",
+                "opt/vespa/var/db/some-file"));
+        assertEquals(expectedContents, actualContents);
+        return context;
     }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 24fcb363f9b..99936f56596 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -120,14 +120,14 @@ public class NodeAgentImplTest {
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
         when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(187500000000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L));
 
         nodeAgent.converge();
 
         verify(dockerOperations, never()).removeContainer(any());
         verify(orchestrator, never()).suspend(any(String.class));
         verify(dockerOperations, never()).pullImageAsyncIfNeeded(any());
-        verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context.containerName()));
+        verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context));
 
         final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository);
         // TODO: Verify this isn't run unless 1st time
@@ -153,11 +153,11 @@ public class NodeAgentImplTest {
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
         when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(217432719360L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(217432719360L));
 
         nodeAgent.converge();
 
-        verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(context.containerName()));
+        verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(context));
     }
 
     @Test
@@ -173,7 +173,7 @@ public class NodeAgentImplTest {
 
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
         when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(187500000000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L));
 
         nodeAgent.converge();
         inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName()));
@@ -215,7 +215,7 @@ public class NodeAgentImplTest {
         when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo"));
         when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
         when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
 
         nodeAgent.converge();
 
@@ -256,7 +256,7 @@ public class NodeAgentImplTest {
 
         when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
         when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true);
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
 
         nodeAgent.converge();
 
@@ -291,7 +291,7 @@ public class NodeAgentImplTest {
                 .thenReturn(Optional.of(secondSpec))
                 .thenReturn(Optional.of(thirdSpec));
         when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true);
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
         when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
 
         nodeAgent.converge();
@@ -458,7 +458,7 @@ public class NodeAgentImplTest {
 
         final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository);
         inOrder.verify(dockerOperations, times(1)).removeContainer(any());
-        inOrder.verify(storageMaintainer, times(1)).cleanupNodeStorage(eq(context.containerName()), eq(node));
+        inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context));
         inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready));
 
         verify(dockerOperations, never()).createContainer(eq(context.containerName()), any(), any());
@@ -513,7 +513,7 @@ public class NodeAgentImplTest {
         when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
         when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo"));
         when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
 
         nodeAgent.tick();
 
@@ -537,7 +537,7 @@ public class NodeAgentImplTest {
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
 
         when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
 
         final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository);
         doThrow(new RuntimeException("Failed 1st time"))
@@ -607,7 +607,7 @@ public class NodeAgentImplTest {
         when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo"));
         when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
         when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
         doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context.containerName()));
 
         try {
@@ -663,7 +663,7 @@ public class NodeAgentImplTest {
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
 
         when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(39625000000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L));
         when(dockerOperations.getContainerStats(eq(context.containerName())))
                 .thenReturn(Optional.of(stats1))
                 .thenReturn(Optional.of(stats2));
@@ -731,7 +731,7 @@ public class NodeAgentImplTest {
         Path tempDirectory = Files.createTempDirectory("foo");
         when(pathResolver.getApplicationStoragePathForHost()).thenReturn(tempDirectory);
         when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
-        when(storageMaintainer.getDiskUsageFor(eq(context.containerName()))).thenReturn(Optional.of(201326592000L));
+        when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
 
         nodeAgent.converge();
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java
deleted file mode 100644
index 893607f1806..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.util;
-
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.component.Environment;
-import com.yahoo.vespa.hosted.node.admin.component.PathResolver;
-import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
-import org.junit.Test;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author freva
- */
-public class EnvironmentTest {
-    private final Environment environment = new Environment.Builder()
-            .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder()))
-            .region("us-east-1")
-            .environment("prod")
-            .system("main")
-            .cloud("mycloud")
-            .pathResolver(new PathResolver())
-            .dockerNetworking(DockerNetworking.HOST_NETWORK)
-            .build();
-
-    @Test
-    public void testPathInNodeToPathInNodeAdminAndHost() {
-        ContainerName containerName = new ContainerName("docker1-1");
-        assertEquals(
-                "/host/home/docker/container-storage/" + containerName.asString(),
-                environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/")).toString());
-
-        assertEquals(
-                "/home/docker/container-storage/" + containerName.asString(),
-                environment.pathInHostFromPathInNode(containerName, Paths.get("/")).toString());
-    }
-
-    @Test
-    public void testAbsolutePathInNodeConversion() {
-        String varPath = getDefaults().underVespaHome("var");
-        ContainerName containerName = new ContainerName("docker1-1");
-        String expected = "/host/home/docker/container-storage/" + containerName.asString() + varPath;
-        String[] absolutePathsInContainer = {"/" + varPath, varPath, varPath + "/"};
-
-        for (String pathInContainer : absolutePathsInContainer) {
-            assertEquals(expected, environment.pathInNodeAdminFromPathInNode(containerName, Paths.get(pathInContainer)).toString());
-        }
-    }
-
-    @Test(expected=IllegalArgumentException.class)
-    public void testNonAbsolutePathInNodeConversion() {
-        Path varPath = Paths.get("some/relative/path");
-        environment.pathInNodeAdminFromPathInNode(new ContainerName("container-1"), varPath);
-    }
-}