Create DiskCleanup

9 files changed, 558 insertions, 1 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java
new file mode 100644
index 00000000000..2ad887514e5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java
@@ -0,0 +1,106 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoField;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith;
+
+/**
+ * @author freva
+ */
+public class CoredumpCleanupRule {
+
+    private static final Comparator<FileAttributes> CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR = Comparator
+            .comparing((FileAttributes fa) -> !fa.filename().contains("vespa-"))
+            .thenComparing(FileAttributes::lastModifiedTime);
+
+    public static DiskCleanupRule forContainer(Path containerCrashPath) {
+        return new ContainerCoredumpCleanupRule(containerCrashPath);
+    }
+
+    public static DiskCleanupRule forHost(Path processedCoredumpsPath) {
+        return new HostCoredumpCleanupRule(processedCoredumpsPath);
+    }
+
+    /** Assigns MEDIUM priority to the oldest, unprocessed coredump and HIGHEST for the remaining */
+    private static class ContainerCoredumpCleanupRule implements DiskCleanupRule {
+        private final Path containerCrashPath;
+
+        private ContainerCoredumpCleanupRule(Path containerCrashPath) {
+            this.containerCrashPath = containerCrashPath;
+        }
+
+        @Override
+        public Collection<PrioritizedFileAttributes> prioritize() {
+            List<FileAttributes> fileAttributes = FileFinder.files(containerCrashPath)
+                    .maxDepth(1).stream()
+                    .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR)
+                    .collect(Collectors.toList());
+
+            return mapFirstAndRemaining(fileAttributes, Priority.MEDIUM, Priority.HIGHEST).collect(Collectors.toList());
+        }
+    }
+
+    /** Assigns MEDIUM priority to the first coredump of the day for each container, HIGH for the remaining */
+    private static class HostCoredumpCleanupRule implements DiskCleanupRule {
+        private final Path processedCoredumpsPath;
+
+        private HostCoredumpCleanupRule(Path processedCoredumpsPath) {
+            this.processedCoredumpsPath = processedCoredumpsPath;
+        }
+
+        @Override
+        public Collection<PrioritizedFileAttributes> prioritize() {
+            Map<String, List<FileAttributes>> fileAttributesByContainerDay = FileFinder.files(processedCoredumpsPath)
+                    .match(nameStartsWith(CoredumpHandler.COREDUMP_FILENAME_PREFIX))
+                    .stream()
+                    .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR)
+                    .collect(Collectors.groupingBy(
+                            // Group FileAttributes by string [container-name]_[day of year], e.g. zt00534-v6-2_234
+                            fa -> containerNameFromProcessedCoredumpPath(fa.path()) + "_" + dayOfYear(fa.lastModifiedTime()),
+                            Collectors.collectingAndThen(
+                                    Collectors.toCollection(ArrayList::new),
+                                    l -> { l.sort(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR); return l; } )));
+
+            return fileAttributesByContainerDay.values().stream()
+                    .flatMap(fa -> mapFirstAndRemaining(fa, Priority.MEDIUM, Priority.HIGH))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Maps list of FileAttributes into list of PrioritizedFileAttributes where the first FileAttribute is given
+     * {@code first} priority, while the remaining FileAttributes are given {@code remaining} priority */
+    private static Stream<PrioritizedFileAttributes> mapFirstAndRemaining(List<FileAttributes> fileAttributes, Priority first, Priority remaining) {
+        return IntStream.range(0, fileAttributes.size())
+                .mapToObj(i -> new PrioritizedFileAttributes(fileAttributes.get(i), i == 0 ? first : remaining));
+    }
+
+    /** Extracts container-name from path under processed-coredumps or empty string */
+    private static String containerNameFromProcessedCoredumpPath(Path path) {
+        if (path.getNameCount() < 3) return ""; // Path is too short
+        return path.getName(path.getNameCount() - 3).toString();
+    }
+
+    /** Returns day number of the year (1-365 (or 366 for leap years)) */
+    private static int dayOfYear(Instant instant) {
+        return instant.atOffset(ZoneOffset.UTC).get(ChronoField.DAY_OF_YEAR);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java
new file mode 100644
index 00000000000..a9c1f8a9600
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java
@@ -0,0 +1,67 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author freva
+ */
+public class DiskCleanup {
+
+    private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName());
+    private static final char[] UNITS = "kMGTPE".toCharArray();
+    private static final Comparator<PrioritizedFileAttributes> PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator
+            .comparing(PrioritizedFileAttributes::priority)
+            .thenComparingLong(f -> f.fileAttributes().size())
+            .reversed();
+
+    public boolean cleanup(TaskContext context, List<DiskCleanupRule> rules, long bytesToRemove) {
+        if (bytesToRemove <= 0) return false;
+
+        long[] btr = new long[] { bytesToRemove };
+        List<Path> deletedPaths = new ArrayList<>();
+        try {
+            rules.stream()
+                    .flatMap(rule -> rule.prioritize().stream())
+                    .sorted(PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR)
+                    .takeWhile(fa -> btr[0] > 0)
+                    .forEach(pfa -> {
+                        if (uncheck(() -> Files.deleteIfExists(pfa.fileAttributes().path()))) {
+                            btr[0] -= pfa.fileAttributes().size();
+                            deletedPaths.add(pfa.fileAttributes().path());
+                        }
+                    });
+
+        } finally {
+            String wantedDeleteSize = bytesToDisplayCount(bytesToRemove);
+            String deletedSize = bytesToDisplayCount(bytesToRemove - btr[0]);
+            if (deletedPaths.size() > 20) {
+                context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize);
+            } else if (deletedPaths.size() > 0) {
+                context.log(logger, "Deleted %s because disk was getting full from: %s", deletedSize, deletedPaths);
+            } else {
+                context.log(logger, "Wanted to delete %s, but failed to find any files to delete", wantedDeleteSize);
+            }
+        }
+
+        return !deletedPaths.isEmpty();
+    }
+
+    static String bytesToDisplayCount(long bytes) {
+        if (bytes < 1000) return bytes + " bytes";
+
+        int unit = -1;
+        for (; bytes >= 1000; unit++) bytes /= 1000;
+        return bytes + " " + UNITS[unit] + "B";
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java
new file mode 100644
index 00000000000..a4c8229d0d7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java
@@ -0,0 +1,31 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+
+import java.util.Collection;
+
+/**
+ * @author freva
+ */
+public interface DiskCleanupRule {
+
+    Collection<PrioritizedFileAttributes> prioritize();
+
+    enum Priority {
+        LOWEST, LOW, MEDIUM, HIGH, HIGHEST
+    }
+
+    class PrioritizedFileAttributes {
+        private final FileFinder.FileAttributes fileAttributes;
+        private final Priority priority;
+
+        public PrioritizedFileAttributes(FileFinder.FileAttributes fileAttributes, Priority priority) {
+            this.fileAttributes = fileAttributes;
+            this.priority = priority;
+        }
+
+        public Priority priority() { return priority; }
+        public FileFinder.FileAttributes fileAttributes() { return fileAttributes; }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java
new file mode 100644
index 00000000000..3ef4e2e4f6a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java
@@ -0,0 +1,48 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+
+/**
+ * Prioritizes files by first scoring them with the given scoring function and then mapping the scores to a
+ * priority within the given range.
+ * The priority room is evenly split between given lowest and highest priority for range [0, 1.0). Scores below 0
+ * are assigned lowest, while scores at or higher than 1 are assigned highest priority.
+ *
+ * Typical use-case is for log files. The scoring function calculates the file age and normalizes it by dividing it
+ * by expected max age of log files. The oldest logs will then by prioritized by highest given priority.
+ *
+ * @author freva
+ */
+public class LinearCleanupRule implements DiskCleanupRule {
+    private final Supplier<List<FileAttributes>> lister;
+    private final Function<FileAttributes, Priority> prioritizer;
+
+    public LinearCleanupRule(Supplier<List<FileAttributes>> lister,
+                             Function<FileAttributes, Double> scorer, Priority lowest, Priority highest) {
+        if (lowest.ordinal() > highest.ordinal())
+            throw new IllegalArgumentException("Lowest priority: " + lowest + " is higher than highest priority: " + highest);
+
+        this.lister = lister;
+
+        Priority[] values = Priority.values();
+        int range = highest.ordinal() - lowest.ordinal() + 1;
+        this.prioritizer = fa -> {
+            int ordinal = (int) (lowest.ordinal() + scorer.apply(fa) * range);
+            return values[Math.max(lowest.ordinal(), Math.min(highest.ordinal(), ordinal))];
+        };
+    }
+
+    @Override
+    public Collection<PrioritizedFileAttributes> prioritize() {
+        return lister.get().stream()
+                .map(fa -> new PrioritizedFileAttributes(fa, prioritizer.apply(fa)))
+                .collect(Collectors.toList());
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java
new file mode 100644
index 00000000000..dff40015522
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java
index f0b064373d2..a41281d0479 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java
@@ -193,7 +193,7 @@ public class FileFinder {
         private final Path path;
         private final BasicFileAttributes attributes;
 
-        FileAttributes(Path path, BasicFileAttributes attributes) {
+        public FileAttributes(Path path, BasicFileAttributes attributes) {
             this.path = path;
             this.attributes = attributes;
         }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java
new file mode 100644
index 00000000000..e8ca2ff8491
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java
@@ -0,0 +1,103 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class CoredumpCleanupRuleTest {
+
+    private final FileSystem fileSystem = TestFileSystem.create();
+
+    @Test
+    public void for_container_test() throws IOException {
+        Path path = fileSystem.getPath("/test/path");
+        DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path);
+
+        assertPriorities(rule, Map.of());
+
+        createFile(path.resolve("core1"), Instant.ofEpochSecond(232));
+        assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM));
+
+        createFile(path.resolve("core2"), Instant.ofEpochSecond(123));
+        assertPriorities(rule, Map.of(
+                "/test/path/core2", Priority.MEDIUM,
+                "/test/path/core1", Priority.HIGHEST));
+
+        createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456));
+        createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256));
+        var expected = Map.of(
+                "/test/path/core2", Priority.HIGHEST,
+                "/test/path/core1", Priority.HIGHEST,
+                "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST,
+                "/test/path/vespa-distributor.core.764", Priority.MEDIUM);
+        assertPriorities(rule, expected);
+
+        // processing core has no effect on this
+        Files.createDirectories(path.resolve("processing/abcd-1234"));
+        createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67));
+        assertPriorities(rule, expected);
+    }
+
+    @Test
+    public void for_host_test() throws IOException {
+        Path path = fileSystem.getPath("/test/path");
+        DiskCleanupRule rule = CoredumpCleanupRule.forHost(path);
+
+        assertPriorities(rule, Map.of());
+
+        createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z"));
+        createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z"));
+        assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM));
+
+        createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z"));
+        createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z"));
+        createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z"));
+        createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z"));
+
+        // Also create a core for a second container: h123b
+        createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z"));
+        createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z"));
+
+        assertPriorities(rule, Map.of(
+                "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH,
+                "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH,
+
+                // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa-
+                "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM,
+                "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM,
+                "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM,
+                "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM,
+                "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM
+        ));
+    }
+
+    private static void createFile(Path path, Instant instant) throws IOException {
+        Files.createDirectories(path.getParent());
+        Files.createFile(path);
+        Files.setLastModifiedTime(path, FileTime.from(instant));
+    }
+
+    private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) {
+        Map<String, Priority> actual = rule.prioritize().stream()
+                .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority));
+
+        assertEquals(new TreeMap<>(expected), new TreeMap<>(actual));
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java
new file mode 100644
index 00000000000..3f383a68b84
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java
@@ -0,0 +1,142 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author freva
+ */
+public class DiskCleanupTest {
+
+    private final TestTaskContext context = new TestTaskContext();
+    private final DiskCleanupTester tester = new DiskCleanupTester();
+    private final DiskCleanup diskCleanup = new DiskCleanup();
+
+    @Test
+    public void nothing_deleted() throws IOException {
+        assertFalse(diskCleanup.cleanup(context, List.of(), 0));
+        assertFalse(diskCleanup.cleanup(context, List.of(), 10));
+
+        DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock();
+        DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock();
+        assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0));
+        assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+
+        tester.createFile("/path/that-should-not-be-deleted", 5);
+        assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+        tester.assertAllFilesExistExcept();
+
+        // Create a file and let rule return it, but before cleanup is run, the file is deleted
+        rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST);
+        Files.delete(tester.path("/path/file-does-not-exist"));
+        assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+    }
+
+    @Test
+    public void delete_test() throws IOException {
+        tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1);
+        tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1);
+        tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1);
+        tester.createFile("/opt/vespa/var/index/something-important", 1);
+
+        DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock()
+                .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM)
+                .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH)
+                .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST)
+                .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST);
+        DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock()
+                .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW)
+                .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH)
+                .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM)
+                .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH);
+
+        // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since
+        // it is >= 10 bytes, no more files are deleted
+        assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+        tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log");
+
+        // Called with the same arguments, but vespa-3.log is still missing...
+        assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+        tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log");
+
+        assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500));
+        tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before
+                // 300 + 190 + 8 + 54
+                "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3");
+    }
+
+    @Test
+    public void bytes_to_display_count_test() {
+        assertEquals("-1 bytes", DiskCleanup.bytesToDisplayCount(-1));
+        assertEquals("123 bytes", DiskCleanup.bytesToDisplayCount(123));
+        assertEquals("1 kB", DiskCleanup.bytesToDisplayCount(1_000));
+        assertEquals("15 MB", DiskCleanup.bytesToDisplayCount(15_000_000));
+        assertEquals("123 GB", DiskCleanup.bytesToDisplayCount(123_456_789_012L));
+        assertEquals("987 TB", DiskCleanup.bytesToDisplayCount(987_654_321_098_765L));
+        assertEquals("2 PB", DiskCleanup.bytesToDisplayCount(2_000_000_000_000_000L));
+        assertEquals("9 EB", DiskCleanup.bytesToDisplayCount(Long.MAX_VALUE));
+
+    }
+
+    private static class DiskCleanupRuleMock implements DiskCleanupRule {
+        private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>();
+
+        private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException {
+            PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes();
+            pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority));
+            return this;
+        }
+
+        @Override
+        public Collection<PrioritizedFileAttributes> prioritize() {
+            return Collections.unmodifiableList(pfa);
+        }
+    }
+
+    private static class DiskCleanupTester {
+        private final FileSystem fileSystem = TestFileSystem.create();
+        private final Set<String> files = new HashSet<>();
+
+        private Path path(String path) {
+            return fileSystem.getPath(path);
+        }
+
+        private Path createFile(String pathStr, int size) throws IOException {
+            Path path = path(pathStr);
+            Files.createDirectories(path.getParent());
+            Files.write(path, new byte[size]);
+            files.add(path.toString());
+            return path;
+        }
+
+        private void assertAllFilesExistExcept(String... deletedPaths) {
+            Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet());
+            Set<String> expected = new HashSet<>(files);
+            expected.removeAll(Set.of(deletedPaths));
+            assertEquals(expected, actual);
+        }
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java
new file mode 100644
index 00000000000..ae2d3af4f21
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java
@@ -0,0 +1,55 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author freva
+ */
+public class LinearCleanupRuleTest {
+
+    @Test
+    public void basic() {
+        assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST);
+
+        assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW);
+        assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM);
+
+        assertRule(Map.of(
+                -5.0, Priority.LOW,
+                0.0, Priority.LOW,
+                0.2, Priority.LOW,
+                0.35, Priority.MEDIUM,
+                0.65, Priority.MEDIUM,
+                0.8, Priority.HIGH,
+                1.0, Priority.HIGH,
+                5.0, Priority.HIGH),
+                Priority.LOW, Priority.HIGH);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void fail_if_high_priority_lower_than_low() {
+        assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST);
+    }
+
+    private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) {
+        Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream()
+                .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score));
+        LinearCleanupRule rule = new LinearCleanupRule(
+                () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high);
+
+        Map<Double, Priority> actualPriorities = rule.prioritize().stream()
+                .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority));
+        assertEquals(expectedPriorities, actualPriorities);
+    }
+}
\ No newline at end of file