Copy FileHelper from node-maintainer

2 files changed, 501 insertions, 0 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java
new file mode 100644
index 00000000000..cf010121c2a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java
@@ -0,0 +1,177 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author freva
+ */
+public class FileHelper {
+    private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName());
+
+    /**
+     * (Recursively) deletes files if they match all the criteria, also deletes empty directories.
+     *
+     * @param basePath      Base path from where to start the search
+     * @param maxAge        Delete files older (last modified date) than maxAge
+     * @param fileNameRegex Delete files where filename matches fileNameRegex
+     * @param recursive     Delete files in sub-directories (with the same criteria)
+     */
+    public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException {
+        Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null);
+
+        for (Path path : listContentsOfDirectory(basePath)) {
+            if (Files.isDirectory(path)) {
+                if (recursive) {
+                    deleteFiles(path, maxAge, fileNameRegex, true);
+                    if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) {
+                        logger.warning("Could not delete directory: " + path.toAbsolutePath());
+                    }
+                }
+            } else if (isPatternMatchingFilename(fileNamePattern, path) &&
+                    isTimeSinceLastModifiedMoreThan(path, maxAge)) {
+                if (! Files.deleteIfExists(path)) {
+                    logger.warning("Could not delete file: " + path.toAbsolutePath());
+                }
+            }
+        }
+    }
+
+    /**
+     * Deletes all files in target directory except the n most recent (by modified date)
+     *
+     * @param basePath          Base path to delete from
+     * @param nMostRecentToKeep Number of most recent files to keep
+     */
+    static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException {
+        if (nMostRecentToKeep < 1) {
+            throw new IllegalArgumentException("Number of files to keep must be a positive number");
+        }
+
+        List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream()
+                .filter(Files::isRegularFile)
+                .sorted(Comparator.comparing(FileHelper::getLastModifiedTime))
+                .skip(nMostRecentToKeep)
+                .collect(Collectors.toList());
+
+        for (Path path : pathsInDeleteDir) {
+            if (!Files.deleteIfExists(path)) {
+                logger.warning("Could not delete file: " + path.toAbsolutePath());
+            }
+        }
+    }
+
+    static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException {
+        for (Path path : listContentsOfDirectory(basePath)) {
+            if (Files.isDirectory(path)) {
+                deleteFilesLargerThan(path, sizeInBytes);
+            } else {
+                if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) {
+                    logger.warning("Could not delete file: " + path.toAbsolutePath());
+                }
+            }
+        }
+    }
+
+    /**
+     * Deletes directories and their contents if they match all the criteria
+     *
+     * @param basePath      Base path to delete the directories from
+     * @param maxAge        Delete directories older (last modified date) than maxAge
+     * @param dirNameRegex  Delete directories where directory name matches dirNameRegex
+     */
+    public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException {
+        Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null);
+
+        for (Path path : listContentsOfDirectory(basePath)) {
+            if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) {
+                boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path)
+                        .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge))
+                        .orElse(true);
+
+                if (mostRecentFileModifiedBeforeMaxAge) {
+                    deleteFiles(path, Duration.ZERO, Optional.empty(), true);
+                    if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) {
+                        logger.warning("Could not delete directory: " + path.toAbsolutePath());
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Similar to rm -rf file:
+     *   - It's not an error if file doesn't exist
+     *   - If file is a directory, it and all content is removed
+     *   - For symlinks: Only the symlink is removed, not what the symlink points to
+     */
+    public static void recursiveDelete(Path basePath) throws IOException {
+        if (Files.isDirectory(basePath)) {
+            for (Path path : listContentsOfDirectory(basePath)) {
+                recursiveDelete(path);
+            }
+        }
+
+        Files.deleteIfExists(basePath);
+    }
+
+    public static void moveIfExists(Path from, Path to) throws IOException {
+        if (Files.exists(from)) {
+            Files.move(from, to);
+        }
+    }
+
+    private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException {
+        return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime));
+    }
+
+    private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) {
+        Instant nowMinusDuration = Instant.now().minus(duration);
+        Instant lastModified = getLastModifiedTime(path).toInstant();
+
+        // Return true also if they are equal for test stability
+        // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration)
+        return !lastModified.isAfter(nowMinusDuration);
+    }
+
+    private static boolean isPatternMatchingFilename(Pattern pattern, Path path) {
+        return pattern == null || pattern.matcher(path.getFileName().toString()).find();
+    }
+
+    /**
+     * @return list all files in a directory, returns empty list if directory does not exist
+     */
+    public static List<Path> listContentsOfDirectory(Path basePath) {
+        try (Stream<Path> directoryStream = Files.list(basePath)) {
+            return directoryStream.collect(Collectors.toList());
+        } catch (NoSuchFileException ignored) {
+            return Collections.emptyList();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e);
+        }
+    }
+
+    static FileTime getLastModifiedTime(Path path) {
+        try {
+            return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS);
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e);
+        }
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java
new file mode 100644
index 00000000000..6b53bc217c4
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java
@@ -0,0 +1,324 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author freva
+ */
+public class FileHelperTest {
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+
+    @Before
+    public void initFiles() throws IOException {
+        for (int i=0; i<10; i++) {
+            File temp = folder.newFile("test_" + i + ".json");
+            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis());
+        }
+
+        for (int i=0; i<7; i++) {
+            File temp = folder.newFile("test_" + i + "_file.test");
+            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis());
+        }
+
+        for (int i=0; i<5; i++) {
+            File temp = folder.newFile(i + "-abc" + ".json");
+            temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis());
+        }
+
+        File temp = folder.newFile("week_old_file.json");
+        temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis());
+    }
+
+    @Test
+    public void testDeleteAll() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false);
+
+        assertEquals(0, getContentsOfDirectory(folder.getRoot()).length);
+    }
+
+    @Test
+    public void testDeletePrefix() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false);
+
+        assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file
+    }
+
+    @Test
+    public void testDeleteSuffix() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false);
+
+        assertEquals(7, getContentsOfDirectory(folder.getRoot()).length);
+    }
+
+    @Test
+    public void testDeletePrefixAndSuffix() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false);
+
+        assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file
+    }
+
+    @Test
+    public void testDeleteOld() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false);
+
+        assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file
+    }
+
+    @Test
+    public void testDeleteWithAllParameters() throws IOException {
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false);
+
+        assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json)
+    }
+
+    @Test
+    public void testDeleteWithSubDirectoriesNoRecursive() throws IOException {
+        initSubDirectories();
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false);
+
+        // 6 test_*.json from test_folder1/
+        // + 9 test_*.json and 4 abc_*.json from test_folder2/
+        // + 13 test_*.json from test_folder2/subSubFolder2/
+        // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
+        // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves
+        assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
+    }
+
+    @Test
+    public void testDeleteWithSubDirectoriesRecursive() throws IOException {
+        initSubDirectories();
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true);
+
+        // 4 abc_*.json from test_folder2/
+        // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
+        // + test_folder2/ itself
+        assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
+    }
+
+    @Test
+    public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException {
+        initSubDirectories();
+
+        FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false);
+
+        assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories
+    }
+
+    @Test
+    public void testGetContentsOfNonExistingDirectory() {
+        Path fakePath = Paths.get("/some/made/up/dir/");
+        assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath));
+    }
+
+    @Test(expected=IllegalArgumentException.class)
+    public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException {
+        FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5);
+    }
+
+    @Test
+    public void testDeleteFilesExceptFiveMostRecent() throws IOException {
+        FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5);
+
+        assertEquals(5, getContentsOfDirectory(folder.getRoot()).length);
+
+        String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"};
+        String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot()))
+                .map(File::getName)
+                .sorted()
+                .toArray(String[]::new);
+
+        assertArrayEquals(oldestFiles, remainingFiles);
+    }
+
+    @Test
+    public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException {
+        String[] filesPreDelete = folder.getRoot().list();
+
+        FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50);
+
+        assertArrayEquals(filesPreDelete, folder.getRoot().list());
+    }
+
+    @Test
+    public void testDeleteFilesLargerThan10B() throws IOException {
+        initSubDirectories();
+
+        File temp1 = new File(folder.getRoot(), "small_file");
+        writeNBytesToFile(temp1, 50);
+
+        File temp2 = new File(folder.getRoot(), "some_file");
+        writeNBytesToFile(temp2, 20);
+
+        File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file");
+        writeNBytesToFile(temp3, 75);
+
+        FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10);
+
+        assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
+        assertFalse(temp1.exists() || temp2.exists() || temp3.exists());
+    }
+
+    @Test
+    public void testDeleteDirectories() throws IOException {
+        initSubDirectories();
+
+        FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2"));
+
+        //23 files in root
+        // + 6 in test_folder1 + test_folder1 itself
+        assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
+    }
+
+    @Test
+    public void testDeleteDirectoriesBasedOnAge() throws IOException {
+        initSubDirectories();
+        // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is
+        // also older than maxAge inside the sub directory, create some files which are newer than maxAge.
+        // deleteDirectories() should NOT delete folder3
+        File subFolder3 = folder.newFolder("test_folder3");
+        File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3");
+
+        for (int j=0; j<11; j++) {
+            File.createTempFile("test_", ".json", subSubFolder3);
+        }
+
+        subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis());
+        subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis());
+
+        FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*"));
+
+        //23 files in root
+        // + 13 in test_folder2
+        // + 13 in subSubFolder2
+        // + 11 in subSubFolder3
+        // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself
+        assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
+    }
+
+    @Test
+    public void testRecursivelyDeleteDirectory() throws IOException {
+        initSubDirectories();
+        FileHelper.recursiveDelete(folder.getRoot().toPath());
+        assertFalse(folder.getRoot().exists());
+    }
+
+    @Test
+    public void testRecursivelyDeleteRegularFile() throws IOException {
+        File file = folder.newFile();
+        assertTrue(file.exists());
+        assertTrue(file.isFile());
+        FileHelper.recursiveDelete(file.toPath());
+        assertFalse(file.exists());
+    }
+
+    @Test
+    public void testRecursivelyDeleteNonExistingFile() throws IOException {
+        File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile();
+        assertFalse(file.exists());
+        FileHelper.recursiveDelete(file.toPath());
+        assertFalse(file.exists());
+    }
+
+    @Test
+    public void testInitSubDirectories() throws IOException {
+        initSubDirectories();
+        assertTrue(folder.getRoot().exists());
+        assertTrue(folder.getRoot().isDirectory());
+
+        Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1");
+        assertTrue(test_folder1.toFile().exists());
+        assertTrue(test_folder1.toFile().isDirectory());
+
+        Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2");
+        assertTrue(test_folder2.toFile().exists());
+        assertTrue(test_folder2.toFile().isDirectory());
+
+        Path subSubFolder2 = test_folder2.resolve("subSubFolder2");
+        assertTrue(subSubFolder2.toFile().exists());
+        assertTrue(subSubFolder2.toFile().isDirectory());
+    }
+
+    @Test
+    public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException {
+        Path symPath = folder.getRoot().toPath().resolve("symlink");
+        Path fakePath = Paths.get("/some/not/existant/file");
+
+        Files.createSymbolicLink(symPath, fakePath);
+        assertTrue(Files.isSymbolicLink(symPath));
+        assertFalse(Files.exists(fakePath));
+
+        // Not possible to set modified time on symlink in java, so just check that it doesn't crash
+        FileHelper.getLastModifiedTime(symPath).toInstant();
+    }
+
+    private void initSubDirectories() throws IOException {
+        File subFolder1 = folder.newFolder("test_folder1");
+        File subFolder2 = folder.newFolder("test_folder2");
+        File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2");
+
+        for (int j=0; j<6; j++) {
+            File temp = File.createTempFile("test_", ".json", subFolder1);
+            temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis());
+        }
+
+        for (int j=0; j<9; j++) {
+            File.createTempFile("test_", ".json", subFolder2);
+        }
+
+        for (int j=0; j<4; j++) {
+            File.createTempFile("abc_", ".txt", subFolder2);
+        }
+
+        for (int j=0; j<13; j++) {
+            File temp = File.createTempFile("test_", ".json", subSubFolder2);
+            temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis());
+        }
+
+        //Must be after all the files have been created
+        subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis());
+        subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis());
+        subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis());
+    }
+
+    private static int getNumberOfFilesAndDirectoriesIn(File folder) {
+        int total = 0;
+        for (File file : getContentsOfDirectory(folder)) {
+            if (file.isDirectory()) {
+                total += getNumberOfFilesAndDirectoriesIn(file);
+            }
+            total++;
+        }
+
+        return total;
+    }
+
+    private static void writeNBytesToFile(File file, int nBytes) throws IOException {
+        Files.write(file.toPath(), new byte[nBytes]);
+    }
+
+    private static File[] getContentsOfDirectory(File directory) {
+        File[] directoryContents = directory.listFiles();
+
+        return directoryContents == null ? new File[0] : directoryContents;
+    }
+}