diff options
author | Harald Musum <musum@yahooinc.com> | 2022-10-10 13:05:11 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2022-10-10 13:05:11 +0200 |
commit | 987e1014e2def189bf30d3e4bc8f719b21e99784 (patch) | |
tree | 6f932fb6737cf5949d5fecd28a0dd1b76dabfd82 | |
parent | 1eebcf0509e4d645a1c4ab119e2bac2e2756ba7d (diff) |
Use a Stream to read files, might go OOM otherwise
Slight rewrite to be able to use a stream for deleting unused file
references, but still keep some of them (the last accessed ones)
2 files changed, 27 insertions, 29 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java index 1fd842d03e5..c1de9b4e5f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java @@ -27,7 +27,8 @@ public class FileDistributionUtil { } public static boolean fileReferenceExistsOnDisk(File downloadDirectory, FileReference applicationPackageReference) { - return FileDistributionCleanup.getFileReferencesOnDisk(downloadDirectory).contains(applicationPackageReference.value()); + return FileDistributionCleanup.getFileReferencesOnDisk(downloadDirectory.toPath()) + .anyMatch(fileReference -> fileReference.equals(applicationPackageReference.value())); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java index 15b84374fc5..de5ce9554b5 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java @@ -5,18 +5,20 @@ import com.yahoo.io.IOUtils; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.nio.file.Files.readAttributes; @@ -55,30 +57,28 @@ public class FileDistributionCleanup { log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse); - List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration); + Stream<String> candidates = sortedUnusedFileReferences(fileReferencesPath.toPath(), fileReferencesInUse, keepFileReferencesDuration); + List<String> fileReferencesDeleted = new ArrayList<>(); // Do not delete the newest ones - List<String> fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - numberToAlwaysKeep)); - if (fileReferencesToDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); - fileReferencesToDelete.forEach(fileReference -> { + final AtomicInteger i = new AtomicInteger(0); + candidates.forEach(fileReference -> { + if (i.incrementAndGet() > numberToAlwaysKeep) { + fileReferencesDeleted.add(fileReference); File file = new File(fileReferencesPath, fileReference); if (!IOUtils.recursiveDeleteDir(file)) log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); - }); - } - return fileReferencesToDelete; + } + }); + return fileReferencesDeleted; } - private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) { - Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + // Sorted, newest first + private Stream<String> sortedUnusedFileReferences(Path fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) { Instant instant = clock.instant().minus(keepFileReferences); - return fileReferencesOnDisk - .stream() + return getFileReferencesOnDisk(fileReferencesPath) .filter(fileReference -> !fileReferencesInUse.contains(fileReference)) - .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant)) - .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath, a)))) - .collect(Collectors.toList()); + .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath.toFile(), fileReference), instant)) + .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath.toFile(), (String) a))).reversed()); } private boolean isLastFileAccessBefore(File fileReference, Instant instant) { @@ -95,15 +95,12 @@ public class FileDistributionCleanup { } } - /** - * Returns all files in the given directory, non-recursive. - */ - public static Set<String> getFileReferencesOnDisk(File directory) { - Set<String> fileReferencesOnDisk = new HashSet<>(); - File[] filesOnDisk = directory.listFiles(); - if (filesOnDisk != null) - fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); - return fileReferencesOnDisk; + public static Stream<String> getFileReferencesOnDisk(Path directory) { + try { + return Files.list(directory).map(path -> path.toFile().getName()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } |