diff options
Diffstat (limited to 'config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java')
-rw-r--r-- | config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java | 61 |
1 files changed, 45 insertions, 16 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index a3dd8d27c0f..3eafdf8f2b4 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -1,17 +1,27 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; +import com.yahoo.io.IOUtils; +import java.util.logging.Level; import com.yahoo.vespa.filedistribution.FileDownloader; -import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup; + import java.io.File; -import java.time.Clock; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.nio.file.Files.readAttributes; /** - * Deletes file references and url downloads on disk that have not been used for some time + * Deletes cached file references and url downloads that have not been used for some time * * @author hmusum */ @@ -21,29 +31,20 @@ class CachedFilesMaintainer implements Runnable { private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; - private static final Duration defaultDurationToKeepFiles = Duration.ofDays(20); - private static final int defaultKeepCount = 20; + private static final Duration defaultDurationToKeepFiles = Duration.ofDays(30); private final File urlDownloadDir; private final File fileReferencesDownloadDir; private final Duration durationToKeepFiles; - private final FileDistributionCleanup cleanup; - private final int keepCount; // keep this many files no matter how old they are or when they were last accessed CachedFilesMaintainer() { - this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles, Clock.systemUTC(), defaultKeepCount); + this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles); } - CachedFilesMaintainer(File fileReferencesDownloadDir, - File urlDownloadDir, - Duration durationToKeepFiles, - Clock clock, - int keepCount) { + CachedFilesMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) { this.fileReferencesDownloadDir = fileReferencesDownloadDir; this.urlDownloadDir = urlDownloadDir; this.durationToKeepFiles = durationToKeepFiles; - this.cleanup = new FileDistributionCleanup(clock); - this.keepCount = keepCount; } @Override @@ -57,7 +58,35 @@ class CachedFilesMaintainer implements Runnable { } private void deleteUnusedFiles(File directory) { - cleanup.deleteUnusedFileReferences(directory, durationToKeepFiles, keepCount, Set.of()); + Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles); + Set<String> filesOnDisk = new HashSet<>(); + File[] files = directory.listFiles(); + if (files != null) + filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet())); + log.log(Level.FINE, () -> "Files on disk (in " + directory + "): " + filesOnDisk); + + Set<String> filesToDelete = filesOnDisk + .stream() + .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant)) + .collect(Collectors.toSet()); + if (filesToDelete.size() > 0) { + log.log(Level.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); + filesToDelete.forEach(fileReference -> { + File file = new File(directory, fileReference); + if (!IOUtils.recursiveDeleteDir(file)) + log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); + }); + } + } + + private boolean isFileLastModifiedBefore(File fileReference, Instant instant) { + BasicFileAttributes fileAttributes; + try { + fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); + return fileAttributes.lastModifiedTime().toInstant().isBefore(instant); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } |