Merge pull request #24368 from vespa-engine/hmusum/refactor-cleanup-of-file-references

Refactor cleanup of file references

8 files changed, 179 insertions, 139 deletions

diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml
index 476f5f99b86..bd1f907f87e 100644
--- a/config-proxy/pom.xml
+++ b/config-proxy/pom.xml
@@ -74,6 +74,12 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>testutil</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java
index 3eafdf8f2b4..a3dd8d27c0f 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java
@@ -1,27 +1,17 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy.filedistribution;
 
-import com.yahoo.io.IOUtils;
-import java.util.logging.Level;
 import com.yahoo.vespa.filedistribution.FileDownloader;
-
+import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup;
 import java.io.File;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Clock;
 import java.time.Duration;
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-import static java.nio.file.Files.readAttributes;
 
 /**
- * Deletes cached file references and url downloads that have not been used for some time
+ * Deletes file references and url downloads on disk that have not been used for some time
  *
  * @author hmusum
  */
@@ -31,20 +21,29 @@ class CachedFilesMaintainer implements Runnable {
 
     private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir;
     private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory;
-    private static final Duration defaultDurationToKeepFiles = Duration.ofDays(30);
+    private static final Duration defaultDurationToKeepFiles = Duration.ofDays(20);
+    private static final int defaultKeepCount = 20;
 
     private final File urlDownloadDir;
     private final File fileReferencesDownloadDir;
     private final Duration durationToKeepFiles;
+    private final FileDistributionCleanup cleanup;
+    private final int keepCount;  // keep this many files no matter how old they are or when they were last accessed
 
     CachedFilesMaintainer() {
-        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles);
+        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles, Clock.systemUTC(), defaultKeepCount);
     }
 
-    CachedFilesMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) {
+    CachedFilesMaintainer(File fileReferencesDownloadDir,
+                          File urlDownloadDir,
+                          Duration durationToKeepFiles,
+                          Clock clock,
+                          int keepCount) {
         this.fileReferencesDownloadDir = fileReferencesDownloadDir;
         this.urlDownloadDir = urlDownloadDir;
         this.durationToKeepFiles = durationToKeepFiles;
+        this.cleanup = new FileDistributionCleanup(clock);
+        this.keepCount = keepCount;
     }
 
     @Override
@@ -58,35 +57,7 @@ class CachedFilesMaintainer implements Runnable {
     }
 
     private void deleteUnusedFiles(File directory) {
-        Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles);
-        Set<String> filesOnDisk = new HashSet<>();
-        File[] files = directory.listFiles();
-        if (files != null)
-            filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet()));
-        log.log(Level.FINE, () -> "Files on disk (in " + directory + "): " + filesOnDisk);
-
-        Set<String> filesToDelete = filesOnDisk
-                .stream()
-                .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant))
-                .collect(Collectors.toSet());
-        if (filesToDelete.size() > 0) {
-            log.log(Level.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete);
-            filesToDelete.forEach(fileReference -> {
-                File file = new File(directory, fileReference);
-                if (!IOUtils.recursiveDeleteDir(file))
-                    log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
-            });
-        }
-    }
-
-    private boolean isFileLastModifiedBefore(File fileReference, Instant instant) {
-        BasicFileAttributes fileAttributes;
-        try {
-            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
-            return fileAttributes.lastModifiedTime().toInstant().isBefore(instant);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+        cleanup.deleteUnusedFileReferences(directory, durationToKeepFiles, keepCount, Set.of());
     }
 
 }
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java
index a491a7b4fc4..835982be44c 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java
@@ -2,14 +2,17 @@
 package com.yahoo.vespa.config.proxy.filedistribution;
 
 import com.yahoo.io.IOUtils;
+import com.yahoo.test.ManualClock;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
-
 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileTime;
 import java.time.Duration;
-import java.time.Instant;
+import java.util.stream.IntStream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -19,9 +22,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
  */
 public class CachedFilesMaintainerTest {
 
+    private static final int numberToAlwaysKeep = 2;
+
     private File cachedFileReferences;
     private File cachedDownloads;
     private CachedFilesMaintainer cachedFilesMaintainer;
+    private final ManualClock clock = new ManualClock();
 
     @TempDir
     public File tempFolder;
@@ -30,28 +36,28 @@ public class CachedFilesMaintainerTest {
     public void setup() throws IOException {
         cachedFileReferences = newFolder(tempFolder, "cachedFileReferences");
         cachedDownloads = newFolder(tempFolder, "cachedDownloads");
-        cachedFilesMaintainer = new CachedFilesMaintainer(cachedFileReferences, cachedDownloads, Duration.ofMinutes(1));
+        cachedFilesMaintainer = new CachedFilesMaintainer(cachedFileReferences,
+                                                          cachedDownloads,
+                                                          Duration.ofMinutes(2),
+                                                          clock,
+                                                          numberToAlwaysKeep);
     }
 
     @Test
-    void require_old_files_to_be_deleted() throws IOException {
+    void require_old_files_to_be_deleted() {
         runMaintainerAndAssertFiles(0, 0);
 
-        File fileReference = writeFile(cachedFileReferences, "fileReference");
-        File download = writeFile(cachedDownloads, "download");
-        runMaintainerAndAssertFiles(1, 1);
+        clock.advance(Duration.ofSeconds(55));
+        // Create file references and downloads
+        createFiles();
 
-        updateLastModifiedTimeStamp(fileReference, Instant.now().minus(Duration.ofMinutes(10)));
-        runMaintainerAndAssertFiles(0, 1);
+        runMaintainerAndAssertFiles(4, 4);
 
-        updateLastModifiedTimeStamp(download, Instant.now().minus(Duration.ofMinutes(10)));
-        runMaintainerAndAssertFiles(0, 0);
-    }
+        clock.advance(Duration.ofMinutes(1));
+        runMaintainerAndAssertFiles(3, 3);
 
-    private void updateLastModifiedTimeStamp(File file, Instant instant) {
-        if (!file.setLastModified(instant.toEpochMilli())) {
-            throw new RuntimeException("Could not set last modified timestamp for '" + file.getAbsolutePath() + "'");
-        }
+        clock.advance(Duration.ofMinutes(100));
+        runMaintainerAndAssertFiles(numberToAlwaysKeep, numberToAlwaysKeep);
     }
 
     private void runMaintainerAndAssertFiles(int fileReferenceCount, int downloadCount) {
@@ -65,10 +71,10 @@ public class CachedFilesMaintainerTest {
         assertEquals(downloadCount, downloads.length);
     }
 
-    private File writeFile(File directory, String filename) throws IOException {
+    private void writeFileAndSetLastAccessedTime(File directory, String filename) throws IOException {
         File file = new File(directory, filename);
         IOUtils.writeFile(file, filename, false);
-        return file;
+        Files.setAttribute(file.toPath(), "lastAccessTime", FileTime.from(clock.instant()));
     }
 
     private static File newFolder(File root, String... subDirs) throws IOException {
@@ -80,4 +86,16 @@ public class CachedFilesMaintainerTest {
         return result;
     }
 
+    private void createFiles() {
+        IntStream.of(0,1,2,3).forEach(i -> {
+            try {
+                writeFileAndSetLastAccessedTime(cachedFileReferences, "fileReference" + i);
+                writeFileAndSetLastAccessedTime(cachedDownloads, "download" + i);
+                clock.advance(Duration.ofMinutes(1));
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        });
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 2a15f724b29..f832c504526 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server;
 import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL;
 import ai.vespa.http.HttpURL.Query;
-import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
+import com.yahoo.component.annotation.Inject;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
@@ -83,20 +83,18 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.stats.LockStats;
 import com.yahoo.vespa.curator.stats.ThreadLockStats;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.nio.file.Files;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -116,11 +114,9 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVE
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse;
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk;
-import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk;
 import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT;
 import static com.yahoo.vespa.curator.Curator.CompletionWaiter;
 import static com.yahoo.yolean.Exceptions.uncheck;
-import static java.nio.file.Files.readAttributes;
 
 /**
  * The API for managing applications.
@@ -590,30 +586,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return fileDistributionStatus.status(getApplication(applicationId), timeout);
     }
 
-    public List<String> deleteUnusedFileDistributionReferences(File fileReferencesPath,
-                                                               Duration keepFileReferencesDuration,
-                                                               int numberToAlwaysKeep) {
-        log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration);
-        if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory");
-
-        Set<String> fileReferencesInUse = getFileReferencesInUse();
-        log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse);
-
-        List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration);
-        // Do not delete the newest ones
-        List<String> fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - numberToAlwaysKeep));
-        if (fileReferencesToDelete.size() > 0) {
-            log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete);
-            fileReferencesToDelete.forEach(fileReference -> {
-                File file = new File(fileReferencesPath, fileReference);
-                if ( ! IOUtils.recursiveDeleteDir(file))
-                    log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
-            });
-        }
-        return fileReferencesToDelete;
+    public void deleteUnusedFileDistributionReferences(File fileReferencesPath, Duration keepFileReferencesDuration) {
+        new FileDistributionCleanup(clock).deleteUnusedFileReferences(fileReferencesPath, keepFileReferencesDuration, getFileReferencesInUse());
     }
 
-    private Set<String> getFileReferencesInUse() {
+    Set<String> getFileReferencesInUse() {
         Set<String> fileReferencesInUse = new HashSet<>();
         for (var applicationId : listApplications()) {
             Application app = getApplication(applicationId);
@@ -624,18 +601,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return fileReferencesInUse;
     }
 
-    private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) {
-        Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath);
-        log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
-        Instant instant = clock.instant().minus(keepFileReferences);
-        return fileReferencesOnDisk
-                .stream()
-                .filter(fileReference -> ! fileReferencesInUse.contains(fileReference))
-                .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant))
-                .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath, a))))
-                .collect(Collectors.toList());
-    }
-
     public Set<FileReference> getFileReferences(ApplicationId applicationId) {
         return getOptionalApplication(applicationId).map(app -> app.getModel().fileReferences()).orElse(Set.of());
     }
@@ -686,20 +651,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                 .collect(Collectors.toList());
     }
 
-    private boolean isLastFileAccessBefore(File fileReference, Instant instant) {
-        return lastAccessed(fileReference).isBefore(instant);
-    }
-
-    private Instant lastAccessed(File fileReference) {
-        BasicFileAttributes fileAttributes;
-        try {
-            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
-            return fileAttributes.lastAccessTime().toInstant();
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     public Optional<String> getApplicationPackageReference(ApplicationId applicationId) {
         Optional<String> applicationPackage = Optional.empty();
         Optional<Session> session = getActiveSession(applicationId);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java
index a1ddad7bfd4..c1de9b4e5f6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java
@@ -5,12 +5,9 @@ import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.FileReference;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.config.server.ConfigServerSpec;
-
+import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup;
 import java.io.File;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -21,17 +18,6 @@ import java.util.stream.Collectors;
  */
 public class FileDistributionUtil {
 
-    /**
-     * Returns all files in the given directory, non-recursive.
-     */
-    public static Set<String> getFileReferencesOnDisk(File directory) {
-        Set<String> fileReferencesOnDisk = new HashSet<>();
-        File[] filesOnDisk = directory.listFiles();
-        if (filesOnDisk != null)
-            fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
-        return fileReferencesOnDisk;
-    }
-
     public static List<String> getOtherConfigServersInCluster(ConfigserverConfig configserverConfig) {
         return ConfigServerSpec.fromConfig(configserverConfig)
                                .stream()
@@ -41,7 +27,8 @@ public class FileDistributionUtil {
     }
 
     public static boolean fileReferenceExistsOnDisk(File downloadDirectory, FileReference applicationPackageReference) {
-        return getFileReferencesOnDisk(downloadDirectory).contains(applicationPackageReference.value());
+        return FileDistributionCleanup.getFileReferencesOnDisk(downloadDirectory.toPath())
+                                      .anyMatch(fileReference -> fileReference.equals(applicationPackageReference.value()));
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index f6aee416c9c..5542d24253b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -20,8 +20,6 @@ import java.time.Duration;
  */
 public class FileDistributionMaintainer extends ConfigServerMaintainer {
 
-    private static final int numberToAlwaysKeep = 20;
-
     private final ApplicationRepository applicationRepository;
     private final File fileReferencesDir;
     private final Duration maxUnusedFileReferenceAge;
@@ -39,7 +37,7 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer {
 
     @Override
     protected double maintain() {
-        applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge, numberToAlwaysKeep);
+        applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge);
         return 1.0;
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 99487230c5d..4f7be104b9c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
 import com.yahoo.vespa.config.util.ConfigUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.model.VespaModelFactory;
 import org.junit.Before;
@@ -293,9 +294,10 @@ public class ApplicationRepositoryTest {
         PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build();
         deployApp(new File("src/test/apps/app"), prepareParams);
 
-        List<String> toBeDeleted = applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir,
-                                                                                                keepFileReferencesDuration,
-                                                                                                2);
+        List<String> toBeDeleted = new FileDistributionCleanup(clock).deleteUnusedFileReferences(fileReferencesDir,
+                                                                                                 keepFileReferencesDuration,
+                                                                                                 2,
+                                                                                                 applicationRepository.getFileReferencesInUse());
         Collections.sort(toBeDeleted);
         assertEquals(List.of("bar0", "foo"), toBeDeleted);
         // bar0 and foo are the only ones that will be deleted (keeps 2 newest no matter how old they are)
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java
new file mode 100644
index 00000000000..9c04e7253bb
--- /dev/null
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/maintenance/FileDistributionCleanup.java
@@ -0,0 +1,107 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.filedistribution.maintenance;
+
+import com.yahoo.io.IOUtils;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Stream;
+
+import static java.nio.file.Files.readAttributes;
+
+/**
+ * Removes file references not used since a configured time, but always keeps a certain number of file references
+ * even when they are unused (unused is based on last access time for the file).
+ *
+ * @author hmusum
+ */
+public class FileDistributionCleanup {
+
+    private static final Logger log = Logger.getLogger(FileDistributionCleanup.class.getName());
+    private static final int numberToAlwaysKeep = 20;
+
+    private final Clock clock;
+
+    public FileDistributionCleanup(Clock clock) {
+        this.clock = clock;
+    }
+
+    public List<String> deleteUnusedFileReferences(File fileReferencesPath,
+                                                   Duration keepFileReferencesDuration,
+                                                   Set<String> fileReferencesInUse) {
+        return deleteUnusedFileReferences(fileReferencesPath,
+                                          keepFileReferencesDuration,
+                                          numberToAlwaysKeep,
+                                          fileReferencesInUse);
+    }
+
+    public List<String> deleteUnusedFileReferences(File fileReferencesDir,
+                                                   Duration keepFileReferencesDuration,
+                                                   int numberToAlwaysKeep,
+                                                   Set<String> fileReferencesInUse) {
+        if (!fileReferencesDir.isDirectory()) throw new RuntimeException(fileReferencesDir + " is not a directory");
+
+        log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration +
+                ", file references in use : " + fileReferencesInUse);
+        List<String> fileReferencesDeleted = new ArrayList<>();
+        Path fileReferencesPath = fileReferencesDir.toPath();
+        try (Stream<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration)) {
+            final AtomicInteger i = new AtomicInteger(0);
+            candidates.forEach(fileReference -> {
+                // Do not delete the newest ones
+                if (i.incrementAndGet() > numberToAlwaysKeep) {
+                    fileReferencesDeleted.add(fileReference);
+                    File file = new File(fileReferencesDir, fileReference);
+                    if (!IOUtils.recursiveDeleteDir(file))
+                        log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
+                }
+            });
+        }
+        return fileReferencesDeleted;
+    }
+
+    // Sorted, newest first
+    private Stream<String> sortedUnusedFileReferences(Path fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) {
+        Instant instant = clock.instant().minus(keepFileReferences);
+        return getFileReferencesOnDisk(fileReferencesPath)
+                .filter(fileReference -> !fileReferencesInUse.contains(fileReference))
+                .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath.toFile(), fileReference), instant))
+                .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath.toFile(), (String) a))).reversed());
+    }
+
+    private boolean isLastFileAccessBefore(File fileReference, Instant instant) {
+        return lastAccessed(fileReference).isBefore(instant);
+    }
+
+    private Instant lastAccessed(File fileReference) {
+        BasicFileAttributes fileAttributes;
+        try {
+            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
+            return fileAttributes.lastAccessTime().toInstant();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static Stream<String> getFileReferencesOnDisk(Path directory) {
+        try {
+            return Files.list(directory).map(path -> path.toFile().getName());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}