Remove unused file refrences based on last modified time

3 files changed, 18 insertions, 16 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index cf79005b1ab..cd61b44be1b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -597,14 +597,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     public List<String> deleteUnusedFileDistributionReferences(File fileReferencesPath,
                                                                Duration keepFileReferencesDuration,
                                                                int numberToAlwaysKeep) {
-        log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration);
         if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory");
 
         Set<String> fileReferencesInUse = getFileReferencesInUse();
         log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse);
+        Instant instant = clock.instant().minus(keepFileReferencesDuration);
+        log.log(Level.FINE, () -> "Remove unused file references last modified before " + instant +
+                " (but keep " + numberToAlwaysKeep + " of those)");
 
-        List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration);
-        // Do not delete the newest ones
+        List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, instant);
         List<String> fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - numberToAlwaysKeep));
         if (fileReferencesToDelete.size() > 0) {
             log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete);
@@ -628,15 +629,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return fileReferencesInUse;
     }
 
-    private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) {
+    private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Instant instant) {
         Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath);
         log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
-        Instant instant = clock.instant().minus(keepFileReferences);
         return fileReferencesOnDisk
                 .stream()
                 .filter(fileReference -> ! fileReferencesInUse.contains(fileReference))
-                .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant))
-                .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath, a))))
+                .filter(fileReference -> isLastModifiedBefore(new File(fileReferencesPath, fileReference), instant))
+                .sorted(Comparator.comparing(a -> lastModified(new File(fileReferencesPath, a))))
                 .collect(Collectors.toList());
     }
 
@@ -690,15 +690,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                 .collect(Collectors.toList());
     }
 
-    private boolean isLastFileAccessBefore(File fileReference, Instant instant) {
-        return lastAccessed(fileReference).isBefore(instant);
+    private boolean isLastModifiedBefore(File fileReference, Instant instant) {
+        return lastModified(fileReference).isBefore(instant);
     }
 
-    private Instant lastAccessed(File fileReference) {
+    private Instant lastModified(File fileReference) {
         BasicFileAttributes fileAttributes;
         try {
             fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
-            return fileAttributes.lastAccessTime().toInstant();
+            return fileAttributes.lastModifiedTime().toInstant();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index f6aee416c9c..0588a126b68 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -20,7 +20,7 @@ import java.time.Duration;
  */
 public class FileDistributionMaintainer extends ConfigServerMaintainer {
 
-    private static final int numberToAlwaysKeep = 20;
+    private static final int numberToAlwaysKeep = 10; // TODO: Reduce to 0 / remove
 
     private final ApplicationRepository applicationRepository;
     private final File fileReferencesDir;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index b185bb5915d..e93b80076e8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -61,7 +61,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.attribute.FileTime;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -298,10 +297,13 @@ public class ApplicationRepositoryTest {
                                                                                                 keepFileReferencesDuration,
                                                                                                 2);
         Collections.sort(toBeDeleted);
-        assertEquals(List.of("bar0", "foo"), toBeDeleted);
-        // bar0 and foo are the only ones that will be deleted (keeps 2 newest no matter how old they are)
+        List<String> expected = new ArrayList<>(List.of("foo", "bar0", "bar1"));
+        Collections.sort(expected);
+        assertEquals(expected, toBeDeleted);
+        // foo, bar0 and bar1 will be deleted, 2 newest ones (bar2 and baz) will be kept no matter how old they are
         assertFalse(filereferenceDirOldest.exists());
         assertFalse(new File(fileReferencesDir, "bar0").exists());
+        assertFalse(new File(fileReferencesDir, "bar1").exists());
         assertTrue(filereferenceDirNewest.exists());
     }
 
@@ -309,7 +311,7 @@ public class ApplicationRepositoryTest {
         assertTrue(filereferenceDir.mkdir());
         File file = new File(filereferenceDir, "bar");
         IOUtils.writeFile(file, Utf8.toBytes("test"));
-        Files.setAttribute(filereferenceDir.toPath(), "lastAccessTime", FileTime.from(clock.instant()));
+        file.setLastModified(clock.instant().toEpochMilli());
         return filereferenceDir;
     }