1 files changed, 91 insertions, 33 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
index 627a15aab65..eab1368a2a1 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
@@ -3,24 +3,29 @@ package com.yahoo.vespa.config.proxy.filedistribution;
 
 import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.io.IOUtils;
+import com.yahoo.vespa.config.util.ConfigUtils;
 import com.yahoo.vespa.filedistribution.FileDownloader;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 import static java.nio.file.Files.readAttributes;
+import static java.util.logging.Level.INFO;
 
 /**
  * Deletes file references and url downloads that have not been used for some time.
@@ -35,27 +40,43 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
     private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir;
     private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory;
     private static final Duration defaultDurationToKeepFiles = Duration.ofDays(30);
+    private static final int defaultOutdatedFilesToKeep = 20;
     private static final Duration interval = Duration.ofMinutes(1);
 
-    private final ScheduledExecutorService executor =
-            new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"));
+    private final Optional<ScheduledExecutorService> executor;
     private final File urlDownloadDir;
     private final File fileReferencesDownloadDir;
     private final Duration durationToKeepFiles;
+    private final int outDatedFilesToKeep;
 
     FileReferencesAndDownloadsMaintainer() {
-        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration());
+        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration(),
+             outDatedFilesToKeep(), configServers());
     }
 
-    FileReferencesAndDownloadsMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) {
+    FileReferencesAndDownloadsMaintainer(File fileReferencesDownloadDir,
+                                         File urlDownloadDir,
+                                         Duration durationToKeepFiles,
+                                         int outdatedFilesToKeep,
+                                         List<String> configServers) {
         this.fileReferencesDownloadDir = fileReferencesDownloadDir;
         this.urlDownloadDir = urlDownloadDir;
         this.durationToKeepFiles = durationToKeepFiles;
-        executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS);
+        this.outDatedFilesToKeep = outdatedFilesToKeep;
+        // Do not run on config servers
+        if (configServers.contains(ConfigUtils.getCanonicalHostName())) {
+            log.log(INFO, "Not running maintainer, since this is on a config server host");
+            executor = Optional.empty();
+        } else {
+            executor = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")));
+            executor.get().scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS);
+        }
     }
 
     @Override
     public void run() {
+        if (executor.isEmpty()) return;
+
         try {
             deleteUnusedFiles(fileReferencesDownloadDir);
             deleteUnusedFiles(urlDownloadDir);
@@ -65,42 +86,62 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
     }
 
     public void close() {
-        executor.shutdownNow();
-        try {
-            if ( ! executor.awaitTermination(10, TimeUnit.SECONDS))
-                throw new RuntimeException("Unable to shutdown " + executor + " before timeout");
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
+        executor.ifPresent(ex -> {
+            ex.shutdownNow();
+            try {
+                if (! ex.awaitTermination(10, TimeUnit.SECONDS))
+                    throw new RuntimeException("Unable to shutdown " + executor + " before timeout");
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
     }
 
     private void deleteUnusedFiles(File directory) {
-        Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles);
-        Set<String> filesOnDisk = new HashSet<>();
+
         File[] files = directory.listFiles();
-        if (files != null)
-            filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet()));
-        log.log(Level.FINE, () -> "Files on disk (in " + directory + "): " + filesOnDisk);
+        if (files == null) return;
+
+        List<File> filesToDelete = filesThatCanBeDeleted(files);
+        filesToDelete.forEach(fileReference -> {
+            if (IOUtils.recursiveDeleteDir(fileReference))
+                log.log(Level.FINE, "Deleted " + fileReference.getAbsolutePath());
+            else
+                log.log(Level.WARNING, "Could not delete " + fileReference.getAbsolutePath());
+        });
+    }
 
-        Set<String> filesToDelete = filesOnDisk
+    private List<File> filesThatCanBeDeleted(File[] files) {
+        Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles);
+
+        Set<File> filesOnDisk = new HashSet<>(List.of(files));
+        log.log(Level.FINE, () -> "Files on disk: " + filesOnDisk);
+        int deleteCount = Math.max(0, filesOnDisk.size() - outDatedFilesToKeep);
+        var canBeDeleted = filesOnDisk
                 .stream()
-                .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant))
-                .collect(Collectors.toSet());
-        if (filesToDelete.size() > 0) {
-            log.log(Level.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete);
-            filesToDelete.forEach(fileReference -> {
-                File file = new File(directory, fileReference);
-                if (!IOUtils.recursiveDeleteDir(file))
-                    log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
-            });
-        }
+                .peek(file -> log.log(Level.FINE, () -> file + ":" + fileLastModifiedTime(file.toPath())))
+                .filter(fileReference -> isFileLastModifiedBefore(fileReference, deleteNotUsedSinceInstant))
+                .sorted(Comparator.comparing(fileReference -> fileLastModifiedTime(fileReference.toPath())))
+                .toList();
+
+        // Make sure we keep some files
+        canBeDeleted = canBeDeleted.subList(0, Math.min(canBeDeleted.size(), deleteCount));
+        if (canBeDeleted.size() > 0)
+            log.log(INFO, "Files that can be deleted (not accessed since " + deleteNotUsedSinceInstant +
+                    ", will also keep " + outDatedFilesToKeep +
+                    " no matter when last accessed): " + canBeDeleted);
+
+        return canBeDeleted;
     }
 
     private boolean isFileLastModifiedBefore(File fileReference, Instant instant) {
-        BasicFileAttributes fileAttributes;
+        return fileLastModifiedTime(fileReference.toPath()).isBefore(instant);
+    }
+
+    private static Instant fileLastModifiedTime(Path fileReference) {
         try {
-            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
-            return fileAttributes.lastModifiedTime().toInstant().isBefore(instant);
+            BasicFileAttributes fileAttributes = readAttributes(fileReference, BasicFileAttributes.class);
+            return fileAttributes.lastModifiedTime().toInstant();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -114,4 +155,21 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
             return defaultDurationToKeepFiles;
     }
 
+    private static int outDatedFilesToKeep() {
+        String env = System.getenv("VESPA_KEEP_FILE_REFERENCES_COUNT");
+        if (env != null && !env.isEmpty())
+            return Integer.parseInt(env);
+        else
+            return defaultOutdatedFilesToKeep;
+    }
+
+    private static List<String> configServers() {
+        String env = System.getenv("VESPA_CONFIGSERVERS");
+        if (env == null || env.isEmpty())
+            return List.of(ConfigUtils.getCanonicalHostName());
+        else {
+            return List.of(env.split(","));
+        }
+    }
+
 }