Merge pull request #25530 from vespa-engine/hmusum/keep-files-longer

Support environment variable for deciding lifetime of file references

3 files changed, 46 insertions, 30 deletions

diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
index edd16c3d23d..8f7aae17e84 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy.filedistribution;
 
-import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.jrt.Supervisor;
 import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool;
@@ -9,9 +8,6 @@ import com.yahoo.vespa.filedistribution.FileDownloader;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
@@ -23,42 +19,37 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType
  */
 public class FileDistributionAndUrlDownload {
 
-    private static final Duration delay = Duration.ofMinutes(1);
-
     private final FileDistributionRpcServer fileDistributionRpcServer;
     private final UrlDownloadRpcServer urlDownloadRpcServer;
-    private final ScheduledExecutorService cleanupExecutor =
-            new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"));
+    private final FileReferencesAndDownloadsMaintainer maintainer;
 
     public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) {
         fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source));
         urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor);
-        cleanupExecutor.scheduleAtFixedRate(new CachedFilesMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS);
+        maintainer = new FileReferencesAndDownloadsMaintainer();
     }
 
     public void close() {
         fileDistributionRpcServer.close();
         urlDownloadRpcServer.close();
-        cleanupExecutor.shutdownNow();
-        try {
-            if ( ! cleanupExecutor.awaitTermination(10, TimeUnit.SECONDS))
-                throw new RuntimeException("Unable to shutdown " + cleanupExecutor + " before timeout");
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
+        maintainer.close();
     }
 
     private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) {
+        return new FileDownloader(new FileDistributionConnectionPool(source, supervisor),
+                                  supervisor,
+                                  Duration.ofMinutes(5),
+                                  acceptedCompressionTypes());
+    }
+
+    private Set<CompressionType> acceptedCompressionTypes() {
         Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip);
         String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES");
         if (env != null && ! env.isEmpty()) {
             String[] types = env.split(",");
             acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet());
         }
-        return new FileDownloader(new FileDistributionConnectionPool(source, supervisor),
-                                  supervisor,
-                                  Duration.ofMinutes(5),
-                                  acceptedCompressionTypes);
+        return acceptedCompressionTypes;
     }
 
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
index 4ca5eb4ee90..5d5775275c3 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy.filedistribution;
 
+import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.io.IOUtils;
 import com.yahoo.vespa.filedistribution.FileDownloader;
 import java.io.File;
@@ -12,6 +13,9 @@ import java.time.Instant;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -19,30 +23,33 @@ import java.util.stream.Collectors;
 import static java.nio.file.Files.readAttributes;
 
 /**
- * Deletes cached file references and url downloads that have not been used for some time
+ * Deletes file references and url downloads that have not been used for some time
  *
  * @author hmusum
  */
-class CachedFilesMaintainer implements Runnable {
-
-    private final static Logger log = Logger.getLogger(CachedFilesMaintainer.class.getName());
+class FileReferencesAndDownloadsMaintainer implements Runnable {
 
+    private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName());
     private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir;
     private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory;
     private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14);
+    private static final Duration interval = Duration.ofMinutes(1);
 
+    private final ScheduledExecutorService executor =
+            new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"));
     private final File urlDownloadDir;
     private final File fileReferencesDownloadDir;
     private final Duration durationToKeepFiles;
 
-    CachedFilesMaintainer() {
-        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles);
+    FileReferencesAndDownloadsMaintainer() {
+        this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration());
     }
 
-    CachedFilesMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) {
+    FileReferencesAndDownloadsMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) {
         this.fileReferencesDownloadDir = fileReferencesDownloadDir;
         this.urlDownloadDir = urlDownloadDir;
         this.durationToKeepFiles = durationToKeepFiles;
+        executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS);
     }
 
     @Override
@@ -55,6 +62,16 @@ class CachedFilesMaintainer implements Runnable {
         }
     }
 
+    public void close() {
+        executor.shutdownNow();
+        try {
+            if ( ! executor.awaitTermination(10, TimeUnit.SECONDS))
+                throw new RuntimeException("Unable to shutdown " + executor + " before timeout");
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private void deleteUnusedFiles(File directory) {
         Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles);
         Set<String> filesOnDisk = new HashSet<>();
@@ -87,4 +104,12 @@ class CachedFilesMaintainer implements Runnable {
         }
     }
 
+    private static Duration keepFileReferencesDuration() {
+        String env = System.getenv("VESPA_KEEP_FILE_REFERENCES_DAYS");
+        if (env != null && !env.isEmpty())
+            return Duration.ofDays(Integer.parseInt(env));
+        else
+            return defaultDurationToKeepFiles;
+    }
+
 }
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java
index a491a7b4fc4..fad021c0119 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java
@@ -17,11 +17,11 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 /**
  * @author hmusum
  */
-public class CachedFilesMaintainerTest {
+public class FileReferencesAndDownloadsMaintainerTest {
 
     private File cachedFileReferences;
     private File cachedDownloads;
-    private CachedFilesMaintainer cachedFilesMaintainer;
+    private FileReferencesAndDownloadsMaintainer cachedFilesMaintainer;
 
     @TempDir
     public File tempFolder;
@@ -30,7 +30,7 @@ public class CachedFilesMaintainerTest {
     public void setup() throws IOException {
         cachedFileReferences = newFolder(tempFolder, "cachedFileReferences");
         cachedDownloads = newFolder(tempFolder, "cachedDownloads");
-        cachedFilesMaintainer = new CachedFilesMaintainer(cachedFileReferences, cachedDownloads, Duration.ofMinutes(1));
+        cachedFilesMaintainer = new FileReferencesAndDownloadsMaintainer(cachedFileReferences, cachedDownloads, Duration.ofMinutes(1));
     }
 
     @Test