diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-01-12 13:43:20 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-12 13:43:20 +0100 |
commit | 61acd5aa3a1bf9098c87fdc098c7982301f0910c (patch) | |
tree | 4052eac9e36fad8dd11a41e2a9316e7b5c4566d2 /config-proxy | |
parent | bf97fcd47792f15d15f5b25fc3373fc654a87baf (diff) | |
parent | 15e4f2e8cb4d09b7af6238e30cdd29a0bb981ef6 (diff) |
Merge pull request #25530 from vespa-engine/hmusum/keep-files-longer
Support environment variable for deciding lifetime of file references
Diffstat (limited to 'config-proxy')
-rw-r--r-- | config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java | 31 | ||||
-rw-r--r-- | config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java (renamed from config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java) | 39 | ||||
-rw-r--r-- | config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java (renamed from config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java) | 6 |
3 files changed, 46 insertions, 30 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index edd16c3d23d..8f7aae17e84 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; @@ -9,9 +8,6 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import java.time.Duration; import java.util.Arrays; import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; @@ -23,42 +19,37 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType */ public class FileDistributionAndUrlDownload { - private static final Duration delay = Duration.ofMinutes(1); - private final FileDistributionRpcServer fileDistributionRpcServer; private final UrlDownloadRpcServer urlDownloadRpcServer; - private final ScheduledExecutorService cleanupExecutor = - new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); + private final FileReferencesAndDownloadsMaintainer maintainer; public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source)); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); - cleanupExecutor.scheduleAtFixedRate(new CachedFilesMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS); + maintainer = new FileReferencesAndDownloadsMaintainer(); } public void close() { fileDistributionRpcServer.close(); urlDownloadRpcServer.close(); - cleanupExecutor.shutdownNow(); - try { - if ( ! cleanupExecutor.awaitTermination(10, TimeUnit.SECONDS)) - throw new RuntimeException("Unable to shutdown " + cleanupExecutor + " before timeout"); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + maintainer.close(); } private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { + return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), + supervisor, + Duration.ofMinutes(5), + acceptedCompressionTypes()); + } + + private Set<CompressionType> acceptedCompressionTypes() { Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); if (env != null && ! env.isEmpty()) { String[] types = env.split(","); acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet()); } - return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), - supervisor, - Duration.ofMinutes(5), - acceptedCompressionTypes); + return acceptedCompressionTypes; } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java index 4ca5eb4ee90..5d5775275c3 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.io.IOUtils; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; @@ -12,6 +13,9 @@ import java.time.Instant; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -19,30 +23,33 @@ import java.util.stream.Collectors; import static java.nio.file.Files.readAttributes; /** - * Deletes cached file references and url downloads that have not been used for some time + * Deletes file references and url downloads that have not been used for some time * * @author hmusum */ -class CachedFilesMaintainer implements Runnable { - - private final static Logger log = Logger.getLogger(CachedFilesMaintainer.class.getName()); +class FileReferencesAndDownloadsMaintainer implements Runnable { + private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName()); private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14); + private static final Duration interval = Duration.ofMinutes(1); + private final ScheduledExecutorService executor = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); private final File urlDownloadDir; private final File fileReferencesDownloadDir; private final Duration durationToKeepFiles; - CachedFilesMaintainer() { - this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles); + FileReferencesAndDownloadsMaintainer() { + this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration()); } - CachedFilesMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) { + FileReferencesAndDownloadsMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) { this.fileReferencesDownloadDir = fileReferencesDownloadDir; this.urlDownloadDir = urlDownloadDir; this.durationToKeepFiles = durationToKeepFiles; + executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); } @Override @@ -55,6 +62,16 @@ class CachedFilesMaintainer implements Runnable { } } + public void close() { + executor.shutdownNow(); + try { + if ( ! executor.awaitTermination(10, TimeUnit.SECONDS)) + throw new RuntimeException("Unable to shutdown " + executor + " before timeout"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void deleteUnusedFiles(File directory) { Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles); Set<String> filesOnDisk = new HashSet<>(); @@ -87,4 +104,12 @@ class CachedFilesMaintainer implements Runnable { } } + private static Duration keepFileReferencesDuration() { + String env = System.getenv("VESPA_KEEP_FILE_REFERENCES_DAYS"); + if (env != null && !env.isEmpty()) + return Duration.ofDays(Integer.parseInt(env)); + else + return defaultDurationToKeepFiles; + } + } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java index a491a7b4fc4..fad021c0119 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java @@ -17,11 +17,11 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; /** * @author hmusum */ -public class CachedFilesMaintainerTest { +public class FileReferencesAndDownloadsMaintainerTest { private File cachedFileReferences; private File cachedDownloads; - private CachedFilesMaintainer cachedFilesMaintainer; + private FileReferencesAndDownloadsMaintainer cachedFilesMaintainer; @TempDir public File tempFolder; @@ -30,7 +30,7 @@ public class CachedFilesMaintainerTest { public void setup() throws IOException { cachedFileReferences = newFolder(tempFolder, "cachedFileReferences"); cachedDownloads = newFolder(tempFolder, "cachedDownloads"); - cachedFilesMaintainer = new CachedFilesMaintainer(cachedFileReferences, cachedDownloads, Duration.ofMinutes(1)); + cachedFilesMaintainer = new FileReferencesAndDownloadsMaintainer(cachedFileReferences, cachedDownloads, Duration.ofMinutes(1)); } @Test |