diff options
author | Harald Musum <musum@yahooinc.com> | 2023-01-12 12:18:11 +0100 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-01-12 12:18:11 +0100 |
commit | 4527658be765161c203e0364daf1b0b46973841d (patch) | |
tree | a816ef2e5e53f840e4af0379b06f769df49538ea /config-proxy | |
parent | 375bc016d96df50a453595415f9251fefc8b0276 (diff) |
Refactor, no functional changes
Diffstat (limited to 'config-proxy')
2 files changed, 30 insertions, 22 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 556677c2195..8f7aae17e84 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; @@ -9,9 +8,6 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import java.time.Duration; import java.util.Arrays; import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; @@ -23,42 +19,37 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType */ public class FileDistributionAndUrlDownload { - private static final Duration delay = Duration.ofMinutes(1); - private final FileDistributionRpcServer fileDistributionRpcServer; private final UrlDownloadRpcServer urlDownloadRpcServer; - private final ScheduledExecutorService cleanupExecutor = - new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); + private final FileReferencesAndDownloadsMaintainer maintainer; public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source)); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); - cleanupExecutor.scheduleAtFixedRate(new FileReferencesAndDownloadsMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS); + maintainer = new FileReferencesAndDownloadsMaintainer(); } public void close() { fileDistributionRpcServer.close(); urlDownloadRpcServer.close(); - cleanupExecutor.shutdownNow(); - try { - if ( ! cleanupExecutor.awaitTermination(10, TimeUnit.SECONDS)) - throw new RuntimeException("Unable to shutdown " + cleanupExecutor + " before timeout"); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + maintainer.close(); } private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { + return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), + supervisor, + Duration.ofMinutes(5), + acceptedCompressionTypes()); + } + + private Set<CompressionType> acceptedCompressionTypes() { Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); if (env != null && ! env.isEmpty()) { String[] types = env.split(","); acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet()); } - return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), - supervisor, - Duration.ofMinutes(5), - acceptedCompressionTypes); + return acceptedCompressionTypes; } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java index 210c2c19561..85d5a7e4af9 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.io.IOUtils; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; @@ -12,6 +13,9 @@ import java.time.Instant; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -25,12 +29,14 @@ import static java.nio.file.Files.readAttributes; */ class FileReferencesAndDownloadsMaintainer implements Runnable { - private final static Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName()); - + private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName()); private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14); + private static final Duration interval = Duration.ofMinutes(1); + private final ScheduledExecutorService executor = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); private final File urlDownloadDir; private final File fileReferencesDownloadDir; private final Duration durationToKeepFiles; @@ -43,6 +49,7 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { this.fileReferencesDownloadDir = fileReferencesDownloadDir; this.urlDownloadDir = urlDownloadDir; this.durationToKeepFiles = durationToKeepFiles; + executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); } @Override @@ -55,6 +62,16 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { } } + public void close() { + executor.shutdownNow(); + try { + if ( ! executor.awaitTermination(10, TimeUnit.SECONDS)) + throw new RuntimeException("Unable to shutdown " + executor + " before timeout"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void deleteUnusedFiles(File directory) { Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles); Set<String> filesOnDisk = new HashSet<>(); |