summaryrefslogtreecommitdiffstats
path: root/config-proxy
diff options
context:
space:
mode:
authorHarald Musum <musum@yahooinc.com>2023-01-12 12:18:11 +0100
committerHarald Musum <musum@yahooinc.com>2023-01-12 12:18:11 +0100
commit4527658be765161c203e0364daf1b0b46973841d (patch)
treea816ef2e5e53f840e4af0379b06f769df49538ea /config-proxy
parent375bc016d96df50a453595415f9251fefc8b0276 (diff)
Refactor, no functional changes
Diffstat (limited to 'config-proxy')
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java31
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java21
2 files changed, 30 insertions, 22 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
index 556677c2195..8f7aae17e84 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java
@@ -1,7 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.proxy.filedistribution;
-import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.subscription.ConfigSourceSet;
import com.yahoo.jrt.Supervisor;
import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool;
@@ -9,9 +8,6 @@ import com.yahoo.vespa.filedistribution.FileDownloader;
import java.time.Duration;
import java.util.Arrays;
import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
@@ -23,42 +19,37 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType
*/
public class FileDistributionAndUrlDownload {
- private static final Duration delay = Duration.ofMinutes(1);
-
private final FileDistributionRpcServer fileDistributionRpcServer;
private final UrlDownloadRpcServer urlDownloadRpcServer;
- private final ScheduledExecutorService cleanupExecutor =
- new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"));
+ private final FileReferencesAndDownloadsMaintainer maintainer;
public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) {
fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source));
urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor);
- cleanupExecutor.scheduleAtFixedRate(new FileReferencesAndDownloadsMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS);
+ maintainer = new FileReferencesAndDownloadsMaintainer();
}
public void close() {
fileDistributionRpcServer.close();
urlDownloadRpcServer.close();
- cleanupExecutor.shutdownNow();
- try {
- if ( ! cleanupExecutor.awaitTermination(10, TimeUnit.SECONDS))
- throw new RuntimeException("Unable to shutdown " + cleanupExecutor + " before timeout");
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
+ maintainer.close();
}
private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) {
+ return new FileDownloader(new FileDistributionConnectionPool(source, supervisor),
+ supervisor,
+ Duration.ofMinutes(5),
+ acceptedCompressionTypes());
+ }
+
+ private Set<CompressionType> acceptedCompressionTypes() {
Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip);
String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES");
if (env != null && ! env.isEmpty()) {
String[] types = env.split(",");
acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet());
}
- return new FileDownloader(new FileDistributionConnectionPool(source, supervisor),
- supervisor,
- Duration.ofMinutes(5),
- acceptedCompressionTypes);
+ return acceptedCompressionTypes;
}
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
index 210c2c19561..85d5a7e4af9 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.proxy.filedistribution;
+import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.io.IOUtils;
import com.yahoo.vespa.filedistribution.FileDownloader;
import java.io.File;
@@ -12,6 +13,9 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -25,12 +29,14 @@ import static java.nio.file.Files.readAttributes;
*/
class FileReferencesAndDownloadsMaintainer implements Runnable {
- private final static Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName());
-
+ private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName());
private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir;
private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory;
private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14);
+ private static final Duration interval = Duration.ofMinutes(1);
+ private final ScheduledExecutorService executor =
+ new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"));
private final File urlDownloadDir;
private final File fileReferencesDownloadDir;
private final Duration durationToKeepFiles;
@@ -43,6 +49,7 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
this.fileReferencesDownloadDir = fileReferencesDownloadDir;
this.urlDownloadDir = urlDownloadDir;
this.durationToKeepFiles = durationToKeepFiles;
+ executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS);
}
@Override
@@ -55,6 +62,16 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
}
}
+ public void close() {
+ executor.shutdownNow();
+ try {
+ if ( ! executor.awaitTermination(10, TimeUnit.SECONDS))
+ throw new RuntimeException("Unable to shutdown " + executor + " before timeout");
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private void deleteUnusedFiles(File directory) {
Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles);
Set<String> filesOnDisk = new HashSet<>();