diff options
author | Harald Musum <musum@yahooinc.com> | 2023-02-01 13:20:10 +0100 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-02-01 13:20:10 +0100 |
commit | 88af4ae0b13faba664f9d5d5c77d37a6b6f09498 (patch) | |
tree | 872662e5f4cc2ee580510f2869f191b685b67b23 /filedistribution/src/main/java | |
parent | 179daa38c12471ec9de4e48ec91865c8a336d8a8 (diff) |
Stop using file distribution feature flags, not needed anymore
Diffstat (limited to 'filedistribution/src/main/java')
2 files changed, 14 insertions, 18 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 63ae8faacfe..68f969df3af 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -10,7 +10,6 @@ import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,8 +18,6 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; - /** * Handles downloads of files (file references only for now) * @@ -40,20 +37,19 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries, - Set<CompressionType> acceptedCompressionTypes) { + Duration sleepBetweenRetries) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -64,8 +60,7 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory, - acceptedCompressionTypes); + downloadDirectory); if (forceDownload) log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)"); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index f14de0b9807..0966561ce5d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -23,6 +23,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; /** * Downloads file reference from config server and keeps track of files being downloaded @@ -31,7 +34,8 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType */ public class FileReferenceDownloader { - private final static Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Set<CompressionType> defaultAcceptedCompressionTypes = Set.of(gzip, lz4, zstd); private final ExecutorService downloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), @@ -42,14 +46,12 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; - private final Set<CompressionType> acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory, - Set<CompressionType> acceptedCompressionTypes) { + File downloadDirectory) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -57,7 +59,6 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); - this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -139,8 +140,8 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); - String[] temp = new String[acceptedCompressionTypes.size()]; - acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + String[] temp = new String[defaultAcceptedCompressionTypes.size()]; + defaultAcceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); request.parameters().add(new StringArray(temp)); return request; } |