diff options
author | Harald Musum <musum@yahooinc.com> | 2022-07-06 10:33:57 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2022-07-06 10:33:57 +0200 |
commit | f7d93a0f3bade34a4a02cdbd8d3942422ed65b2f (patch) | |
tree | a82b30d72ca49a93bf81facd7943dab9ebe3ed8e /filedistribution/src/main | |
parent | 8dae227258dde84db5116922fbc616dc1d70d3a7 (diff) |
Wire in use of compression types and flags for file distribution
VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES is not set anywhere
yet, will be set through host-admin
Diffstat (limited to 'filedistribution/src/main')
3 files changed, 30 insertions, 11 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5941ed536a8..dc87ae2a0b4 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -6,11 +6,11 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; - import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,6 +19,8 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** * Handles downloads of files (file references only for now) * @@ -37,19 +39,20 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries) { + Duration sleepBetweenRetries, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -60,7 +63,8 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory); + downloadDirectory, + acceptedCompressionTypes); } public Optional<File> getFile(FileReferenceDownload fileReferenceDownload) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java index 8d6f428eaef..f3a8cf9299d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -18,7 +18,7 @@ public class FileReferenceDownload { private final boolean downloadFromOtherSourceIfNotFound; private final String client; - public FileReferenceDownload(FileReference fileReference, String client) { + public FileReferenceDownload(FileReference fileReference, String client) { this(fileReference, client, true); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 0267feb9ffc..7078c5aae6c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -5,14 +5,16 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; - import java.io.File; import java.time.Duration; import java.time.Instant; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -20,8 +22,10 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** - * Downloads file reference using rpc requests to config server and keeps track of files being downloaded + * Downloads file reference from config server and keeps track of files being downloaded * * @author hmusum */ @@ -38,12 +42,14 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; + private final Set<CompressionType> acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory) { + File downloadDirectory, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -51,6 +57,7 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); + this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -132,6 +139,9 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); + String[] temp = new String[acceptedCompressionTypes.size()]; + acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + request.parameters().add(new StringArray(temp)); return request; } @@ -160,4 +170,9 @@ public class FileReferenceDownloader { } } + private static Set<CompressionType> requireNonEmpty(Set<CompressionType> s) { + if (Objects.requireNonNull(s).isEmpty()) throw new IllegalArgumentException("set must be non-empty"); + return s; + } + } |