diff options
8 files changed, 141 insertions, 39 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 68570722117..edd16c3d23d 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -6,11 +6,15 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; - import java.time.Duration; +import java.util.Arrays; +import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; /** * Keeps track of file distribution and url download rpc servers. @@ -45,9 +49,16 @@ public class FileDistributionAndUrlDownload { } private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { + Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); + String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); + if (env != null && ! env.isEmpty()) { + String[] types = env.split(","); + acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet()); + } return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), supervisor, - Duration.ofMinutes(5)); + Duration.ofMinutes(5), + acceptedCompressionTypes); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index f0c34e83713..179d8923378 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -21,6 +21,8 @@ import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -28,6 +30,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -35,11 +38,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; public class FileServer { @@ -52,6 +55,7 @@ public class FileServer { private final FileDirectory root; private final ExecutorService executor; private final FileDownloader downloader; + private final List<CompressionType> compressionTypes; // compression types to use, in preferred order // TODO: Move to filedistribution module, so that it can be used by both clients and servers private enum FileApiErrorCodes { @@ -86,21 +90,24 @@ public class FileServer { @SuppressWarnings("WeakerAccess") // Created by dependency injection @Inject - public FileServer(ConfigserverConfig configserverConfig) { + public FileServer(ConfigserverConfig configserverConfig, FlagSource flagSource) { this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())), - createFileDownloader(getOtherConfigServersInCluster(configserverConfig))); + createFileDownloader(getOtherConfigServersInCluster(configserverConfig), + compressionTypes(Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value())), + compressionTypesAsList(Flags.FILE_DISTRIBUTION_COMPRESSION_TYPES_TO_SERVE.bindTo(flagSource).value())); } // For testing only public FileServer(File rootDir) { - this(rootDir, createFileDownloader(List.of())); + this(rootDir, createFileDownloader(List.of(), Set.of(gzip)), List.of(gzip)); } - public FileServer(File rootDir, FileDownloader fileDownloader) { + FileServer(File rootDir, FileDownloader fileDownloader, List<CompressionType> compressionTypes) { this.downloader = fileDownloader; this.root = new FileDirectory(rootDir); this.executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("file-server-")); + this.compressionTypes = compressionTypes; } boolean hasFile(String fileReference) { @@ -145,6 +152,7 @@ public class FileServer { if (file.isDirectory()) { Path tempFile = Files.createTempFile("filereferencedata", reference.value()); CompressionType compressionType = chooseCompressionType(acceptedCompressionTypes); + log.log(Level.FINE, () -> "accepted compression types=" + acceptedCompressionTypes + ", compression type to use=" + compressionType); File compressedFile = new FileReferenceCompressor(compressed, compressionType).compress(file.getParentFile(), tempFile.toFile()); return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile); } else { @@ -195,9 +203,14 @@ public class FileServer { return (fileExists ? FileApiErrorCodes.OK : FileApiErrorCodes.NOT_FOUND); } - // TODO: Use lz4 for testing only, add zstd when we have support for (de)compressing zstd input and output streams + /* Choose the first compression type (list is in preferred order) that matches an accepted compression type, or fail */ private CompressionType chooseCompressionType(Set<CompressionType> acceptedCompressionTypes) { - return acceptedCompressionTypes.contains(lz4) ? lz4 : gzip; + for (CompressionType compressionType : compressionTypes) { + if (acceptedCompressionTypes.contains(compressionType)) + return compressionType; + } + throw new RuntimeException("Could not find a compression type that can be used. Accepted compression types: " + + acceptedCompressionTypes + ", compression types server can use: " + compressionTypes); } boolean hasFileDownloadIfNeeded(FileReferenceDownload fileReferenceDownload) { @@ -228,14 +241,27 @@ public class FileServer { executor.shutdown(); } - private static FileDownloader createFileDownloader(List<String> configServers) { + private static FileDownloader createFileDownloader(List<String> configServers, Set<CompressionType> acceptedCompressionTypes) { Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true); return new FileDownloader(configServers.isEmpty() ? FileDownloader.emptyConnectionPool() : createConnectionPool(configServers, supervisor), supervisor, - timeout); + timeout, + acceptedCompressionTypes); + } + + private static LinkedHashSet<CompressionType> compressionTypes(List<String> compressionTypes) { + return compressionTypes.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private static List<CompressionType> compressionTypesAsList(List<String> compressionTypes) { + return compressionTypes.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toList()); } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index ae4b205c06e..12972e5c465 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -18,20 +18,23 @@ import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.flags.FlagSource; - +import com.yahoo.vespa.flags.Flags; import java.io.File; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; /** * Verifies that all active sessions has an application package on local disk. * If not, the package is downloaded with file distribution. This can happen e.g. - * if a configserver is down when the application is deployed. + * if a config server is down when the application is deployed. * * @author gjoranv */ @@ -53,7 +56,10 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { this.applicationRepository = applicationRepository; this.configserverConfig = applicationRepository.configserverConfig(); this.downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); - this.fileDownloader = createFileDownloader(configserverConfig, downloadDirectory, supervisor); + this.fileDownloader = createFileDownloader(configserverConfig, + downloadDirectory, + supervisor, + Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value()); } @Override @@ -94,14 +100,18 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { private static FileDownloader createFileDownloader(ConfigserverConfig configserverConfig, File downloadDirectory, - Supervisor supervisor) { + Supervisor supervisor, + List<String> flagValues) { List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(configserverConfig); ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = (otherConfigServersInCluster.isEmpty()) ? FileDownloader.emptyConnectionPool() : new FileDistributionConnectionPool(configSourceSet, supervisor); - return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300)); + Set<CompressionType> acceptedCompressionTypes = flagValues.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toSet()); + return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300), acceptedCompressionTypes); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 9498db1d1e0..39219471bb1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -8,8 +8,10 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -39,7 +41,7 @@ public class FileServerTest { @Before public void setup() throws IOException { File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath()); - fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir)); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), List.of(gzip, lz4)); } @Test @@ -79,11 +81,25 @@ public class FileServerTest { CompletableFuture<byte []> content = new CompletableFuture<>(); fileServer.startFileServing(new FileReference("12y"), new FileReceiver(content), Set.of(gzip)); assertEquals(new String(content.get()), "dummy-data"); + } - IOUtils.writeFile(dir + "/12z/f1", "dummy-data-2", true); - content = new CompletableFuture<>(); - fileServer.startFileServing(new FileReference("12z"), new FileReceiver(content), Set.of(gzip, lz4)); - assertEquals(new String(content.get()), "dummy-data-2"); + @Test + public void requireThatWeCanReplayDirWithLz4() throws IOException, InterruptedException, ExecutionException { + File rootDir = new File(temporaryFolder.newFolder("fileserver-root-3").getAbsolutePath()); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), List.of(lz4, gzip)); // prefer lz4 + File dir = getFileServerRootDir(); + IOUtils.writeFile(dir + "/subdir/12z/f1", "dummy-data-2", true); + CompletableFuture<byte []> content = new CompletableFuture<>(); + fileServer.startFileServing(new FileReference("subdir"), new FileReceiver(content), Set.of(gzip, lz4)); + + // Decompress with lz4 and check contents + var compressor = new FileReferenceCompressor(FileReferenceData.Type.compressed, lz4); + File downloadedFileCompressed = new File(dir + "/downloaded-file-compressed"); + IOUtils.writeFile(downloadedFileCompressed, content.get()); + File downloadedFileUncompressed = new File(dir + "/downloaded-file-uncompressed"); + compressor.decompress(downloadedFileCompressed, downloadedFileUncompressed); + assertTrue(downloadedFileUncompressed.isDirectory()); + assertEquals("dummy-data-2", IOUtils.readFile(new File(downloadedFileUncompressed, "12z/f1"))); } @Test @@ -124,7 +140,7 @@ public class FileServerTest { private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); - return new FileServer(new ConfigserverConfig(configBuilder)); + return new FileServer(new ConfigserverConfig(configBuilder), new InMemoryFlagSource()); } private static class FileReceiver implements FileServer.Receiver { @@ -149,7 +165,8 @@ public class FileServerTest { new Supervisor(new Transport("mock")).setDropEmptyBuffers(true), downloadDirectory, Duration.ofMillis(100), - Duration.ofMillis(100)); + Duration.ofMillis(100), + Set.of(gzip)); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5941ed536a8..dc87ae2a0b4 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -6,11 +6,11 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; - import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,6 +19,8 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** * Handles downloads of files (file references only for now) * @@ -37,19 +39,20 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries) { + Duration sleepBetweenRetries, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -60,7 +63,8 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory); + downloadDirectory, + acceptedCompressionTypes); } public Optional<File> getFile(FileReferenceDownload fileReferenceDownload) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java index 8d6f428eaef..f3a8cf9299d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -18,7 +18,7 @@ public class FileReferenceDownload { private final boolean downloadFromOtherSourceIfNotFound; private final String client; - public FileReferenceDownload(FileReference fileReference, String client) { + public FileReferenceDownload(FileReference fileReference, String client) { this(fileReference, client, true); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 0267feb9ffc..7078c5aae6c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -5,14 +5,16 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; - import java.io.File; import java.time.Duration; import java.time.Instant; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -20,8 +22,10 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** - * Downloads file reference using rpc requests to config server and keeps track of files being downloaded + * Downloads file reference from config server and keeps track of files being downloaded * * @author hmusum */ @@ -38,12 +42,14 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; + private final Set<CompressionType> acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory) { + File downloadDirectory, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -51,6 +57,7 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); + this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -132,6 +139,9 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); + String[] temp = new String[acceptedCompressionTypes.size()]; + acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + request.parameters().add(new StringArray(temp)); return request; } @@ -160,4 +170,9 @@ public class FileReferenceDownloader { } } + private static Set<CompressionType> requireNonEmpty(Set<CompressionType> s) { + if (Objects.requireNonNull(s).isEmpty()) throw new IllegalArgumentException("set must be non-empty"); + return s; + } + } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index f5cd1760e89..629ea5915df 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -17,7 +17,6 @@ import net.jpountz.xxhash.XXHashFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; - import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,6 +25,7 @@ import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,6 +41,7 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { private static final Duration sleepBetweenRetries = Duration.ofMillis(10); + private static final Set<FileReferenceData.CompressionType> acceptedCompressionTypes = Set.of(gzip); private MockConnection connection; private FileDownloader fileDownloader; @@ -53,7 +54,7 @@ public class FileDownloaderTest { downloadDir = Files.createTempDirectory("filedistribution").toFile(); connection = new MockConnection(); supervisor = new Supervisor(new Transport()).setDropEmptyBuffers(true); - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(1), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(1)); } catch (IOException e) { e.printStackTrace(); fail(e.getMessage()); @@ -167,7 +168,7 @@ public class FileDownloaderTest { @Test public void getFileWhenConnectionError() throws IOException { - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(2), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(2)); File downloadDir = fileDownloader.downloadDirectory(); int timesToFail = 2; @@ -201,7 +202,7 @@ public class FileDownloaderTest { public void getFileWhenDownloadInProgress() throws IOException, ExecutionException, InterruptedException { ExecutorService executor = Executors.newFixedThreadPool(2); String filename = "abc.jar"; - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(3), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(3)); File downloadDir = fileDownloader.downloadDirectory(); // Delay response so that we can make a second request while downloading the file from the first request @@ -241,7 +242,7 @@ public class FileDownloaderTest { Duration timeout = Duration.ofMillis(200); MockConnection connectionPool = new MockConnection(); connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); - FileDownloader fileDownloader = new FileDownloader(connectionPool, supervisor, downloadDir, timeout, sleepBetweenRetries); + FileDownloader fileDownloader = createDownloader(connectionPool, timeout); FileReference xyzzy = new FileReference("xyzzy"); // Should download since we do not have the file on disk fileDownloader.downloadIfNeeded(new FileReferenceDownload(xyzzy, "test")); @@ -264,6 +265,16 @@ public class FileDownloaderTest { assertEquals("content", IOUtils.readFile(downloadedFile)); } + @Test + public void testCompressionTypes() { + try { + createDownloader(connection, Duration.ofSeconds(1), Set.of()); + fail("expected to fail when set is empty"); + } catch (IllegalArgumentException e) { + // ignore + } + } + private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { File fileReferenceDir = new File(dir, fileReferenceString); fileReferenceDir.mkdir(); @@ -302,6 +313,14 @@ public class FileDownloaderTest { return fileDownloader.getFile(new FileReferenceDownload(fileReference, "test")); } + private FileDownloader createDownloader(MockConnection connection, Duration timeout) { + return createDownloader(connection, timeout, acceptedCompressionTypes); + } + + private FileDownloader createDownloader(MockConnection connection, Duration timeout, Set<FileReferenceData.CompressionType> acceptedCompressionTypes) { + return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries, acceptedCompressionTypes); + } + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { private ResponseHandler responseHandler; |