From 88af4ae0b13faba664f9d5d5c77d37a6b6f09498 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 1 Feb 2023 13:20:10 +0100 Subject: Stop using file distribution feature flags, not needed anymore --- .../yahoo/vespa/filedistribution/FileDownloader.java | 17 ++++++----------- .../filedistribution/FileReferenceDownloader.java | 15 ++++++++------- .../vespa/filedistribution/FileDownloaderTest.java | 18 +----------------- 3 files changed, 15 insertions(+), 35 deletions(-) (limited to 'filedistribution') diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 63ae8faacfe..68f969df3af 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -10,7 +10,6 @@ import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,8 +18,6 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; - /** * Handles downloads of files (file references only for now) * @@ -40,20 +37,19 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set acceptedCompressionTypes) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set acceptedCompressionTypes) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries, - Set acceptedCompressionTypes) { + Duration sleepBetweenRetries) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -64,8 +60,7 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory, - acceptedCompressionTypes); + downloadDirectory); if (forceDownload) log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)"); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index f14de0b9807..0966561ce5d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -23,6 +23,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; /** * Downloads file reference from config server and keeps track of files being downloaded @@ -31,7 +34,8 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType */ public class FileReferenceDownloader { - private final static Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private static final Set defaultAcceptedCompressionTypes = Set.of(gzip, lz4, zstd); private final ExecutorService downloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), @@ -42,14 +46,12 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; - private final Set acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory, - Set acceptedCompressionTypes) { + File downloadDirectory) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -57,7 +59,6 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); - this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -139,8 +140,8 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); - String[] temp = new String[acceptedCompressionTypes.size()]; - acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + String[] temp = new String[defaultAcceptedCompressionTypes.size()]; + defaultAcceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); request.parameters().add(new StringArray(temp)); return request; } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index ffef06e6367..d593a783064 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -25,7 +25,6 @@ import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,7 +40,6 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { private static final Duration sleepBetweenRetries = Duration.ofMillis(10); - private static final Set acceptedCompressionTypes = Set.of(gzip); private MockConnection connection; private FileDownloader fileDownloader; @@ -265,16 +263,6 @@ public class FileDownloaderTest { assertEquals("content", IOUtils.readFile(downloadedFile)); } - @Test - public void testCompressionTypes() { - try { - createDownloader(connection, Duration.ofSeconds(1), Set.of()); - fail("expected to fail when set is empty"); - } catch (IllegalArgumentException e) { - // ignore - } - } - private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { File fileReferenceDir = new File(dir, fileReferenceString); fileReferenceDir.mkdir(); @@ -314,11 +302,7 @@ public class FileDownloaderTest { } private FileDownloader createDownloader(MockConnection connection, Duration timeout) { - return createDownloader(connection, timeout, acceptedCompressionTypes); - } - - private FileDownloader createDownloader(MockConnection connection, Duration timeout, Set acceptedCompressionTypes) { - return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries, acceptedCompressionTypes); + return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { -- cgit v1.2.3