diff options
author | Harald Musum <musum@yahooinc.com> | 2023-09-17 19:24:26 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-09-17 19:24:26 +0200 |
commit | 8f26508bf1d99839e78e4ba95e6ad3c4bffd22fb (patch) | |
tree | 044a25fe376bad431b6524a8f1d41a59a661fd42 /config-proxy | |
parent | c61179d038ff3e9b865906a9c001f6b720fb7d8a (diff) |
Split out downloaders into separate classes
Diffstat (limited to 'config-proxy')
3 files changed, 72 insertions, 31 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/S3Downloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/S3Downloader.java new file mode 100644 index 00000000000..76a6c29a56a --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/S3Downloader.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.config.proxy.filedistribution; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; + +public class S3Downloader { + + // TODO: Avoid hardcoding + private static final String ZTS_URL = "https://zts.athenz.ouroath.com:4443/zts/v1"; + + Optional<File> downloadFile(String fileName, File targetDir) throws IOException { + throw new UnsupportedOperationException("Download of S3 urls not implemented"); + } + +} diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 41ca5ceab82..7ec60985007 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -13,14 +13,8 @@ import com.yahoo.yolean.Exceptions; import net.jpountz.xxhash.XXHashFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.file.Files; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -102,31 +96,9 @@ class UrlDownloadRpcServer { } private static Optional<File> downloadFile(String url, File downloadDir) throws IOException { - long start = System.currentTimeMillis(); - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - if (connection.getResponseCode() == 200) - log.log(Level.INFO, "Downloading URL '" + url + "'"); - else - throw new RuntimeException("Download of URL '" + url + "' failed, got response code " + connection.getResponseCode()); - - Files.createDirectories(downloadDir.toPath()); - File contentsPath = new File(downloadDir, CONTENTS_FILE_NAME); - try (ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream())) { - try (FileOutputStream fos = new FileOutputStream((contentsPath.getAbsolutePath()))) { - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - - if (contentsPath.exists() && contentsPath.length() > 0) { - new RequestTracker().trackRequest(downloadDir); - log.log(Level.FINE, () -> "URL '" + url + "' available at " + contentsPath); - log.log(Level.INFO, String.format("Download of URL '%s' done in %.3f seconds", - url, (System.currentTimeMillis() - start) / 1000.0)); - return Optional.of(contentsPath); - } else { - log.log(Level.SEVERE, "Downloaded URL '" + url + "' not found, returning error"); - return Optional.empty(); - } - } - } + return (url.startsWith("s3://")) + ? new S3Downloader().downloadFile(url, downloadDir) + : new UrlDownloader().downloadFile(url, downloadDir); } private static String urlToDirName(String uri) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java new file mode 100644 index 00000000000..5fa2a12f608 --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy.filedistribution; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Download of urls + * + * @author hmusum + */ +class UrlDownloader { + + private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); + private static final String CONTENTS_FILE_NAME = "contents"; + + Optional<File> downloadFile(String url, File downloadDir) throws IOException { + long start = System.currentTimeMillis(); + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + if (connection.getResponseCode() != 200) + throw new RuntimeException("Download of URL '" + url + "' failed, got response code " + connection.getResponseCode()); + + log.log(Level.INFO, "Downloading URL '" + url + "'"); + Files.createDirectories(downloadDir.toPath()); + File contentsPath = new File(downloadDir, CONTENTS_FILE_NAME); + try (ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream())) { + try (FileOutputStream fos = new FileOutputStream((contentsPath.getAbsolutePath()))) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + if (contentsPath.exists() && contentsPath.length() > 0) { + new RequestTracker().trackRequest(downloadDir); + log.log(Level.FINE, () -> "URL '" + url + "' available at " + contentsPath); + log.log(Level.INFO, String.format("Download of URL '%s' done in %.3f seconds", + url, (System.currentTimeMillis() - start) / 1000.0)); + return Optional.of(contentsPath); + } else { + log.log(Level.SEVERE, "Downloaded URL '" + url + "' not found, returning error"); + return Optional.empty(); + } + } + } + } + +} |