diff options
Diffstat (limited to 'config-proxy/src/main')
-rw-r--r-- | config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java | 1 | ||||
-rw-r--r-- | config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java | 145 |
2 files changed, 146 insertions, 0 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 6390c9ca165..6274ec77e01 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -89,6 +89,7 @@ public class ProxyServer implements Runnable { this.configClient = createClient(clientUpdater, delayedResponses, source, timingValues, memoryCache, configClient); this.fileDownloader = new FileDownloader(new JRTConnectionPool(source)); new FileDistributionRpcServer(supervisor, fileDownloader); + new UrlDownloadRpcServer(supervisor); } static ProxyServer createTestServer(ConfigSourceSet source) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java new file mode 100644 index 00000000000..29d94cce3cd --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -0,0 +1,145 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +import static com.yahoo.vespa.config.UrlDownloader.DOES_NOT_EXIST; +import static com.yahoo.vespa.config.UrlDownloader.HTTP_ERROR; +import static com.yahoo.vespa.config.UrlDownloader.INTERNAL_ERROR; + +/** + * An RPC server that handles URL download requests. + * + * @author lesters + */ +public class UrlDownloadRpcServer { + private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName()); + + private static final String CONTENTS_FILE_NAME = "contents"; + private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; + + private final File downloadBaseDir; + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + new DaemonThreadFactory("Rpc download executor")); + + public UrlDownloadRpcServer(Supervisor supervisor) { + supervisor.addMethod(new Method("url.waitFor", "s", "s", this, "download") + .methodDesc("get path to url download") + .paramDesc(0, "url", "url") + .returnDesc(0, "path", "path to file")); + downloadBaseDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void download(Request req) { + req.detach(); + rpcDownloadExecutor.execute(() -> downloadFile(req)); + } + + private void downloadFile(Request req) { + String url = req.parameters().get(0).asString(); + File downloadDir = new File(this.downloadBaseDir, urlToDirName(url)); + + try { + URL website = new URL(url); + HttpURLConnection connection = (HttpURLConnection) website.openConnection(); + setIfModifiedSince(connection, downloadDir); // don't download if we already have the file + + if (connection.getResponseCode() == 200) { + log.log(LogLevel.INFO, "Downloading URL '" + url + "'"); + downloadFile(req, connection, downloadDir); + + } else if (connection.getResponseCode() == 304) { + log.log(LogLevel.INFO, "URL '" + url + "' already downloaded (server response: 304)"); + req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); + + } else { + log.log(LogLevel.ERROR, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); + req.setError(HTTP_ERROR, String.valueOf(connection.getResponseCode())); + } + + } catch (Throwable e) { + log.log(LogLevel.WARNING, "Download of URL '" + url + "' got exception: " + e.getMessage()); + req.setError(INTERNAL_ERROR, "Download of URL '" + url + "' internal error"); + } + req.returnRequest(); + } + + private static void downloadFile(Request req, HttpURLConnection connection, File downloadDir) throws IOException { + String url = connection.getURL().toString(); + Files.createDirectories(downloadDir.toPath()); + File contentsPath = new File(downloadDir, CONTENTS_FILE_NAME); + try (ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream())) { + try (FileOutputStream fos = new FileOutputStream((contentsPath.getAbsolutePath()))) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + if (contentsPath.exists() && contentsPath.length() > 0) { + writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); + req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); + log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); + } else { + log.log(LogLevel.INFO, "Downloaded URL '" + url + "' not found, returning error"); + req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); + } + } + } + } + + private static String urlToDirName(String uri) { + return String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + } + + private static void setIfModifiedSince(HttpURLConnection connection, File downloadDir) throws IOException { + File contents = new File(downloadDir, CONTENTS_FILE_NAME); + if (contents.exists() && contents.length() > 0) { + long lastModified = readLastModifiedTimestamp(downloadDir); + if (lastModified > 0) { + connection.setIfModifiedSince(lastModified); + } + } + } + + private static long readLastModifiedTimestamp(File downloadDir) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + if (lastModified.exists() && lastModified.length() > 0) { + try (BufferedReader br = new BufferedReader(new FileReader(lastModified))) { + String timestamp = br.readLine(); + return Long.parseLong(timestamp); + } + } + return 0; + } + + private static void writeLastModifiedTimestamp(File downloadDir, long timestamp) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + try (BufferedWriter lastModifiedWriter = new BufferedWriter(new FileWriter(lastModified.getAbsolutePath()))) { + lastModifiedWriter.write(Long.toString(timestamp)); + } + } + +} |