diff options
7 files changed, 286 insertions, 5 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 6390c9ca165..6274ec77e01 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -89,6 +89,7 @@ public class ProxyServer implements Runnable { this.configClient = createClient(clientUpdater, delayedResponses, source, timingValues, memoryCache, configClient); this.fileDownloader = new FileDownloader(new JRTConnectionPool(source)); new FileDistributionRpcServer(supervisor, fileDownloader); + new UrlDownloadRpcServer(supervisor); } static ProxyServer createTestServer(ConfigSourceSet source) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java new file mode 100644 index 00000000000..29d94cce3cd --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -0,0 +1,145 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +import static com.yahoo.vespa.config.UrlDownloader.DOES_NOT_EXIST; +import static com.yahoo.vespa.config.UrlDownloader.HTTP_ERROR; +import static com.yahoo.vespa.config.UrlDownloader.INTERNAL_ERROR; + +/** + * An RPC server that handles URL download requests. + * + * @author lesters + */ +public class UrlDownloadRpcServer { + private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName()); + + private static final String CONTENTS_FILE_NAME = "contents"; + private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; + + private final File downloadBaseDir; + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + new DaemonThreadFactory("Rpc download executor")); + + public UrlDownloadRpcServer(Supervisor supervisor) { + supervisor.addMethod(new Method("url.waitFor", "s", "s", this, "download") + .methodDesc("get path to url download") + .paramDesc(0, "url", "url") + .returnDesc(0, "path", "path to file")); + downloadBaseDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void download(Request req) { + req.detach(); + rpcDownloadExecutor.execute(() -> downloadFile(req)); + } + + private void downloadFile(Request req) { + String url = req.parameters().get(0).asString(); + File downloadDir = new File(this.downloadBaseDir, urlToDirName(url)); + + try { + URL website = new URL(url); + HttpURLConnection connection = (HttpURLConnection) website.openConnection(); + setIfModifiedSince(connection, downloadDir); // don't download if we already have the file + + if (connection.getResponseCode() == 200) { + log.log(LogLevel.INFO, "Downloading URL '" + url + "'"); + downloadFile(req, connection, downloadDir); + + } else if (connection.getResponseCode() == 304) { + log.log(LogLevel.INFO, "URL '" + url + "' already downloaded (server response: 304)"); + req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); + + } else { + log.log(LogLevel.ERROR, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); + req.setError(HTTP_ERROR, String.valueOf(connection.getResponseCode())); + } + + } catch (Throwable e) { + log.log(LogLevel.WARNING, "Download of URL '" + url + "' got exception: " + e.getMessage()); + req.setError(INTERNAL_ERROR, "Download of URL '" + url + "' internal error"); + } + req.returnRequest(); + } + + private static void downloadFile(Request req, HttpURLConnection connection, File downloadDir) throws IOException { + String url = connection.getURL().toString(); + Files.createDirectories(downloadDir.toPath()); + File contentsPath = new File(downloadDir, CONTENTS_FILE_NAME); + try (ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream())) { + try (FileOutputStream fos = new FileOutputStream((contentsPath.getAbsolutePath()))) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + if (contentsPath.exists() && contentsPath.length() > 0) { + writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); + req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); + log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); + } else { + log.log(LogLevel.INFO, "Downloaded URL '" + url + "' not found, returning error"); + req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); + } + } + } + } + + private static String urlToDirName(String uri) { + return String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + } + + private static void setIfModifiedSince(HttpURLConnection connection, File downloadDir) throws IOException { + File contents = new File(downloadDir, CONTENTS_FILE_NAME); + if (contents.exists() && contents.length() > 0) { + long lastModified = readLastModifiedTimestamp(downloadDir); + if (lastModified > 0) { + connection.setIfModifiedSince(lastModified); + } + } + } + + private static long readLastModifiedTimestamp(File downloadDir) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + if (lastModified.exists() && lastModified.length() > 0) { + try (BufferedReader br = new BufferedReader(new FileReader(lastModified))) { + String timestamp = br.readLine(); + return Long.parseLong(timestamp); + } + } + return 0; + } + + private static void writeLastModifiedTimestamp(File downloadDir, long timestamp) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + try (BufferedWriter lastModifiedWriter = new BufferedWriter(new FileWriter(lastModified.getAbsolutePath()))) { + lastModifiedWriter.write(Long.toString(timestamp)); + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 8a12405d505..4415ac21948 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -20,7 +20,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Stack; import java.util.logging.Logger; /** @@ -36,15 +40,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; + private final UrlDownloader urlDownloader; private final Stack<NamedBuilder> stack = new Stack<>(); public ConfigPayloadApplier(T builder) { - this(builder, new IdentityPathAcquirer()); + this(builder, new IdentityPathAcquirer(), null); } - public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer, UrlDownloader urlDownloader) { this.rootBuilder = builder; this.pathAcquirer = pathAcquirer; + this.urlDownloader = urlDownloader; debug("rootBuilder=" + rootBuilder); } @@ -207,6 +213,12 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { if (isPathField(builder, methodName)) { FileReference wrappedPath = resolvePath((String)value); invokeSetter(builder, methodName, key, wrappedPath); + + // Need to convert url into actual file if 'url' type is used + } else if (isUrlField(builder, methodName)) { + UrlReference url = resolveUrl((String)value); + invokeSetter(builder, methodName, key, url); + } else { invokeSetter(builder, methodName, key, value); } @@ -258,7 +270,8 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { // Need to convert url into actual file if 'url' type is used } else if (isUrlField(builder, methodName)) { - throw new UnsupportedOperationException("'url' type is not yet implemented"); + UrlReference url = resolveUrl(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, url); } else { Object object = getValueFromInspector(value); @@ -276,6 +289,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return newFileReference(path.toString()); } + private UrlReference resolveUrl(String url) { + if (urlDownloader == null || !urlDownloader.isValid()) { + throw new RuntimeException("Resolving url field failed due to missing or invalid URL downloader."); + } + File file = urlDownloader.waitFor(new UrlReference(url), 10 * 60); + return new UrlReference(file.getAbsolutePath()); + } + private FileReference newFileReference(String fileReference) { try { Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); @@ -515,4 +536,5 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return new File(fileReference.value()).toPath(); } } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index ac3f490182a..163e010cdd6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -26,6 +26,7 @@ public class ConfigTransformer<T extends ConfigInstance> { private final Class<T> clazz; private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + private static volatile UrlDownloader urlDownloader; /** * For internal use only * @@ -36,6 +37,10 @@ public class ConfigTransformer<T extends ConfigInstance> { pathAcquirer; } + public static void setUrlDownloader(UrlDownloader urlDownloader) { + ConfigTransformer.urlDownloader = urlDownloader; + } + /** * Create a transformer capable of converting payloads to clazz * @@ -53,7 +58,7 @@ public class ConfigTransformer<T extends ConfigInstance> { */ public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { ConfigInstance.Builder builder = getRootBuilder(); - ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer, urlDownloader); creator.applyPayload(payload); return builder; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java new file mode 100644 index 00000000000..cd72c83d100 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -0,0 +1,98 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.UrlReference; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.util.logging.Logger; + +/** + * @author lesters + */ +public class UrlDownloader { + + private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); + + private static final int BASE_ERROR_CODE = 0x10000; + public static final int DOES_NOT_EXIST = BASE_ERROR_CODE + 1; + public static final int INTERNAL_ERROR = BASE_ERROR_CODE + 2; + public static final int HTTP_ERROR = BASE_ERROR_CODE + 3; + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Spec spec; + private Target target; + + public UrlDownloader() { + spec = new Spec(Defaults.getDefaults().vespaHostname(), Defaults.getDefaults().vespaConfigProxyRpcPort()); + connect(); + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + private void connect() { + int timeRemaining = 5000; + try { + while (timeRemaining > 0) { + target = supervisor.connectSync(spec); + if (target.isValid()) { + log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + return; + } + Thread.sleep(500); + timeRemaining -= 500; + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public boolean isValid() { + return target.isValid(); + } + + private boolean temporaryError(Request req) { + return false; // Currently, none of the errors are considered temporary + } + + public File waitFor(UrlReference urlReference, long timeout) { + long start = System.currentTimeMillis() / 1000; + long timeLeft = timeout; + do { + Request request = new Request("url.waitFor"); + request.parameters().add(new StringValue(urlReference.value())); + + double rpcTimeout = Math.min(timeLeft, 60.0); + log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + target.invokeSync(request, rpcTimeout); + + if (request.checkReturnTypes("s")) { + return new File(request.returnValues().get(0).asString()); + } else if (!request.isError()) { + throw new RuntimeException("Invalid response: " + request.returnValues()); + } else if (temporaryError(request)) { + log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted sleep between retries of waitFor", e); + } + } else { + throw new RuntimeException("Wait for " + urlReference + " failed:" + request.errorMessage() + " (" + request.errorCode() + ")"); + } + timeLeft = start + timeout - System.currentTimeMillis() / 1000; + } while (timeLeft > 0); + + throw new RuntimeException("Timed out waiting for " + urlReference + " after " + timeout); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index bb4b57e8983..e84c8b340a4 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -11,6 +11,7 @@ import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.Osgi; import com.yahoo.vespa.config.ConfigTransformer; +import com.yahoo.vespa.config.UrlDownloader; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -31,6 +32,7 @@ public class Container { private volatile ComponentRegistry<ServerProvider> serverProviderRegistry; private volatile ComponentRegistry<AbstractComponent> componentRegistry; private volatile FileAcquirer fileAcquirer; + private volatile UrlDownloader urlDownloader; private volatile BundleLoader bundleLoader; @@ -50,6 +52,8 @@ public class Container { public void shutdown() { if (fileAcquirer != null) fileAcquirer.shutdown(); + if (urlDownloader != null) + urlDownloader.shutdown(); } //Used to acquire files originating from the application package. @@ -147,4 +151,9 @@ public class Container { }); } + public void setupUrlDownloader() { + this.urlDownloader = new UrlDownloader(); + ConfigTransformer.setUrlDownloader(urlDownloader); + } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index f4fae683877..1cb4e1d4555 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -191,6 +191,7 @@ public final class ConfiguredApplication implements Application { private static void hackToInitializeServer(QrConfig config) { try { Container.get().setupFileAcquirer(config.filedistributor()); + Container.get().setupUrlDownloader(); com.yahoo.container.Server.get().initialize(config); } catch (Exception e) { log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); |