diff options
author | Lester Solbakken <lesters@users.noreply.github.com> | 2019-01-11 08:14:39 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-01-11 08:14:39 +0100 |
commit | 158dcad12781af923ea3856f988fdf9c6d5e3e0b (patch) | |
tree | 400201604c017dccdeed9f11563f368331805c42 /config | |
parent | 971b57d0d6985bc33543cda13bca95096f7cfa92 (diff) | |
parent | ab54ef0073c78aaf0abe3769a6b9333779d3465a (diff) |
Merge pull request #8069 from vespa-engine/lesters/download-urls
Lesters/download urls
Diffstat (limited to 'config')
3 files changed, 135 insertions, 9 deletions
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 8a12405d505..40d79afc854 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -20,7 +20,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Stack; import java.util.logging.Logger; /** @@ -36,15 +40,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; + private final UrlDownloader urlDownloader; private final Stack<NamedBuilder> stack = new Stack<>(); public ConfigPayloadApplier(T builder) { - this(builder, new IdentityPathAcquirer()); + this(builder, new IdentityPathAcquirer(), null); } - public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer, UrlDownloader urlDownloader) { this.rootBuilder = builder; this.pathAcquirer = pathAcquirer; + this.urlDownloader = urlDownloader; debug("rootBuilder=" + rootBuilder); } @@ -207,6 +213,12 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { if (isPathField(builder, methodName)) { FileReference wrappedPath = resolvePath((String)value); invokeSetter(builder, methodName, key, wrappedPath); + + // Need to convert url into actual file if 'url' type is used + } else if (isUrlField(builder, methodName)) { + UrlReference url = resolveUrl((String)value); + invokeSetter(builder, methodName, key, url); + } else { invokeSetter(builder, methodName, key, value); } @@ -258,7 +270,8 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { // Need to convert url into actual file if 'url' type is used } else if (isUrlField(builder, methodName)) { - throw new UnsupportedOperationException("'url' type is not yet implemented"); + UrlReference url = resolveUrl(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, url); } else { Object object = getValueFromInspector(value); @@ -276,6 +289,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return newFileReference(path.toString()); } + private UrlReference resolveUrl(String url) { + if (urlDownloader == null || !urlDownloader.isValid()) { + throw new RuntimeException("Resolving url field failed due to missing or invalid URL downloader."); + } + File file = urlDownloader.waitFor(new UrlReference(url), 60 * 60); + return new UrlReference(file.getAbsolutePath()); + } + private FileReference newFileReference(String fileReference) { try { Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); @@ -343,18 +364,19 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { * Checks whether or not this field is of type 'path', in which * case some special handling might be needed. Caches the result. */ + private Set<String> pathFieldSet = new HashSet<>(); private boolean isPathField(Object builder, String methodName) { // Paths are stored as FileReference in Builder. - return isFieldType(builder, methodName, FileReference.class); + return isFieldType(pathFieldSet, builder, methodName, FileReference.class); } + private Set<String> urlFieldSet = new HashSet<>(); private boolean isUrlField(Object builder, String methodName) { // Urls are stored as UrlReference in Builder. - return isFieldType(builder, methodName, UrlReference.class); + return isFieldType(urlFieldSet, builder, methodName, UrlReference.class); } - private Set<String> fieldSet = new HashSet<>(); - private boolean isFieldType(Object builder, String methodName, java.lang.reflect.Type type) { + private boolean isFieldType(Set<String> fieldSet, Object builder, String methodName, java.lang.reflect.Type type) { String key = fieldKey(builder, methodName); if (fieldSet.contains(key)) { return true; @@ -515,4 +537,5 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return new File(fileReference.value()).toPath(); } } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index ac3f490182a..163e010cdd6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -26,6 +26,7 @@ public class ConfigTransformer<T extends ConfigInstance> { private final Class<T> clazz; private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + private static volatile UrlDownloader urlDownloader; /** * For internal use only * @@ -36,6 +37,10 @@ public class ConfigTransformer<T extends ConfigInstance> { pathAcquirer; } + public static void setUrlDownloader(UrlDownloader urlDownloader) { + ConfigTransformer.urlDownloader = urlDownloader; + } + /** * Create a transformer capable of converting payloads to clazz * @@ -53,7 +58,7 @@ public class ConfigTransformer<T extends ConfigInstance> { */ public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { ConfigInstance.Builder builder = getRootBuilder(); - ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer, urlDownloader); creator.applyPayload(payload); return builder; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java new file mode 100644 index 00000000000..4947b618f50 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -0,0 +1,98 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.UrlReference; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.util.logging.Logger; + +/** + * @author lesters + */ +public class UrlDownloader { + + private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); + + private static final int BASE_ERROR_CODE = 0x10000; + public static final int DOES_NOT_EXIST = BASE_ERROR_CODE + 1; + public static final int INTERNAL_ERROR = BASE_ERROR_CODE + 2; + public static final int HTTP_ERROR = BASE_ERROR_CODE + 3; + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Spec spec; + private Target target; + + public UrlDownloader() { + spec = new Spec(Defaults.getDefaults().vespaHostname(), Defaults.getDefaults().vespaConfigProxyRpcPort()); + connect(); + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + private void connect() { + int timeRemaining = 5000; + try { + while (timeRemaining > 0) { + target = supervisor.connectSync(spec); + if (target.isValid()) { + log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + return; + } + Thread.sleep(500); + timeRemaining -= 500; + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public boolean isValid() { + return target.isValid(); + } + + private boolean temporaryError(Request req) { + return false; // Currently, none of the errors are considered temporary + } + + public File waitFor(UrlReference urlReference, long timeout) { + long start = System.currentTimeMillis() / 1000; + long timeLeft = timeout; + do { + Request request = new Request("url.waitFor"); + request.parameters().add(new StringValue(urlReference.value())); + + double rpcTimeout = Math.min(timeLeft, 60 * 60.0); + log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + target.invokeSync(request, rpcTimeout); + + if (request.checkReturnTypes("s")) { + return new File(request.returnValues().get(0).asString()); + } else if (!request.isError()) { + throw new RuntimeException("Invalid response: " + request.returnValues()); + } else if (temporaryError(request)) { + log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted sleep between retries of waitFor", e); + } + } else { + throw new RuntimeException("Wait for " + urlReference + " failed: " + request.errorMessage() + " (" + request.errorCode() + ")"); + } + timeLeft = start + timeout - System.currentTimeMillis() / 1000; + } while (timeLeft > 0); + + throw new RuntimeException("Timed out waiting for " + urlReference + " after " + timeout); + } + +} |