diff options
Diffstat (limited to 'config')
-rw-r--r-- | config/abi-spec.json | 299 | ||||
-rwxr-xr-x | config/pom.xml | 4 | ||||
-rw-r--r-- | config/src/apps/vespa-configproxy-cmd/main.cpp | 2 | ||||
-rw-r--r-- | config/src/apps/vespa-configproxy-cmd/methods.cpp | 2 | ||||
-rw-r--r-- | config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java | 39 | ||||
-rw-r--r-- | config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java | 7 | ||||
-rw-r--r-- | config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java | 98 |
7 files changed, 440 insertions, 11 deletions
diff --git a/config/abi-spec.json b/config/abi-spec.json new file mode 100644 index 00000000000..f6ab43f2c9b --- /dev/null +++ b/config/abi-spec.json @@ -0,0 +1,299 @@ +{ + "com.yahoo.config.subscription.CfgConfigPayloadBuilder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public com.yahoo.vespa.config.ConfigPayload deserialize(java.util.List)", + "public com.yahoo.vespa.config.ConfigPayloadBuilder deserializeToBuilder(java.util.List)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigDebug": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public static void logDebug(java.util.logging.Logger, long, com.yahoo.vespa.config.ConfigKey, java.lang.String)", + "public static void logDebug(java.util.logging.Logger, com.yahoo.config.ConfigInstance$Builder, java.lang.String, java.lang.String)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigGetter": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.Class)", + "public void <init>(com.yahoo.config.subscription.ConfigSource, java.lang.Class)", + "public synchronized com.yahoo.config.ConfigInstance getConfig(java.lang.String)", + "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String)", + "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String, com.yahoo.config.subscription.ConfigSource)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigHandle": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "protected void <init>(com.yahoo.config.subscription.impl.ConfigSubscription)", + "public boolean isChanged()", + "public com.yahoo.config.ConfigInstance getConfig()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigInstanceSerializer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.Serializer" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.slime.Slime)", + "public void <init>(com.yahoo.slime.Slime, com.yahoo.slime.Cursor)", + "public com.yahoo.config.Serializer createInner(java.lang.String)", + "public com.yahoo.config.Serializer createArray(java.lang.String)", + "public com.yahoo.config.Serializer createInner()", + "public com.yahoo.config.Serializer createMap(java.lang.String)", + "public void serialize(java.lang.String, boolean)", + "public void serialize(java.lang.String, double)", + "public void serialize(java.lang.String, int)", + "public void serialize(java.lang.String, long)", + "public void serialize(java.lang.String, java.lang.String)", + "public void serialize(boolean)", + "public void serialize(double)", + "public void serialize(long)", + "public void serialize(int)", + "public void serialize(java.lang.String)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigInstanceUtil": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public static void setValues(com.yahoo.config.ConfigBuilder, com.yahoo.config.ConfigBuilder)", + "public static com.yahoo.config.ConfigInstance getNewInstance(java.lang.Class, java.lang.String, com.yahoo.vespa.config.ConfigPayload)", + "public static java.lang.Object getField(com.yahoo.config.ConfigBuilder, java.lang.String)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigInterruptedException": { + "superClass": "java.lang.RuntimeException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.Throwable)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigSet": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void addBuilder(java.lang.String, com.yahoo.config.ConfigInstance$Builder)", + "public com.yahoo.config.ConfigInstance$Builder get(com.yahoo.vespa.config.ConfigKey)", + "public boolean contains(com.yahoo.vespa.config.ConfigKey)", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigSource": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigSourceSet": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(java.util.List)", + "public void <init>(java.lang.String[])", + "public void <init>(java.lang.String)", + "public java.util.Set getSources()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()", + "public static com.yahoo.config.subscription.ConfigSourceSet createDefault()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void configure(com.yahoo.config.ConfigInstance)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.ConfigSubscriber$State": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.subscription.ConfigSubscriber$State[] values()", + "public static com.yahoo.config.subscription.ConfigSubscriber$State valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State OPEN", + "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State FROZEN", + "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State CLOSED" + ] + }, + "com.yahoo.config.subscription.ConfigSubscriber": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.config.subscription.ConfigSource)", + "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String)", + "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String, long)", + "protected void checkStateBeforeSubscribe()", + "protected void subscribeAndHandleErrors(com.yahoo.config.subscription.impl.ConfigSubscription, com.yahoo.vespa.config.ConfigKey, com.yahoo.config.subscription.ConfigHandle, com.yahoo.vespa.config.TimingValues)", + "public boolean nextConfig()", + "public boolean nextConfig(long)", + "public boolean nextGeneration()", + "public boolean nextGeneration(long)", + "protected void throwIfExceptionSet(com.yahoo.config.subscription.impl.ConfigSubscription)", + "public void close()", + "protected void closeRequesters()", + "public java.lang.String toString()", + "public java.lang.Thread startConfigThread(java.lang.Runnable)", + "protected com.yahoo.config.subscription.ConfigSubscriber$State state()", + "public void reload(long)", + "public com.yahoo.config.subscription.ConfigSource getSource()", + "public java.util.Map requesters()", + "public boolean isClosed()", + "public com.yahoo.config.subscription.ConfigHandle subscribe(com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber, java.lang.Class, java.lang.String)", + "public long getGeneration()", + "public boolean isInternalRedeploy()", + "protected void finalize()" + ], + "fields": [ + "protected final java.util.List subscriptionHandles", + "protected java.util.Map requesters" + ] + }, + "com.yahoo.config.subscription.ConfigURI": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.String getConfigId()", + "public com.yahoo.config.subscription.ConfigSource getSource()", + "public static com.yahoo.config.subscription.ConfigURI createFromId(java.lang.String)", + "public static com.yahoo.config.subscription.ConfigURI createFromIdAndSource(java.lang.String, com.yahoo.config.subscription.ConfigSource)" + ], + "fields": [] + }, + "com.yahoo.config.subscription.DirSource": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.io.File)", + "public java.io.File getDir()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.FileSource": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.io.File)", + "public java.io.File getFile()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.JarSource": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.util.jar.JarFile, java.lang.String)", + "public java.util.jar.JarFile getJarFile()", + "public java.lang.String getPath()" + ], + "fields": [] + }, + "com.yahoo.config.subscription.RawSource": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.subscription.ConfigSource" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)" + ], + "fields": [ + "public final java.lang.String payload" + ] + } +}
\ No newline at end of file diff --git a/config/pom.xml b/config/pom.xml index e1e04e17033..ea5948a30bb 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -192,6 +192,10 @@ <updateReleaseInfo>true</updateReleaseInfo> </configuration> </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> </plugins> </build> </project> diff --git a/config/src/apps/vespa-configproxy-cmd/main.cpp b/config/src/apps/vespa-configproxy-cmd/main.cpp index 63fa1285bb2..bb908e7268b 100644 --- a/config/src/apps/vespa-configproxy-cmd/main.cpp +++ b/config/src/apps/vespa-configproxy-cmd/main.cpp @@ -22,7 +22,7 @@ Application::parseOpts() char c = '?'; const char *optArg = NULL; int optInd = 0; - while ((c = GetOpt("m:s:p:", optArg, optInd)) != -1) { + while ((c = GetOpt("m:s:p:h", optArg, optInd)) != -1) { switch (c) { case 'm': _flags.method = optArg; diff --git a/config/src/apps/vespa-configproxy-cmd/methods.cpp b/config/src/apps/vespa-configproxy-cmd/methods.cpp index 6a143667102..6218c54e8b3 100644 --- a/config/src/apps/vespa-configproxy-cmd/methods.cpp +++ b/config/src/apps/vespa-configproxy-cmd/methods.cpp @@ -14,7 +14,7 @@ const Method methods[] = { { "cachefull", "listCachedConfigFull", 0 }, { "sources", "listSourceConnections", 0 }, { "statistics", "printStatistics", 0 }, - { "setmode", "setMode", 1 }, // { default | memorycache | diskcache } + { "setmode", "setMode", 1 }, // { default | memorycache } { "updatesources", "updateSources", 1 }, { 0, 0, 0} }; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 8a12405d505..40d79afc854 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -20,7 +20,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Stack; import java.util.logging.Logger; /** @@ -36,15 +40,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; + private final UrlDownloader urlDownloader; private final Stack<NamedBuilder> stack = new Stack<>(); public ConfigPayloadApplier(T builder) { - this(builder, new IdentityPathAcquirer()); + this(builder, new IdentityPathAcquirer(), null); } - public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer, UrlDownloader urlDownloader) { this.rootBuilder = builder; this.pathAcquirer = pathAcquirer; + this.urlDownloader = urlDownloader; debug("rootBuilder=" + rootBuilder); } @@ -207,6 +213,12 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { if (isPathField(builder, methodName)) { FileReference wrappedPath = resolvePath((String)value); invokeSetter(builder, methodName, key, wrappedPath); + + // Need to convert url into actual file if 'url' type is used + } else if (isUrlField(builder, methodName)) { + UrlReference url = resolveUrl((String)value); + invokeSetter(builder, methodName, key, url); + } else { invokeSetter(builder, methodName, key, value); } @@ -258,7 +270,8 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { // Need to convert url into actual file if 'url' type is used } else if (isUrlField(builder, methodName)) { - throw new UnsupportedOperationException("'url' type is not yet implemented"); + UrlReference url = resolveUrl(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, url); } else { Object object = getValueFromInspector(value); @@ -276,6 +289,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return newFileReference(path.toString()); } + private UrlReference resolveUrl(String url) { + if (urlDownloader == null || !urlDownloader.isValid()) { + throw new RuntimeException("Resolving url field failed due to missing or invalid URL downloader."); + } + File file = urlDownloader.waitFor(new UrlReference(url), 60 * 60); + return new UrlReference(file.getAbsolutePath()); + } + private FileReference newFileReference(String fileReference) { try { Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); @@ -343,18 +364,19 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { * Checks whether or not this field is of type 'path', in which * case some special handling might be needed. Caches the result. */ + private Set<String> pathFieldSet = new HashSet<>(); private boolean isPathField(Object builder, String methodName) { // Paths are stored as FileReference in Builder. - return isFieldType(builder, methodName, FileReference.class); + return isFieldType(pathFieldSet, builder, methodName, FileReference.class); } + private Set<String> urlFieldSet = new HashSet<>(); private boolean isUrlField(Object builder, String methodName) { // Urls are stored as UrlReference in Builder. - return isFieldType(builder, methodName, UrlReference.class); + return isFieldType(urlFieldSet, builder, methodName, UrlReference.class); } - private Set<String> fieldSet = new HashSet<>(); - private boolean isFieldType(Object builder, String methodName, java.lang.reflect.Type type) { + private boolean isFieldType(Set<String> fieldSet, Object builder, String methodName, java.lang.reflect.Type type) { String key = fieldKey(builder, methodName); if (fieldSet.contains(key)) { return true; @@ -515,4 +537,5 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return new File(fileReference.value()).toPath(); } } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index ac3f490182a..163e010cdd6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -26,6 +26,7 @@ public class ConfigTransformer<T extends ConfigInstance> { private final Class<T> clazz; private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + private static volatile UrlDownloader urlDownloader; /** * For internal use only * @@ -36,6 +37,10 @@ public class ConfigTransformer<T extends ConfigInstance> { pathAcquirer; } + public static void setUrlDownloader(UrlDownloader urlDownloader) { + ConfigTransformer.urlDownloader = urlDownloader; + } + /** * Create a transformer capable of converting payloads to clazz * @@ -53,7 +58,7 @@ public class ConfigTransformer<T extends ConfigInstance> { */ public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { ConfigInstance.Builder builder = getRootBuilder(); - ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer, urlDownloader); creator.applyPayload(payload); return builder; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java new file mode 100644 index 00000000000..4947b618f50 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -0,0 +1,98 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.UrlReference; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.util.logging.Logger; + +/** + * @author lesters + */ +public class UrlDownloader { + + private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); + + private static final int BASE_ERROR_CODE = 0x10000; + public static final int DOES_NOT_EXIST = BASE_ERROR_CODE + 1; + public static final int INTERNAL_ERROR = BASE_ERROR_CODE + 2; + public static final int HTTP_ERROR = BASE_ERROR_CODE + 3; + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Spec spec; + private Target target; + + public UrlDownloader() { + spec = new Spec(Defaults.getDefaults().vespaHostname(), Defaults.getDefaults().vespaConfigProxyRpcPort()); + connect(); + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + private void connect() { + int timeRemaining = 5000; + try { + while (timeRemaining > 0) { + target = supervisor.connectSync(spec); + if (target.isValid()) { + log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + return; + } + Thread.sleep(500); + timeRemaining -= 500; + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public boolean isValid() { + return target.isValid(); + } + + private boolean temporaryError(Request req) { + return false; // Currently, none of the errors are considered temporary + } + + public File waitFor(UrlReference urlReference, long timeout) { + long start = System.currentTimeMillis() / 1000; + long timeLeft = timeout; + do { + Request request = new Request("url.waitFor"); + request.parameters().add(new StringValue(urlReference.value())); + + double rpcTimeout = Math.min(timeLeft, 60 * 60.0); + log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + target.invokeSync(request, rpcTimeout); + + if (request.checkReturnTypes("s")) { + return new File(request.returnValues().get(0).asString()); + } else if (!request.isError()) { + throw new RuntimeException("Invalid response: " + request.returnValues()); + } else if (temporaryError(request)) { + log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted sleep between retries of waitFor", e); + } + } else { + throw new RuntimeException("Wait for " + urlReference + " failed: " + request.errorMessage() + " (" + request.errorCode() + ")"); + } + timeLeft = start + timeout - System.currentTimeMillis() / 1000; + } while (timeLeft > 0); + + throw new RuntimeException("Timed out waiting for " + urlReference + " after " + timeout); + } + +} |