aboutsummaryrefslogtreecommitdiffstats
path: root/config
diff options
context:
space:
mode:
Diffstat (limited to 'config')
-rw-r--r--config/abi-spec.json299
-rwxr-xr-xconfig/pom.xml4
-rw-r--r--config/src/apps/vespa-configproxy-cmd/main.cpp2
-rw-r--r--config/src/apps/vespa-configproxy-cmd/methods.cpp2
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java39
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java7
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java98
7 files changed, 440 insertions, 11 deletions
diff --git a/config/abi-spec.json b/config/abi-spec.json
new file mode 100644
index 00000000000..f6ab43f2c9b
--- /dev/null
+++ b/config/abi-spec.json
@@ -0,0 +1,299 @@
+{
+ "com.yahoo.config.subscription.CfgConfigPayloadBuilder": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public com.yahoo.vespa.config.ConfigPayload deserialize(java.util.List)",
+ "public com.yahoo.vespa.config.ConfigPayloadBuilder deserializeToBuilder(java.util.List)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigDebug": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public static void logDebug(java.util.logging.Logger, long, com.yahoo.vespa.config.ConfigKey, java.lang.String)",
+ "public static void logDebug(java.util.logging.Logger, com.yahoo.config.ConfigInstance$Builder, java.lang.String, java.lang.String)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigGetter": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.lang.Class)",
+ "public void <init>(com.yahoo.config.subscription.ConfigSource, java.lang.Class)",
+ "public synchronized com.yahoo.config.ConfigInstance getConfig(java.lang.String)",
+ "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String)",
+ "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String, com.yahoo.config.subscription.ConfigSource)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigHandle": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "protected void <init>(com.yahoo.config.subscription.impl.ConfigSubscription)",
+ "public boolean isChanged()",
+ "public com.yahoo.config.ConfigInstance getConfig()",
+ "public java.lang.String toString()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigInstanceSerializer": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.Serializer"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(com.yahoo.slime.Slime)",
+ "public void <init>(com.yahoo.slime.Slime, com.yahoo.slime.Cursor)",
+ "public com.yahoo.config.Serializer createInner(java.lang.String)",
+ "public com.yahoo.config.Serializer createArray(java.lang.String)",
+ "public com.yahoo.config.Serializer createInner()",
+ "public com.yahoo.config.Serializer createMap(java.lang.String)",
+ "public void serialize(java.lang.String, boolean)",
+ "public void serialize(java.lang.String, double)",
+ "public void serialize(java.lang.String, int)",
+ "public void serialize(java.lang.String, long)",
+ "public void serialize(java.lang.String, java.lang.String)",
+ "public void serialize(boolean)",
+ "public void serialize(double)",
+ "public void serialize(long)",
+ "public void serialize(int)",
+ "public void serialize(java.lang.String)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigInstanceUtil": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public static void setValues(com.yahoo.config.ConfigBuilder, com.yahoo.config.ConfigBuilder)",
+ "public static com.yahoo.config.ConfigInstance getNewInstance(java.lang.Class, java.lang.String, com.yahoo.vespa.config.ConfigPayload)",
+ "public static java.lang.Object getField(com.yahoo.config.ConfigBuilder, java.lang.String)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigInterruptedException": {
+ "superClass": "java.lang.RuntimeException",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.lang.Throwable)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigSet": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public void addBuilder(java.lang.String, com.yahoo.config.ConfigInstance$Builder)",
+ "public com.yahoo.config.ConfigInstance$Builder get(com.yahoo.vespa.config.ConfigKey)",
+ "public boolean contains(com.yahoo.vespa.config.ConfigKey)",
+ "public java.lang.String toString()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigSource": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public",
+ "interface",
+ "abstract"
+ ],
+ "methods": [],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigSourceSet": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public void <init>(java.util.List)",
+ "public void <init>(java.lang.String[])",
+ "public void <init>(java.lang.String)",
+ "public java.util.Set getSources()",
+ "public boolean equals(java.lang.Object)",
+ "public int hashCode()",
+ "public java.lang.String toString()",
+ "public static com.yahoo.config.subscription.ConfigSourceSet createDefault()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public",
+ "interface",
+ "abstract"
+ ],
+ "methods": [
+ "public abstract void configure(com.yahoo.config.ConfigInstance)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.ConfigSubscriber$State": {
+ "superClass": "java.lang.Enum",
+ "interfaces": [],
+ "attributes": [
+ "public",
+ "final",
+ "enum"
+ ],
+ "methods": [
+ "public static com.yahoo.config.subscription.ConfigSubscriber$State[] values()",
+ "public static com.yahoo.config.subscription.ConfigSubscriber$State valueOf(java.lang.String)"
+ ],
+ "fields": [
+ "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State OPEN",
+ "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State FROZEN",
+ "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State CLOSED"
+ ]
+ },
+ "com.yahoo.config.subscription.ConfigSubscriber": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public void <init>(com.yahoo.config.subscription.ConfigSource)",
+ "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String)",
+ "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String, long)",
+ "protected void checkStateBeforeSubscribe()",
+ "protected void subscribeAndHandleErrors(com.yahoo.config.subscription.impl.ConfigSubscription, com.yahoo.vespa.config.ConfigKey, com.yahoo.config.subscription.ConfigHandle, com.yahoo.vespa.config.TimingValues)",
+ "public boolean nextConfig()",
+ "public boolean nextConfig(long)",
+ "public boolean nextGeneration()",
+ "public boolean nextGeneration(long)",
+ "protected void throwIfExceptionSet(com.yahoo.config.subscription.impl.ConfigSubscription)",
+ "public void close()",
+ "protected void closeRequesters()",
+ "public java.lang.String toString()",
+ "public java.lang.Thread startConfigThread(java.lang.Runnable)",
+ "protected com.yahoo.config.subscription.ConfigSubscriber$State state()",
+ "public void reload(long)",
+ "public com.yahoo.config.subscription.ConfigSource getSource()",
+ "public java.util.Map requesters()",
+ "public boolean isClosed()",
+ "public com.yahoo.config.subscription.ConfigHandle subscribe(com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber, java.lang.Class, java.lang.String)",
+ "public long getGeneration()",
+ "public boolean isInternalRedeploy()",
+ "protected void finalize()"
+ ],
+ "fields": [
+ "protected final java.util.List subscriptionHandles",
+ "protected java.util.Map requesters"
+ ]
+ },
+ "com.yahoo.config.subscription.ConfigURI": {
+ "superClass": "java.lang.Object",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public java.lang.String getConfigId()",
+ "public com.yahoo.config.subscription.ConfigSource getSource()",
+ "public static com.yahoo.config.subscription.ConfigURI createFromId(java.lang.String)",
+ "public static com.yahoo.config.subscription.ConfigURI createFromIdAndSource(java.lang.String, com.yahoo.config.subscription.ConfigSource)"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.DirSource": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.io.File)",
+ "public java.io.File getDir()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.FileSource": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.io.File)",
+ "public java.io.File getFile()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.JarSource": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.util.jar.JarFile, java.lang.String)",
+ "public java.util.jar.JarFile getJarFile()",
+ "public java.lang.String getPath()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.config.subscription.RawSource": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.subscription.ConfigSource"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.lang.String)"
+ ],
+ "fields": [
+ "public final java.lang.String payload"
+ ]
+ }
+} \ No newline at end of file
diff --git a/config/pom.xml b/config/pom.xml
index e1e04e17033..ea5948a30bb 100755
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -192,6 +192,10 @@
<updateReleaseInfo>true</updateReleaseInfo>
</configuration>
</plugin>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>abi-check-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/config/src/apps/vespa-configproxy-cmd/main.cpp b/config/src/apps/vespa-configproxy-cmd/main.cpp
index 63fa1285bb2..bb908e7268b 100644
--- a/config/src/apps/vespa-configproxy-cmd/main.cpp
+++ b/config/src/apps/vespa-configproxy-cmd/main.cpp
@@ -22,7 +22,7 @@ Application::parseOpts()
char c = '?';
const char *optArg = NULL;
int optInd = 0;
- while ((c = GetOpt("m:s:p:", optArg, optInd)) != -1) {
+ while ((c = GetOpt("m:s:p:h", optArg, optInd)) != -1) {
switch (c) {
case 'm':
_flags.method = optArg;
diff --git a/config/src/apps/vespa-configproxy-cmd/methods.cpp b/config/src/apps/vespa-configproxy-cmd/methods.cpp
index 6a143667102..6218c54e8b3 100644
--- a/config/src/apps/vespa-configproxy-cmd/methods.cpp
+++ b/config/src/apps/vespa-configproxy-cmd/methods.cpp
@@ -14,7 +14,7 @@ const Method methods[] = {
{ "cachefull", "listCachedConfigFull", 0 },
{ "sources", "listSourceConnections", 0 },
{ "statistics", "printStatistics", 0 },
- { "setmode", "setMode", 1 }, // { default | memorycache | diskcache }
+ { "setmode", "setMode", 1 }, // { default | memorycache }
{ "updatesources", "updateSources", 1 },
{ 0, 0, 0}
};
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
index 8a12405d505..40d79afc854 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
@@ -20,7 +20,11 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.nio.file.Path;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
import java.util.logging.Logger;
/**
@@ -36,15 +40,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
private final ConfigInstance.Builder rootBuilder;
private final ConfigTransformer.PathAcquirer pathAcquirer;
+ private final UrlDownloader urlDownloader;
private final Stack<NamedBuilder> stack = new Stack<>();
public ConfigPayloadApplier(T builder) {
- this(builder, new IdentityPathAcquirer());
+ this(builder, new IdentityPathAcquirer(), null);
}
- public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) {
+ public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer, UrlDownloader urlDownloader) {
this.rootBuilder = builder;
this.pathAcquirer = pathAcquirer;
+ this.urlDownloader = urlDownloader;
debug("rootBuilder=" + rootBuilder);
}
@@ -207,6 +213,12 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
if (isPathField(builder, methodName)) {
FileReference wrappedPath = resolvePath((String)value);
invokeSetter(builder, methodName, key, wrappedPath);
+
+ // Need to convert url into actual file if 'url' type is used
+ } else if (isUrlField(builder, methodName)) {
+ UrlReference url = resolveUrl((String)value);
+ invokeSetter(builder, methodName, key, url);
+
} else {
invokeSetter(builder, methodName, key, value);
}
@@ -258,7 +270,8 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
// Need to convert url into actual file if 'url' type is used
} else if (isUrlField(builder, methodName)) {
- throw new UnsupportedOperationException("'url' type is not yet implemented");
+ UrlReference url = resolveUrl(Utf8.toString(value.asUtf8()));
+ invokeSetter(builder, methodName, url);
} else {
Object object = getValueFromInspector(value);
@@ -276,6 +289,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
return newFileReference(path.toString());
}
+ private UrlReference resolveUrl(String url) {
+ if (urlDownloader == null || !urlDownloader.isValid()) {
+ throw new RuntimeException("Resolving url field failed due to missing or invalid URL downloader.");
+ }
+ File file = urlDownloader.waitFor(new UrlReference(url), 60 * 60);
+ return new UrlReference(file.getAbsolutePath());
+ }
+
private FileReference newFileReference(String fileReference) {
try {
Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class);
@@ -343,18 +364,19 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
* Checks whether or not this field is of type 'path', in which
* case some special handling might be needed. Caches the result.
*/
+ private Set<String> pathFieldSet = new HashSet<>();
private boolean isPathField(Object builder, String methodName) {
// Paths are stored as FileReference in Builder.
- return isFieldType(builder, methodName, FileReference.class);
+ return isFieldType(pathFieldSet, builder, methodName, FileReference.class);
}
+ private Set<String> urlFieldSet = new HashSet<>();
private boolean isUrlField(Object builder, String methodName) {
// Urls are stored as UrlReference in Builder.
- return isFieldType(builder, methodName, UrlReference.class);
+ return isFieldType(urlFieldSet, builder, methodName, UrlReference.class);
}
- private Set<String> fieldSet = new HashSet<>();
- private boolean isFieldType(Object builder, String methodName, java.lang.reflect.Type type) {
+ private boolean isFieldType(Set<String> fieldSet, Object builder, String methodName, java.lang.reflect.Type type) {
String key = fieldKey(builder, methodName);
if (fieldSet.contains(key)) {
return true;
@@ -515,4 +537,5 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
return new File(fileReference.value()).toPath();
}
}
+
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
index ac3f490182a..163e010cdd6 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
@@ -26,6 +26,7 @@ public class ConfigTransformer<T extends ConfigInstance> {
private final Class<T> clazz;
private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer();
+ private static volatile UrlDownloader urlDownloader;
/**
* For internal use only *
@@ -36,6 +37,10 @@ public class ConfigTransformer<T extends ConfigInstance> {
pathAcquirer;
}
+ public static void setUrlDownloader(UrlDownloader urlDownloader) {
+ ConfigTransformer.urlDownloader = urlDownloader;
+ }
+
/**
* Create a transformer capable of converting payloads to clazz
*
@@ -53,7 +58,7 @@ public class ConfigTransformer<T extends ConfigInstance> {
*/
public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) {
ConfigInstance.Builder builder = getRootBuilder();
- ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer);
+ ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer, urlDownloader);
creator.applyPayload(payload);
return builder;
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java
new file mode 100644
index 00000000000..4947b618f50
--- /dev/null
+++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java
@@ -0,0 +1,98 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config;
+
+import com.yahoo.config.UrlReference;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.io.File;
+import java.util.logging.Logger;
+
+/**
+ * @author lesters
+ */
+public class UrlDownloader {
+
+ private static final Logger log = Logger.getLogger(UrlDownloader.class.getName());
+
+ private static final int BASE_ERROR_CODE = 0x10000;
+ public static final int DOES_NOT_EXIST = BASE_ERROR_CODE + 1;
+ public static final int INTERNAL_ERROR = BASE_ERROR_CODE + 2;
+ public static final int HTTP_ERROR = BASE_ERROR_CODE + 3;
+
+ private final Supervisor supervisor = new Supervisor(new Transport());
+ private final Spec spec;
+ private Target target;
+
+ public UrlDownloader() {
+ spec = new Spec(Defaults.getDefaults().vespaHostname(), Defaults.getDefaults().vespaConfigProxyRpcPort());
+ connect();
+ }
+
+ public void shutdown() {
+ supervisor.transport().shutdown().join();
+ }
+
+ private void connect() {
+ int timeRemaining = 5000;
+ try {
+ while (timeRemaining > 0) {
+ target = supervisor.connectSync(spec);
+ if (target.isValid()) {
+ log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this));
+ return;
+ }
+ Thread.sleep(500);
+ timeRemaining -= 500;
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public boolean isValid() {
+ return target.isValid();
+ }
+
+ private boolean temporaryError(Request req) {
+ return false; // Currently, none of the errors are considered temporary
+ }
+
+ public File waitFor(UrlReference urlReference, long timeout) {
+ long start = System.currentTimeMillis() / 1000;
+ long timeLeft = timeout;
+ do {
+ Request request = new Request("url.waitFor");
+ request.parameters().add(new StringValue(urlReference.value()));
+
+ double rpcTimeout = Math.min(timeLeft, 60 * 60.0);
+ log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout");
+ target.invokeSync(request, rpcTimeout);
+
+ if (request.checkReturnTypes("s")) {
+ return new File(request.returnValues().get(0).asString());
+ } else if (!request.isError()) {
+ throw new RuntimeException("Invalid response: " + request.returnValues());
+ } else if (temporaryError(request)) {
+ log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage());
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted sleep between retries of waitFor", e);
+ }
+ } else {
+ throw new RuntimeException("Wait for " + urlReference + " failed: " + request.errorMessage() + " (" + request.errorCode() + ")");
+ }
+ timeLeft = start + timeout - System.currentTimeMillis() / 1000;
+ } while (timeLeft > 0);
+
+ throw new RuntimeException("Timed out waiting for " + urlReference + " after " + timeout);
+ }
+
+}