diff options
Diffstat (limited to 'config/src/main')
103 files changed, 10955 insertions, 0 deletions
diff --git a/config/src/main/java/.gitignore b/config/src/main/java/.gitignore new file mode 100644 index 00000000000..4cb44b1b2b5 --- /dev/null +++ b/config/src/main/java/.gitignore @@ -0,0 +1 @@ +/deploy diff --git a/config/src/main/java/com/yahoo/config/codegen/package-info.java b/config/src/main/java/com/yahoo/config/codegen/package-info.java new file mode 100644 index 00000000000..b83f806cf87 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/codegen/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.codegen; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java new file mode 100644 index 00000000000..1216efcbec5 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.collections.Pair; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.StringNode; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; + +import java.util.*; + +/** + * Deserializes config payload (cfg format) to a ConfigPayload. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.6 + */ +public class CfgConfigPayloadBuilder { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(CfgConfigPayloadBuilder.class.getName()); + + /** + * Deserializes a config payload to slime + * + * @param lines a list with config payload strings + * @return an instance of the config class + */ + public ConfigPayload deserialize(List<String> lines) { + return ConfigPayload.fromBuilder(deserializeToBuilder(lines)); + } + + public ConfigPayloadBuilder deserializeToBuilder(List<String> lines) { + int lineNum = 1; + ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(); + for (String line : lines) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "line " + lineNum + ": '" + line + "'"); + } + parseLine(line, lineNum, payloadBuilder); + lineNum++; + } + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "payload=" + payloadBuilder.toString()); + } + return payloadBuilder; + } + + private void parseLine(final String line, int lineNum, ConfigPayloadBuilder payloadBuilder) { + String trimmedLine = line.trim(); + if (trimmedLine.startsWith("#")) return; + Pair<String, String> fieldAndValue = parseFieldAndValue(trimmedLine); + String field = fieldAndValue.getFirst(); + String value = fieldAndValue.getSecond(); + if (field==null || value==null) { + log.log(LogLevel.DEBUG, "Got field without value in line " + lineNum + ": " + line + ", skipping"); + return; + } + field=field.trim(); + value=value.trim(); + validateField(field, trimmedLine, lineNum); + validateValue(value, trimmedLine, lineNum); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "field=" + field + ",value=" + value); + } + List<String> fields = parseFieldList(field); + ConfigPayloadBuilder currentBuilder = payloadBuilder; + for (int fieldNum = 0; fieldNum < fields.size(); fieldNum++) { + String fieldName = fields.get(fieldNum); + boolean isLeaf = (fieldNum == fields.size() - 1); + if (isLeaf) { + if (isArray(fieldName)) { + // array leaf + ConfigPayloadBuilder.Array array = currentBuilder.getArray(getArrayName(fieldName)); + array.set(getArrayIndex(fieldName), removeQuotes(value)); + } else if (isMap(fieldName)) { + // map leaf + ConfigPayloadBuilder.MapBuilder map = currentBuilder.getMap(getMapName(fieldName)); + map.put(getMapKey(fieldName), removeQuotes(value)); + } else { + // scalar leaf value + currentBuilder.setField(fieldName, removeQuotes(value)); + } + } else { + if (isArray(fieldName)) { + // array of structs + ConfigPayloadBuilder.Array array = currentBuilder.getArray(getArrayName(fieldName)); + currentBuilder = array.get(getArrayIndex(fieldName)); + } else if (isMap(fieldName)) { + // map of structs + ConfigPayloadBuilder.MapBuilder map = currentBuilder.getMap(getMapName(fieldName)); + currentBuilder = map.get(getMapKey(fieldName)); + } else { + // struct + currentBuilder = currentBuilder.getObject(fieldName); + } + } + } + } + + // split on space, but not if inside { } (map key) + Pair<String, String> parseFieldAndValue(String line) { + String field=null; + String value; + StringBuffer sb = new StringBuffer(); + boolean inMapKey = false; + for (char c : line.toCharArray()) { + if (c=='{') inMapKey=true; + if (c=='}') inMapKey=false; + if (c==' ' && !inMapKey) { + if (field==null) { + field = sb.toString(); + sb = new StringBuffer(); + continue; + } + } + sb.append(c); + } + value = sb.toString(); + return new Pair<>(field, value); + } + + // split on dot, but not if inside { } (map key) + List<String> parseFieldList(String field) { + List<String> ret = new ArrayList<>(); + StringBuffer sb = new StringBuffer(); + boolean inMapKey = false; + for (char c : field.toCharArray()) { + if (c=='{') inMapKey=true; + if (c=='}') inMapKey=false; + if (c=='.' && !inMapKey) { + ret.add(sb.toString()); + sb = new StringBuffer(); + continue; + } + sb.append(c); + } + ret.add(sb.toString()); + return ret; + } + + // TODO Need more validation + private void validateField(String field, String line, int lineNum) { + if (field.length() == 0) { + throw new ConfigurationRuntimeException("Error on line " + lineNum + ": " + line + "\n" + + "'" + field + "' is not a valid field name"); + } + } + + // TODO Need more validation + private void validateValue(String value, String line, int lineNum) { + if (value.length() == 0) { + throw new ConfigurationRuntimeException("Error on line " + lineNum + ": " + line + "\n" + + "'" + value + "' is not a valid value"); + } + } + + private boolean isArray(String name) { + return name.endsWith("]"); + } + + private boolean isMap(String name) { + return name.contains("{"); + } + + private String removeQuotes(String s) { + return StringNode.unescapeQuotedString(s); + } + + private String getMapName(String name) { + if (name.contains("{")) { + return name.substring(0, name.indexOf("{")); + } else { + return name; + } + } + + private String getMapKey(String name) { + if (name.contains("{")) { + return removeQuotes(name.substring(name.indexOf("{") + 1, name.indexOf("}"))); + } else { + return ""; + } + } + + private String getArrayName(String name) { + if (name.contains("[")) { + return name.substring(0, name.indexOf("[")); + } else { + return name; + } + } + + private int getArrayIndex(String name) { + if (name.contains("[")) { + return Integer.parseInt(name.substring(name.indexOf("[") + 1, name.indexOf("]"))); + } else { + return 0; + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java new file mode 100644 index 00000000000..add088cf349 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.logging.Logger; + +// Debug class that provides useful helper routines +public class ConfigDebug { + public static void logDebug(Logger logger, long timestamp, ConfigKey<?> key, String logmessage) { + if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { + logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); + } + } + + public static void logDebug(Logger log, ConfigInstance.Builder builder, String configId, String logmessage) { + ConfigKey<?> key = new ConfigKey<>(builder.getDefName(), configId, builder.getDefNamespace()); + logDebug(log, 0, key, logmessage); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java new file mode 100755 index 00000000000..be4ff9f1b79 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + + +import com.yahoo.config.ConfigInstance; + +/** + * This is a simple config getter that retrieves a config with a given class and configId through a + * simple method call. No subscription is retained when the config has been returned to the client. + * + * This class is mainly targeted to unit tests that do not want the extra complexity incurred by setting + * up their own subscriber. Another use-case is clients that get config, do a task, and exit, e.g. + * command-line tools. + * + * @author gjoranv + */ +public class ConfigGetter<T extends ConfigInstance> { + + private final Class<T> clazz; + private final ConfigSource source; + + /** + * Creates a ConfigGetter for class <code>clazz</code> + * + * @param clazz a config class + */ + public ConfigGetter(Class<T> clazz) { + this(null, clazz); + } + + /** + * Creates a ConfigGetter for class <code>clazz</code> with the specified + * {@link ConfigSource}. + * + * @param source a {@link ConfigSource} + * @param clazz a config class + */ + // TODO This is the order of arguments in com.yahoo.config.ConfigGetter and kept here, I would like to switch the order + public ConfigGetter(ConfigSource source, Class<T> clazz) { + this.clazz = clazz; + this.source = source; + } + + /** + * Returns an instance of the config class specified in the constructor. + * + * @param configId a config id to use when getting the config + * @return an instance of a config class + */ + public synchronized T getConfig(String configId) { + ConfigSubscriber subscriber; + ConfigHandle<T> h; + if (source == null) { + subscriber = new ConfigSubscriber(); + } else { + subscriber = new ConfigSubscriber(source); + } + h = subscriber.subscribe(clazz, configId); + subscriber.nextConfig(); + T ret = h.getConfig(); + subscriber.close(); + return ret; + } + + /** + * Creates a ConfigGetter instance and returns an instance of the config class <code>c</code>. + * + * @param c a config class + * @param configId a config id to use when getting the config + * @return an instance of a config class + */ + public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId) { + ConfigGetter<T> getter = new ConfigGetter<T>(c); + return getter.getConfig(configId); + } + + /** + * Creates a ConfigGetter instance and returns an instance of the config class <code>c</code>. + * + * @param c a config class + * @param configId a config id to use when getting the config + * @param source a {@link ConfigSource} + * @return an instance of a config class + */ + public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId, ConfigSource source) { + ConfigGetter<T> getter = new ConfigGetter<T>(source, c); + return getter.getConfig(configId); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java new file mode 100644 index 00000000000..6cc10be8627 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.ConfigSubscription; + +/** + * A config handle represents one config in the context of one active subscription on a {@link ConfigSubscriber}. + * It will contain meta data of the subscription of that particular config, as well as access to the {@link com.yahoo.config.ConfigInstance} itself. + * + * @param <T> the type of the config + * @author vegardh + * @since 5.1 + */ +public class ConfigHandle<T extends ConfigInstance> { + + private ConfigSubscription<T> sub; + private boolean changed = false; + + protected ConfigHandle(ConfigSubscription<T> sub) { + this.sub = sub; + } + + /** + * Returns true if: + * + * The config generation for the {@link ConfigSubscriber} that produced this is the first one in its life cycle. (Typically first time config.) + * or + * All configs for the subscriber have a new generation since the last time nextConfig() was called + * AND it's the same generation AND there is a change in <strong>this</strong> handle's config. + * (Typically calls for a reconfig.) + * + * @return there is a new config + */ + public boolean isChanged() { + return changed; + } + + void setChanged(boolean changed) { + this.changed = changed; + } + + ConfigSubscription<T> subscription() { + return sub; + } + + /** + * The config of this handle + * + * @return the config that this handle holds + */ + public T getConfig() { + // TODO throw if subscriber not frozen? + return sub.getConfig(); + } + + @Override + public String toString() { + return "Handle changed: " + changed + "\nSub:\n" + sub.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java new file mode 100644 index 00000000000..cdb654fcbbb --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.Serializer; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; + +/** + * Implements a config instance serializer, serializing a config instance to a slime object. + * + * @author lulf + * @since 5.1.14 + */ +public class ConfigInstanceSerializer implements Serializer { + private final Slime slime; + private final Cursor root; + public ConfigInstanceSerializer(Slime slime) { + this.slime = slime; + root = slime.setObject(); + } + + public ConfigInstanceSerializer(Slime slime, Cursor root) { + this.slime = slime; + this.root = root; + } + + @Override + public Serializer createInner(String name) { + Cursor childRoot = root.setObject(name); + return new ConfigInstanceSerializer(slime, childRoot); + } + + @Override + public Serializer createArray(String name) { + return new ConfigInstanceSerializer(slime, root.setArray(name)); + } + + @Override + public Serializer createInner() { + return new ConfigInstanceSerializer(slime, root.addObject()); + } + + @Override + public Serializer createMap(String name) { + return createInner(name); + } + + public void serialize(String name, boolean value) { + root.setBool(name, value); + } + + public void serialize(String name, double value) { + root.setDouble(name, value); + } + + public void serialize(String name, int value) { + root.setLong(name, value); + } + + public void serialize(String name, long value) { + root.setLong(name, value); + } + + public void serialize(String name, String value) { + root.setString(name, value); + } + + @Override + public void serialize(boolean value) { + root.addBool(value); + } + + @Override + public void serialize(double value) { + root.addDouble(value); + } + + @Override + public void serialize(long value) { + root.addLong(value); + } + + @Override + public void serialize(int value) { + root.addLong(value); + } + + @Override + public void serialize(String value) { + root.addString(value); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java new file mode 100644 index 00000000000..3c36bf7f105 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.*; + +/** + * @author gjoranv + * @since 5.1.6 + */ +public class ConfigInstanceUtil { + + /** + * Copies all values that have been explicitly set on the source to the destination. + * Values that have not been explicitly set in the source builder, will be left unchanged + * in the destination. + * + * @param destination The builder to copy values into. + * @param source The builder to copy values from. Unset values are not copied. + * @param <BUILDER> The builder class. + */ + public static<BUILDER extends ConfigBuilder> void setValues(BUILDER destination, BUILDER source) { + try { + Method setter = destination.getClass().getDeclaredMethod("override", destination.getClass()); + setter.setAccessible(true); + setter.invoke(destination, source); + setter.setAccessible(false); + } catch (Exception e) { + throw new ConfigurationRuntimeException("Could not set values on config builder." + + destination.getClass().getName(), e); + } + } + + public static <T extends ConfigInstance> T getNewInstance(Class<T> type, + String configId, + ConfigPayload payload) { + T instance; + try { + ConfigTransformer<?> transformer = new ConfigTransformer<T>(type); + ConfigBuilder instanceBuilder = transformer.toConfigBuilder(payload); + Constructor<T> constructor = type.getConstructor(instanceBuilder.getClass()); + instance = constructor.newInstance((ConfigInstance.Builder) instanceBuilder); + + // Workaround for JDK7, where compilation fails due to fields being + // private and not accessible from T. Reference it as a + // ConfigInstance to work around it. See + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7022052 for + // more information. + ConfigInstance i = instance; + i.postInitialize(configId); + setConfigId(i, configId); + + } catch (InstantiationException | InvocationTargetException | NoSuchMethodException | + NoSuchFieldException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed creating new instance of '" + type.getCanonicalName() + + "' for config id '" + configId + "': " + Exceptions.toMessageString(e), e); + } + return instance; + } + + private static void setConfigId(ConfigInstance instance, String configId) + throws NoSuchFieldException, IllegalAccessException { + Field configIdField = ConfigInstance.class.getDeclaredField("configId"); + configIdField.setAccessible(true); + configIdField.set(instance, configId); + configIdField.setAccessible(false); + } + + /** + * Gets the value of a private field on a Builder. + * @param builder a {@link com.yahoo.config.ConfigBuilder} + * @param fieldName a config field name + * @return the value of the private field + */ + public static Object getField(ConfigBuilder builder, String fieldName) { + try { + Field f = builder.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + return f.get(builder); + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java new file mode 100644 index 00000000000..ed1dca50096 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * This exception is thrown when any blocking call within the Config API is interrupted. + * @author lulf + * @since 5.1 + */ +@SuppressWarnings("serial") +public class ConfigInterruptedException extends RuntimeException { + public ConfigInterruptedException(Throwable cause) { + super(cause); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java new file mode 100644 index 00000000000..1b516f333fa --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +/** + * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s + * + * @author vegardh + * @since 5.1 + */ +public class ConfigSet implements ConfigSource { + private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); + + /** + * Inserts a new builder in this set. If an existing entry exists, it is overwritten. + * + * @param configId The config id for this builder. + * @param builder The builder that will produce config for the particular config id. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public void addBuilder(String configId, ConfigInstance.Builder builder) { + Class<?> configClass = builder.getClass().getDeclaringClass(); + //System.out.println("Declaring class for builder " + builder + " is " + configClass); + ConfigKey<?> key = new ConfigKey(configClass, configId); + configs.put(key, builder); + } + + /** + * Returns a Builder matching the given key, or null if no match + * + * @param key a config key to get a Builder for + * @return a ConfigInstance + */ + public ConfigInstance.Builder get(ConfigKey<?> key) { + return configs.get(key); + } + + /** + * Returns true if this set contains a config instance matching the given key + * + * @param key a config key + * @return a ConfigInstance + */ + public boolean contains(ConfigKey<?> key) { + return configs.containsKey(key); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<ConfigKey<?>, ConfigInstance.Builder> entry : configs.entrySet()) { + sb.append(entry.getKey()).append("=>").append(entry.getValue()); + } + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java new file mode 100644 index 00000000000..44b65e4ba12 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * A type of source of config + * @author vegardh + * @since 5.1 + * + */ +public interface ConfigSource { + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java new file mode 100755 index 00000000000..4c1757c0ba8 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.log.LogLevel; + +import java.util.*; +import java.util.logging.Logger; + + +/** +* An immutable set of connection endpoints, where each endpoint points to either a + * remote configserver or a configproxy. + * + * Two sets are said to be equal if they contain the same sources, independent of order, + * upper/lower-casing and whitespaces. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigSourceSet implements ConfigSource +{ + private static final Logger log = Logger.getLogger(ConfigSourceSet.class.getName()); + private final Set<String> sources = new LinkedHashSet<String>(); + + /** + * Creates an empty ConfigSourceSet, mostly used for unit testing. + */ + public ConfigSourceSet() { + } + + /** + * Creates a ConfigSourceSet containing all the unique given input addresses. + * Each address is trimmed and lower-cased before adding. + * + * @param addresses Connection endpoints on the format "tcp/host:port". + */ + public ConfigSourceSet(List<String> addresses) { + for (String a : addresses) { + sources.add(a.trim().toLowerCase()); + } + } + + /** + * Creates a ConfigSourceSet containing all the unique given input addresses. + * Each address is trimmed and lower-cased before adding. + * + * @param addresses Connection endpoints on the format "tcp/host:port". + */ + public ConfigSourceSet(String[] addresses) { + this(Arrays.asList(addresses)); + } + + /** + * Convenience constructor to create a ConfigSourceSet with only one input address. + * + * @param address Connection endpoint on the format "tcp/host:port". + */ + public ConfigSourceSet(String address) { + this(new String[] {address}); + } + + /** + * Returns an unmodifiable set containing all sources in this ConfigSourceSet. Iteration order is + * guaranteed to be the same as that of the list or array that was given when this set was created. + * + * @return All sources in this ConfigSourceSet. + */ + public Set<String> getSources() { + return Collections.unmodifiableSet(sources); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof ConfigSourceSet)) { + return false; + } + ConfigSourceSet css = (ConfigSourceSet)o; + return sources.equals(css.sources); + } + + public int hashCode() { + return sources.hashCode(); + } + + public String toString() { + return sources.toString(); + } + + /** + * Create a new source set using the environment variables or system properties + * @return a new source set if available, null if not. + */ + public static ConfigSourceSet createDefault() { + String configSources = System.getenv("VESPA_CONFIG_SOURCES"); + if (configSources != null) { + log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + return new ConfigSourceSet(checkSourcesSyntax(configSources)); + } else { + String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; + String[] sourceSet = checkSourcesSyntax(System.getProperty("configsources")); + return new ConfigSourceSet(sourceSet == null ? def : sourceSet); + } + } + + /** + * Check sources syntax and convert it to a proper source set by checking if + * sources start with the required "tcp/" prefix and add that prefix if not. + * + * @param sources a source set as a comma-separated string + * @return a String array with sources, or null if the input source set was null + */ + private static String[] checkSourcesSyntax(String sources) { + String[] sourceSet = null; + if (sources != null) { + sourceSet = sources.split(","); + int i = 0; + for (String s : sourceSet) { + if (!s.startsWith("tcp/")) { + sourceSet[i] = "tcp/" + sourceSet[i]; + } + i++; + } + } + return sourceSet; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java new file mode 100644 index 00000000000..2322726057e --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -0,0 +1,449 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.impl.ConfigSubscription; +import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; + +/** + * Used for subscribing to one or more configs. Can optionally be given a {@link ConfigSource} for the configs + * that will be used when {@link #subscribe(Class, String)} is called. + * + * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the + * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. + * + * @author vegardh + * @since 5.1 + */ +public class ConfigSubscriber { + private Logger log = Logger.getLogger(getClass().getName()); + private State state = State.OPEN; + protected List<ConfigHandle<? extends ConfigInstance>> subscriptionHandles = new ArrayList<>(); + private final ConfigSource source; + private long generation = -1; + + /** + * Reuse requesters for equal source sets, limit number if many subscriptions. + */ + protected Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); + + /** + * The states of the subscriber. Affects the validity of calling certain methods. + * + */ + protected enum State { + OPEN, FROZEN, CLOSED + } + + /** + * Constructs a new subscriber. The default Vespa network config source will be used, which is the address of + * a config proxy (part of vespa_base) running locally. It can also be changed by setting VESPA_CONFIG_SOURCES. + */ + public ConfigSubscriber() { + this(JRTConfigRequester.defaultSourceSet); + } + + /** + * Constructs a new subscriber with the given source. + * + * @param source a {@link ConfigSource} that will be used when {@link #subscribe(Class, String)} is called. + */ + public ConfigSubscriber(ConfigSource source) { + this.source = source; + } + + /** + * Subscribes on the given type of {@link ConfigInstance} with the given config id. + * + * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml, or null if you are using a local {@link ConfigSource} which does not use config id. + * Also supported: raw:, file:, dir: or jar: config id which addresses config locally in the same way. + * + * @return a ConfigHandle + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId) { + return subscribe(configClass, configId, source, new TimingValues()); + } + + /** + * Subscribes on the given type of {@link ConfigInstance} with the given config id and subscribe timeout. + * + * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml, or possibly raw:, file:, dir: or jar: type config which addresses config locally. + * @param timeoutMillis The time to wait for a config to become available, in milliseconds + * @return a ConfigHandle + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, long timeoutMillis) { + return subscribe(configClass, configId, source, new TimingValues().setSubscribeTimeout(timeoutMillis)); + } + + // for testing + <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, ConfigSource source, TimingValues timingValues) { + checkStateBeforeSubscribe(); + final ConfigKey<T> configKey = new ConfigKey<>(configClass, configId); + ConfigSubscription<T> sub = ConfigSubscription.get(configKey, this, source, timingValues); + ConfigHandle<T> handle = new ConfigHandle<>(sub); + subscribeAndHandleErrors(sub, configKey, handle, timingValues); + return handle; + } + + protected void checkStateBeforeSubscribe() { + if (state != State.OPEN) + throw new IllegalStateException("Adding subscription after calling nextConfig() is not allowed"); + } + + protected void subscribeAndHandleErrors(ConfigSubscription<?> sub, ConfigKey<?> configKey, ConfigHandle<?> handle, TimingValues timingValues) { + subscriptionHandles.add(handle); + // Must block here until something available from the subscription, so we know that it offers something when the user calls nextConfig + boolean subOk = sub.subscribe(timingValues.getSubscribeTimeout()); + throwIfExceptionSet(sub); + if (!subOk) { + //sub.close(); + //subscriptionHandles.remove(handle); + throw new ConfigurationRuntimeException("Subscribe for '" + configKey + "' timed out (timeout was " + timingValues.getSubscribeTimeout() + " ms): " + sub); + } + } + + /** + * Use this for waiting for a new config that has changed. + * + * Returns true if: + * + * It is the first time nextConfig() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextConfig() was called, AND they have the same generation AND there is a change in config for at least one + * of the configs. (Typically calls for a reconfig.) + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @return true if a config/reconfig of your system should happen + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextConfig() { + return nextConfig(TimingValues.defaultNextConfigTimeout); + } + + /** + * Use this for waiting for a new config that has changed, with the given timeout. + * + * Returns true if: + * + * It is the first time nextConfig() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextConfig() was called, AND they have the same generation AND there is a change in config for at least one + * of the configs. (Typically calls for a reconfig.) + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out, no handle will have the changed flag set. You should not configure anything then. + * + * @param timeoutMillis timeout in milliseconds + * @return true if a config/reconfig of your system should happen + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextConfig(long timeoutMillis) { + return acquireSnapshot(timeoutMillis, true); + } + + /** + * Use this for waiting for a new config generation. + * + * Returns true if: + * + * It is the first time nextGeneration() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextGeneration() was called, AND they have the same generation. Note that + * none of the configs have to be changed, but they might be. + * + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @return true if generations for all configs have been updated. + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextGeneration() { + return nextGeneration(TimingValues.defaultNextConfigTimeout); + } + + /** + * Use this for waiting for a new config generation, with the given timeout + * + * Returns true if: + * + * It is the first time nextGeneration() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextGeneration() was called, AND they have the same generation. Note that + * none of the configs have to be changed, but they might be. + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @param timeoutMillis timeout in milliseconds + * @return true if generations for all configs have been updated. + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextGeneration(long timeoutMillis) { + return acquireSnapshot(timeoutMillis, false); + } + + /** + * Acquire a snapshot of all configs with the same generation within a timeout. + * @param timeoutInMillis timeout to wait in milliseconds + * @param requireChange if set, at least one config have to change + * @return true, if a new config generation has been found for all configs (additionally requires + * that at lest one of them has changed if <code>requireChange</code> is true), false otherwise + */ + private boolean acquireSnapshot(long timeoutInMillis, boolean requireChange) { + if (state == State.CLOSED) return false; + long started = System.currentTimeMillis(); + long timeLeftMillis = timeoutInMillis; + state = State.FROZEN; + boolean anyConfigChanged = false; + boolean allGenerationsChanged = true; + boolean allGenerationsTheSame = true; + Long currentGenChecker = null; + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + h.setChanged(false); // Reset this flag, if it was set, the user should have acted on it the last time this method returned true. + } + boolean reconfigDue; + do { + // Keep on polling the subscriptions until we have a new generation across the board, or it times out + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + ConfigSubscription<? extends ConfigInstance> subscription = h.subscription(); + if (!subscription.nextConfig(timeLeftMillis)) { + // This subscriber has no new state and we know it has exhausted all time + return false; + } + throwIfExceptionSet(subscription); + if (currentGenChecker == null) currentGenChecker = subscription.getGeneration(); + if (!currentGenChecker.equals(subscription.getGeneration())) allGenerationsTheSame = false; + allGenerationsChanged = allGenerationsChanged && subscription.isGenerationChanged(); + if (subscription.isConfigChanged()) anyConfigChanged = true; + timeLeftMillis = timeLeftMillis - (System.currentTimeMillis() - started); + } + reconfigDue = (anyConfigChanged || !requireChange) && allGenerationsChanged && allGenerationsTheSame; + if (!reconfigDue && timeLeftMillis > 0) { + sleep(10); + } + } while (!reconfigDue && timeLeftMillis > 0); + if (reconfigDue) { + // This indicates the clients will possibly reconfigure their services, so "reset" changed-logic in subscriptions. + // Also if appropriate update the changed flag on the handler, which clients use. + markSubsChangedSeen(); + generation = subscriptionHandles.get(0).subscription().getGeneration(); + } + return reconfigDue; + } + + private void sleep(long i) { + try { + Thread.sleep(i); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + } + + /** + * If a {@link ConfigSubscription} has its exception set, reset that field and throw it + * + * @param sub {@link ConfigSubscription} + */ + protected void throwIfExceptionSet(ConfigSubscription<? extends ConfigInstance> sub) { + RuntimeException subThrowable = sub.getException(); + if (subThrowable != null) { + sub.setException(null); + throw subThrowable; + } + } + + private void markSubsChangedSeen() { + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + ConfigSubscription<? extends ConfigInstance> sub = h.subscription(); + h.setChanged(sub.isConfigChanged()); + sub.resetChangedFlags(); + } + } + + /** + * Closes all open {@link ConfigSubscription}s + */ + public void close() { + state = State.CLOSED; + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + h.subscription().close(); + } + closeRequesters(); + log.log(LogLevel.DEBUG, "Config subscriber has been closed."); + } + + /** + * Closes all open requesters + */ + protected void closeRequesters() { + for (JRTConfigRequester requester : requesters.values()) { + requester.close(); + } + } + + @Override + public String toString() { + String ret = "Subscriber state:" + state; + for (ConfigHandle<?> h : subscriptionHandles) { + ret = ret + "\n" + h.toString(); + } + return ret; + } + + /** + * Convenience method to start a daemon thread called "Vespa config thread" with the given runnable. If you want the runnable to + * handle a {@link ConfigSubscriber} or {@link ConfigHandle} you have declared locally outside, declare them as final to make it work. + * + * @param runnable a class implementing {@link java.lang.Runnable} + * @return the newly started thread + */ + public Thread startConfigThread(Runnable runnable) { + Thread t = new Thread(runnable); + t.setDaemon(true); + t.setName("Vespa config thread"); + t.start(); + return t; + } + + protected State state() { + return state; + } + + /** + * Sets all subscriptions under this subscriber to have the given generation. This is intended for testing, to emulate a + * reload-config operation. + * + * @param generation a generation number + */ + public void reload(long generation) { + for (ConfigHandle<?> h : subscriptionHandles) { + h.subscription().reload(generation); + } + } + + /** + * The source used by this subscriber. + * + * @return the {@link ConfigSource} used by this subscriber + */ + public ConfigSource getSource() { + return source; + } + + /** + * Implementation detail, do not use. + * @return requesters + */ + public Map<ConfigSourceSet, JRTConfigRequester> requesters() { + return requesters; + } + + public boolean isClosed() { + return state == State.CLOSED; + } + + /** + * Use this convenience method if you only want to subscribe on <em>one</em> config, and want generic error handling. + * Implement {@link SingleSubscriber} and pass to this method. + * You will get initial config, and a config thread will be started. The method will throw in your thread if initial + * configuration fails, and the config thread will print a generic error message (but continue) if it fails thereafter. The config + * thread will stop if you {@link #close()} this {@link ConfigSubscriber}. + * + * @param <T> ConfigInstance type + * @param singleSubscriber The object to receive config + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml + * @return The handle of the config + * @see #startConfigThread(Runnable) + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(final SingleSubscriber<T> singleSubscriber, Class<T> configClass, String configId) { + if (!subscriptionHandles.isEmpty()) + throw new IllegalStateException("Can not start single-subscription because subscriptions were previously opened on this."); + final ConfigHandle<T> handle = subscribe(configClass, configId); + if (!nextConfig()) + throw new ConfigurationRuntimeException("Initial config of " + configClass.getName() + " failed."); + singleSubscriber.configure(handle.getConfig()); + startConfigThread(new Runnable() { + @Override + public void run() { + while (!isClosed()) { + try { + if (nextConfig()) { + if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); + } + } catch (Exception e) { + log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + } + } + } + }); + return handle; + } + + /** + * The current generation of configs known by this subscriber. + * + * @return the current generation of configs known by this subscriber + */ + public long getGeneration() { + return generation; + } + + /** + * Convenience interface for clients who only subscribe to one config. Implement this, and pass it to {@link ConfigSubscriber#subscribe(SingleSubscriber, Class, String)}. + * + * @author vegardh + */ + public interface SingleSubscriber<T extends ConfigInstance> { + public void configure(T config); + } + + /** + * Finalizer to ensure that we do not leak resources on reconfig. Though finalizers are bad, + * this is not a performance critical object as it will be deconstructed typically container reconfig. + */ + @Override + protected void finalize() throws Throwable { + try { + if (!isClosed()) { + close(); + } + } finally { + super.finalize(); + } + } + + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java new file mode 100644 index 00000000000..c492a04b1f6 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +import com.yahoo.config.subscription.impl.JRTConfigRequester; + +/** + * A Config URI is a class that can be used to encapsulate a config source and a config id into one + * object to simplify parameter passing. + * + * @author lulf + * @since 5.1 + */ +public class ConfigURI { + private String configId; + private ConfigSource source; + + private ConfigURI(String configId, ConfigSource source) { + this.configId = configId; + this.source = source; + } + + public String getConfigId() { + return configId; + } + + public ConfigSource getSource() { + return source; + } + + public static ConfigURI createFromId(String configId) { + return new ConfigURI(getConfigId(configId), getConfigSource(configId)); + } + + private static ConfigSource getConfigSource(String configId) { + if (configId.startsWith("file:")) { + return new FileSource(new File(configId.substring(5))); + } else if (configId.startsWith("dir:")) { + return new DirSource(new File(configId.substring(4))); + } else { + return JRTConfigRequester.defaultSourceSet; + } + } + + private static String getConfigId(String configId) { + if (configId.startsWith("file:") || configId.startsWith("dir:")) { + return ""; + } else { + return configId; + } + } + + public static ConfigURI createFromIdAndSource(String configId, ConfigSource source) { + return new ConfigURI(configId, source); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java new file mode 100644 index 00000000000..d9e425022f0 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +/** + * Source specifying config from a local directory + * @author vegardh + * @since 5.1 + * + */ +public class DirSource implements ConfigSource { + private final File dir; + + public DirSource(File dir) { + if (!dir.isDirectory()) throw new IllegalArgumentException("Not a directory: "+dir); + this.dir = dir; + } + + public File getDir() { + return dir; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java new file mode 100644 index 00000000000..d7634a000b6 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +/** + * Source specifying config from one local file + * @author vegardh + * @since 5.1 + * + */ +public class FileSource implements ConfigSource { + private final File file; + + public FileSource(File file) { + if (!file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file); + this.file = file; + } + + public File getFile() { + return file; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java new file mode 100644 index 00000000000..021b3e72025 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.jar.JarFile; + +/** + * Source specifying config as a jar file entry + * @author vegardh + * @since 5.1 + * + */ +public class JarSource implements ConfigSource { + private final String path; + private final JarFile jarFile; + + /** + * Creates a new jar source + * @param jarFile the jar file to use as a source + * @param path the path within the jar file, or null to use the default config/ + */ + public JarSource(JarFile jarFile, String path) { + this.path = path; + this.jarFile = jarFile; + } + + public JarFile getJarFile() { + return jarFile; + } + + public String getPath() { + return path; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java new file mode 100644 index 00000000000..74b3d508652 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * Source specifying raw config, where payload is given programmatically + * @author vegardh + * @since 5.1 + * + */ +public class RawSource implements ConfigSource { + public final String payload; + + /** + * New source with the given payload on Vespa cfg format + * @param payload config payload + */ + public RawSource(String payload) { + this.payload = payload; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java new file mode 100644 index 00000000000..3778ee38d98 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.reflect.Constructor; + +/** + * Subscription on a programmatically built set of configs + * @author vegardh + * @since 5.1 + */ +public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + + private final ConfigSet set; + private final ConfigKey<T> subKey; + + ConfigSetSubscription(ConfigKey<T> key, + ConfigSubscriber subscriber, ConfigSource cset) { + super(key, subscriber); + if (!(cset instanceof ConfigSet)) throw new IllegalArgumentException("Source is not a ConfigSet: "+cset); + this.set=(ConfigSet) cset; + subKey = new ConfigKey<T>(configClass, key.getConfigId()); + if (!set.contains(subKey)) { + throw new IllegalArgumentException("The given ConfigSet "+set+" does not contain a config for "+subKey); + } + setGeneration(0l); + } + + @Override + public boolean nextConfig(long timeout) { + long end = System.currentTimeMillis() + timeout; + do { + ConfigInstance myInstance = getNewInstance(); + // User forced reload + if (checkReloaded()) { + updateInstance(myInstance); + return true; + } + if (!myInstance.equals(config)) { + generation++; + updateInstance(myInstance); + return true; + } + sleep(10); + } while (System.currentTimeMillis() < end); + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + private void sleep(int milliSecondsToSleep) { + try { + Thread.sleep(milliSecondsToSleep); + } catch (InterruptedException e) { + throw new RuntimeException("nextConfig aborted", e); + } + } + + @SuppressWarnings("unchecked") + private void updateInstance(ConfigInstance myInstance) { + if (!myInstance.equals(config)) { + setConfigChanged(true); + } + setConfig((T) myInstance); + setGenerationChanged(true); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + + public ConfigInstance getNewInstance() { + try { + ConfigInstance.Builder builder = set.get(subKey); + Constructor<?> constructor = builder.getClass().getDeclaringClass().getConstructor(builder.getClass()); + return (ConfigInstance) constructor.newInstance(builder); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java new file mode 100644 index 00000000000..0909dc6e1a2 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -0,0 +1,310 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.File; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.DirSource; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.config.subscription.JarSource; +import com.yahoo.config.subscription.RawSource; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.DefContent; + +/** + * Represents one active subscription to one config + * + * @author vegardh + * @since 5.1 + */ +public abstract class ConfigSubscription<T extends ConfigInstance> { + protected static Logger log = Logger.getLogger(ConfigSubscription.class.getName()); + protected ConfigSubscriber subscriber; + protected boolean configChanged = false; + protected boolean generationChanged = false; + protected volatile T config = null; + protected Long generation = null; + protected ConfigKey<T> key; + protected Class<T> configClass; + private volatile RuntimeException exception = null; + private State state = State.OPEN; + /** + * If non-null: The user has set this generation explicitly. nextConfig should take this into account. + * Access to these variables _must_ be synchronized, as nextConfig and reload() is likely to be run from + * independent threads. + */ + private boolean doReload = false; + private long reloadedGeneration = -1; + + enum State { + OPEN, CLOSED + } + + /** + * Initializes one subscription + * + * @param key a {@link ConfigKey} + * @param subscriber the subscriber for this subscription + */ + ConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber) { + this.key = key; + this.configClass = key.getConfigClass(); + this.subscriber = subscriber; + } + + + /** + * Correct type of ConfigSubscription instance based on type of source or form of config id + * + * @param key a {@link ConfigKey} + * @param subscriber the subscriber for this subscription + * @return a subclass of a ConfigsSubscription + */ + public static <T extends ConfigInstance> ConfigSubscription<T> get(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { + String configId = key.getConfigId(); + if (source instanceof RawSource || configId.startsWith("raw:")) return getRawSub(key, subscriber, source); + if (source instanceof FileSource || configId.startsWith("file:")) return getFileSub(key, subscriber, source); + if (source instanceof DirSource || configId.startsWith("dir:")) return getDirFileSub(key, subscriber, source); + if (source instanceof JarSource || configId.startsWith("jar:")) return getJarSub(key, subscriber, source); + if (source instanceof ConfigSet) return new ConfigSetSubscription<>(key, subscriber, source); + if (source instanceof ConfigSourceSet) return new JRTConfigSubscription<>(key, subscriber, source, timingValues); + throw new IllegalArgumentException("Unknown source type: "+source); + } + + private static <T extends ConfigInstance> JarConfigSubscription<T> getJarSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String jarName; + String path="config/"; + if (source instanceof JarSource) { + JarSource js = (JarSource) source; + jarName=js.getJarFile().getName(); + if (js.getPath()!=null) path=js.getPath(); + } else { + jarName=key.getConfigId().replace("jar:", "").replaceFirst("\\!/.*", ""); + if (key.getConfigId().contains("!/")) path = key.getConfigId().replaceFirst(".*\\!/", ""); + } + return new JarConfigSubscription<>(key, subscriber, jarName, path); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getFileSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + File file = ((source instanceof FileSource))?((FileSource)source).getFile():new File(key.getConfigId().replace("file:", "")); + return new FileConfigSubscription<>(key, subscriber, file); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getRawSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String payload = ((source instanceof RawSource)?((RawSource)source).payload:key.getConfigId().replace("raw:", "")); + return new RawConfigSubscription<>(key, subscriber,payload); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getDirFileSub(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String dir = key.getConfigId().replace("dir:", ""); + if (source instanceof DirSource) { + dir = ((DirSource)source).getDir().toString(); + } + if (!dir.endsWith(File.separator)) dir = dir + File.separator; + String name = getConfigFilenameNoVersion(key); + File file = new File(dir + name); + if (!file.exists()) { + throw new IllegalArgumentException("Could not find a config file for '" + key.getName() + "' in '" + dir + "'"); + } + return new FileConfigSubscription<>(key, subscriber, file); + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object o) { + if (o instanceof ConfigSubscription) { + ConfigSubscription<T> other = (ConfigSubscription<T>) o; + return key.equals(other.key) && + subscriber.equals(other.subscriber); + } + return false; + } + + void setConfigChanged(boolean changed) { + this.configChanged = changed; + } + + void setGenerationChanged(boolean genChanged) { + this.generationChanged = genChanged; + } + + /** + * Called from {@link ConfigSubscriber} when the changed status of this config is propagated to the clients + */ + public void resetChangedFlags() { + setConfigChanged(false); + setGenerationChanged(false); + } + + public boolean isConfigChanged() { + return configChanged; + } + + public boolean isGenerationChanged() { + return generationChanged; + } + + void setConfig(T config) { + this.config = config; + } + + /** + * The config object of this subscription + * + * @return the ConfigInstance (the config) of this subscription + */ + public T getConfig() { + return config; + } + + /** + * The generation of this subscription + * + * @return the generation of this subscription + */ + public Long getGeneration() { + return generation; + } + + /** + * The class of the subscription's desired {@link ConfigInstance} + * @return the config class + */ + public Class<T> getConfigClass() { + return configClass; + } + + void setGeneration(Long generation) { + this.generation = generation; + } + + @Override + public String toString() { + StringBuilder s = new StringBuilder(key.toString()); + s.append(", Current generation: ").append(generation) + .append(", Generation changed: ").append(generationChanged) + .append(", Config changed: ").append(configChanged); + if (exception != null) + s.append(", Exception: ").append(exception); + return s.toString(); + } + + /** + * The config key which this subscription uses to identify its config + * + * @return the ConfigKey for this subscription + */ + public ConfigKey<T> getKey() { + return key; + } + + /** + * Polls this subscription for a change. The method is guaranteed to use all of the given timeout before returning false. It will also take into account a user-set generation, + * that can be set by {@link ConfigSubscriber#reload(long)}. + * + * @param timeout in milliseconds + * @return false if timed out, true if the state of {@link #configChanged}, {@link #generationChanged} or {@link #exception} changed. If true, the {@link #config} field will be set also. + * has changed + */ + public abstract boolean nextConfig(long timeout); + + /** + * Will block until the next {@link #nextConfig(long)} is guaranteed to return an answer (or throw) immediately (i.e. not block) + * + * @param timeout in milliseconds + * @return false if timed out + */ + public abstract boolean subscribe(long timeout); + + /** + * Called by for example network threads to signal that the user thread should throw this exception immediately + * + * @param e a RuntimeException + */ + public void setException(RuntimeException e) { + this.exception = e; + } + + /** + * Gets an exception set by for example a network thread. If not null, it indicates that it should be + * thrown in the user's thread immediately. + * + * @return a RuntimeException if there exists one + */ + public RuntimeException getException() { + return exception; + } + + /** + * Returns true if an exception set by for example a network thread has been caught. + * + * @return true if there exists an exception for this subscription + */ + boolean hasException() { + return exception != null; + } + + public void close() { + state = State.CLOSED; + } + + State getState() { + return state; + } + + /** + * Returns the file name corresponding to the given key's defName and version. + * + * @param key a {@link ConfigKey} + * @return file name with version number. + */ + static <T extends ConfigInstance> String getConfigFilenameNoVersion(ConfigKey<T> key) { + StringBuilder filename = new StringBuilder(key.getName()); + filename.append(".cfg"); + return filename.toString(); + } + + /** + * Force this into the given generation, used in testing + * @param generation a config generation + */ + public synchronized void reload(long generation) { + this.doReload = true; + this.reloadedGeneration = generation; + } + + /** + * True if someone has set the {@link #reloadedGeneration} number by calling {@link #reload(long)} + * and hence wants to force a given generation programmatically. If that is the case, + * sets the {@link #generation} and {@link #generationChanged} fields accordingly. + * @return true if {@link #reload(long)} has been called, false otherwise + */ + protected synchronized boolean checkReloaded() { + if (doReload) { + // User has called reload + generation = reloadedGeneration; + setGenerationChanged(true); + doReload = false; + return true; + } + return false; + } + + /** + * The config definition schema + * + * @return the config definition for this subscription + */ + public DefContent getDefContent() { + return (DefContent.fromClass(configClass)); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java new file mode 100644 index 00000000000..c1b1e3daaff --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.log.LogLevel; + +/** + * Subscription used when config id is file:... + * @author vegardh + * @since 5.1 + * + */ +public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + + final File file; + long ts; + + FileConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, File f) { + super(key, subscriber); + setGeneration(0l); + file=f; + if (!file.exists() && !file.isFile()) + throw new IllegalArgumentException("Not a file: "+file); + } + + @Override + public boolean nextConfig(long timeout) { + if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); + if (checkReloaded()) { + // TODO: Temporary log messages for debugging. + log.log(LogLevel.INFO, "User forced config reload at " + System.currentTimeMillis()); + // User forced reload + updateConfig(); + log.log(LogLevel.INFO, "Config updated at " + System.currentTimeMillis() + ", changed: " + isConfigChanged()); + log.log(LogLevel.INFO, "Config: " + config.toString()); + return true; + } + if (file.lastModified()!=ts) { + updateConfig(); + generation++; + setGenerationChanged(true); + return true; + } + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + private void updateConfig() { + ts=file.lastModified(); + ConfigInstance prev = config; + try { + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readFile(file).split("\n"))); + config = payload.toInstance(configClass, key.getConfigId()); + } catch (IOException e) { + throw new ConfigurationRuntimeException(e); + } + setConfigChanged(!config.equals(prev)); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java new file mode 100644 index 00000000000..26963428914 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.vespa.config.RawConfig; + +/** + * A config handle which does not use the config class, but payload instead. To be used in proxy? + * + * @author vegardh + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class GenericConfigHandle extends ConfigHandle { + + private final GenericJRTConfigSubscription genSub; + + public GenericConfigHandle(GenericJRTConfigSubscription sub) { + super(sub); + genSub = sub; + } + + public RawConfig getRawConfig() { + return genSub.getRawConfig(); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java new file mode 100644 index 00000000000..5a155e42aca --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.List; +import java.util.Map; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; + +/** + * A subscriber that can subscribe without the class. Used by configproxy. + * + * @author vegardh + */ +public class GenericConfigSubscriber extends ConfigSubscriber { + /** + * Constructs a new subscriber using the given pool of requesters (JRTConfigRequester holds 1 connection which in + * turn is subject to failover across the elems in the source set.) + * The behaviour is undefined if the map key is different from the source set the requester was built with. + * See also {@link JRTConfigRequester#get(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} + * + * @param requesters a map from config source set to config requester + */ + public GenericConfigSubscriber(Map<ConfigSourceSet, JRTConfigRequester> requesters) { + this.requesters = requesters; + } + + public GenericConfigSubscriber() { + super(); + } + + /** + * Subscribes to config without using the class. For internal use in config proxy. + * + * @param key the {@link ConfigKey to subscribe to} + * @param defContent the config definition content for the config to subscribe to + * @param source the config source to use + * @param timingValues {@link TimingValues} + * @return generic handle + */ + public GenericConfigHandle subscribe(ConfigKey<?> key, List<String> defContent, ConfigSource source, TimingValues timingValues) { + checkStateBeforeSubscribe(); + GenericJRTConfigSubscription sub = new GenericJRTConfigSubscription(key, defContent, this, source, timingValues); + GenericConfigHandle handle = new GenericConfigHandle(sub); + subscribeAndHandleErrors(sub, key, handle, timingValues); + return handle; + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId) { + throw new UnsupportedOperationException(); + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, long timeoutMillis) { + throw new UnsupportedOperationException(); + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(SingleSubscriber<T> singleSubscriber, Class<T> configClass, String configId) { + throw new UnsupportedOperationException(); + } + + /** + * Do nothing, since we share requesters + */ + public void closeRequesters() { + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java new file mode 100644 index 00000000000..5e88e86be71 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.List; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; + +/** + * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. + * Used by config proxy. + * @author vegardh + * + */ +@SuppressWarnings("rawtypes") +public class GenericJRTConfigSubscription extends JRTConfigSubscription { + + private RawConfig config; + private final List<String> defContent; + + @SuppressWarnings("unchecked") + public GenericJRTConfigSubscription(ConfigKey<?> key, + List<String> defContent, + ConfigSubscriber subscriber, + ConfigSource source, + TimingValues timingValues) { + super(key, subscriber, source, timingValues); + this.defContent = defContent; + } + + @Override + protected void setNewConfig(JRTClientConfigRequest jrtReq) { + this.config = RawConfig.createFromResponseParameters(jrtReq); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "in setNewConfig, config=" + this.config); + } + } + + // This method is overridden because config needs to have its generation + // updated if _only_ generation has changed + @Override + void setGeneration(Long generation) { + super.setGeneration(generation); + if (this.config != null) { + this.config.setGeneration(generation); + } + } + + public RawConfig getRawConfig() { + return config; + } + + /** + * The config definition schema + * + * @return the config definition for this subscription + */ + @Override + public DefContent getDefContent() { + return (DefContent.fromList(defContent)); + } + + @Override + public ConfigInstance getConfig() { + return null; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java new file mode 100644 index 00000000000..2e2e71989a5 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -0,0 +1,362 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.protocol.JRTConfigRequestFactory; +import com.yahoo.vespa.config.protocol.Trace; + +/** + * This class fetches config payload using JRT, and acts as the callback target. + * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} + * as context, and puts the requests objects on a queue on the subscription, + * for handling by the user thread. + * + * @author vegardh + * @since 5.1 + */ +// Note: this is similar to old JRTSource +public class JRTConfigRequester implements RequestWaiter { + private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName()); + public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault(); + private static final int TRACELEVEL = 6; + private final TimingValues timingValues; + private int fatalFailures = 0; // independent of transientFailures + private int transientFailures = 0; // independent of fatalFailures + private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + private long suspendWarned; + private long noApplicationWarned; + private static final long delayBetweenWarnings = 60000; //ms + private final ConnectionPool connectionPool; + static final float randomFraction = 0.2f; + /* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */ + private static final Double additionalTimeForClientTimeout = 5.0; + + private static final SimpleDateFormat yyyyMMddz; + + static { + yyyyMMddz = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + yyyyMMddz.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + /** + * Returns a new requester + * @param connectionPool The connectionPool to use + * @param timingValues The timing values + * @return new requester object + */ + public static JRTConfigRequester get(ConnectionPool connectionPool, TimingValues timingValues) { + return new JRTConfigRequester(connectionPool, timingValues); + } + + /** + * New requester + * @param connectionPool the connectionPool this requester should use + * @param timingValues timeouts and delays used when sending JRT config requests + */ + JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + this.connectionPool = connectionPool; + this.timingValues = timingValues; + } + + /** + * Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription} + * + * @param sub a subscription + */ + public <T extends ConfigInstance> void request(JRTConfigSubscription<T> sub) { + JRTClientConfigRequest req = JRTConfigRequestFactory.createFromSub(sub); + doRequest(sub, req, timingValues.getSubscribeTimeout()); + } + + private <T extends ConfigInstance> void doRequest(JRTConfigSubscription<T> sub, + JRTClientConfigRequest req, long timeout) { + com.yahoo.vespa.config.Connection connection = connectionPool.getCurrent(); + req.getRequest().setContext(new RequestContext(sub, req, connection)); + boolean reqOK = req.validateParameters(); + if (!reqOK) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); + // Add some time to the timeout, we never want it to time out in JRT during normal operation + double jrtClientTimeout = getClientTimeout(timeout); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Requesting config for " + sub + " on connection " + connection + " with RPC timeout " + jrtClientTimeout + ",defcontent=" + + req.getDefContent().asString()); + } + connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); + } + + @SuppressWarnings("unchecked") + @Override + public void handleRequestDone(Request req) { + JRTConfigSubscription<ConfigInstance> sub = null; + try { + RequestContext context = (RequestContext) req.getContext(); + sub = context.sub; + doHandle(sub, context.jrtReq, context.connection); + } catch (RuntimeException e) { + if (sub != null) { + // Sets this field, it will get thrown from the user thread + sub.setException(e); + } else { + // Very unlikely + log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + + Exceptions.toMessageString(e)); + } + } + } + + protected void doHandle(JRTConfigSubscription<ConfigInstance> sub, JRTClientConfigRequest jrtReq, Connection connection) { + if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing + boolean validResponse = jrtReq.validateResponse(); + Trace trace = jrtReq.getResponseTrace(); + trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, trace.toString()); + } + if (validResponse) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + jrtReq + "\nSpec: " + connection); + } + handleOKRequest(jrtReq, sub, connection); + } else { + logWhenErrorResponse(jrtReq, connection); + handleFailedRequest(jrtReq, sub, connection); + } + } + + private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { + switch (jrtReq.errorCode()) { + case com.yahoo.jrt.ErrorCode.CONNECTION: + log.log(LogLevel.DEBUG, "Request callback failed: " + jrtReq.errorMessage() + + "\nConnection spec: " + connection); + break; + case ErrorCode.APPLICATION_NOT_LOADED: + case ErrorCode.UNKNOWN_VESPA_VERSION: + final long now = System.currentTimeMillis(); + if (noApplicationWarned < (now - delayBetweenWarnings)) { + log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + ". Connection spec: " + connection.getAddress()); + noApplicationWarned = now; + } + break; + default: + log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + " . Req error message: " + jrtReq.errorMessage()); + break; + } + } + + private void handleFailedRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub, Connection connection) { + final boolean configured = (sub.getConfig() != null); + if (configured) { + // The subscription object has an "old" config, which is all we have to offer back now + log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + } + final ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); + connectionPool.setError(connection, jrtReq.errorCode()); + long delay = calculateFailedRequestDelay(errorType, transientFailures, fatalFailures, timingValues, configured); + if (errorType == ErrorType.TRANSIENT) { + handleTransientlyFailed(jrtReq, sub, delay, connection); + } else { + handleFatallyFailed(jrtReq, sub, delay); + } + } + + static long calculateFailedRequestDelay(ErrorType errorCode, int transientFailures, int fatalFailures, + TimingValues timingValues, boolean configured) { + long delay; + if (configured) + delay = timingValues.getConfiguredErrorDelay(); + else + delay = timingValues.getUnconfiguredDelay(); + if (errorCode == ErrorType.TRANSIENT) { + delay = delay * Math.min((transientFailures + 1), timingValues.getMaxDelayMultiplier()); + } else { + delay = timingValues.getFixedDelay() + (delay * Math.min(fatalFailures, timingValues.getMaxDelayMultiplier())); + delay = timingValues.getPlusMinusFractionRandom(delay, randomFraction); + } + return delay; + } + + private void handleTransientlyFailed(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, + long delay, + Connection connection) { + long now = System.currentTimeMillis(); + transientFailures++; + if (suspendWarned < (now - delayBetweenWarnings)) { + log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + + " failed or timed out, clients will keep existing config, will keep trying."); + suspendWarned = now; + } + if (sub.getState() != ConfigSubscription.State.OPEN) return; + scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); + } + + private long calculateErrorTimeout() { + return timingValues.getPlusMinusFractionRandom(timingValues.getErrorTimeout(), randomFraction); + } + + /** + * This handles a fatal error both in the case that the subscriber is configured and not. + * The difference is in the delay (passed from outside) and the log level used for + * error message. + * + * @param jrtReq a JRT config request + * @param sub a config subscription + * @param delay delay before sending a new request + */ + private void handleFatallyFailed(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, long delay) { + if (sub.getState() != ConfigSubscription.State.OPEN) return; + fatalFailures++; + // The logging depends on whether we are configured or not. + Level logLevel = sub.getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; + String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + + " in " + delay + " ms"; + log.log(logLevel, logMessage); + scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); + } + + private void handleOKRequest(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, + Connection connection) { + // Reset counters pertaining to error handling here + fatalFailures = 0; + transientFailures = 0; + suspendWarned = 0; + connection.setSuccess(); + sub.setLastCallBackOKTS(System.currentTimeMillis()); + if (jrtReq.hasUpdatedGeneration()) { + // We only want this latest generation to be in the queue, we do not preserve history in this system + handleEmptyPayload(jrtReq, sub); + sub.getReqQueue().clear(); + boolean putOK = sub.getReqQueue().offer(jrtReq); + if (!putOK) { + sub.setException(new ConfigurationRuntimeException("Could not put returned request on queue of subscription " + sub)); + } + } + if (sub.getState() != ConfigSubscription.State.OPEN) return; + scheduleNextRequest(jrtReq, sub, + calculateSuccessDelay(), + calculateSuccessTimeout()); + } + + private long calculateSuccessTimeout() { + return timingValues.getPlusMinusFractionRandom(timingValues.getSuccessTimeout(), randomFraction); + } + + private long calculateSuccessDelay() { + return timingValues.getPlusMinusFractionRandom(timingValues.getFixedDelay(), randomFraction); + } + + /** + * This works around an optimization in the protocol: the payload is not set if it is not changed (seen from the server). + * So, if the sub's queue has a still _unprocessed_ req with payload, and the current one has no payload, + * i.e. it wasn't changed, save the one in the earlier req before clearing the queue. + * + * @param jrtReq a JRT config request + * @param sub a config subscription + */ + private void handleEmptyPayload(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub) { + if (jrtReq.containsPayload()) { + JRTClientConfigRequest reqInQueue = sub.getReqQueue().poll(); // Just take it out, we were about to clear the queue anyway + if (reqInQueue != null) { + jrtReq.updateRequestPayload(reqInQueue.getNewPayload(), reqInQueue.hasUpdatedConfig()); + } + } + } + + private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) { + if (delay < 0) delay = 0; + JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "My timing values: " + timingValues); + log.log(LogLevel.DEBUG, "Scheduling new request " + delay + " millis from now for " + jrtReqNew.getConfigKey()); + } + scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delay, TimeUnit.MILLISECONDS); + } + + /** + * Task that can be scheduled in a timer for executing a getConfig request + */ + private class GetConfigTask implements Runnable { + private final JRTClientConfigRequest jrtReq; + private final JRTConfigSubscription<?> sub; + + public GetConfigTask(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<?> sub) { + this.jrtReq = jrtReq; + this.sub = sub; + } + + public void run() { + doRequest(sub, jrtReq, jrtReq.getTimeout()); + } + } + + public void close() { + suspendWarned = System.currentTimeMillis(); // Avoid printing warnings after this + connectionPool.close(); + scheduler.shutdown(); + } + + private class JRTSourceThreadFactory implements ThreadFactory { + @SuppressWarnings("NullableProblems") + @Override + public Thread newThread(Runnable runnable) { + ThreadFactory tf = Executors.defaultThreadFactory(); + Thread t = tf.newThread(runnable); + // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system + t.setDaemon(true); + return t; + } + } + + @SuppressWarnings("rawtypes") + private static class RequestContext { + final JRTConfigSubscription sub; + final JRTClientConfigRequest jrtReq; + final Connection connection; + + private RequestContext(JRTConfigSubscription sub, JRTClientConfigRequest jrtReq, Connection connection) { + this.sub = sub; + this.jrtReq = jrtReq; + this.connection = connection; + } + } + + int getTransientFailures() { + return transientFailures; + } + + int getFatalFailures() { + return fatalFailures; + } + + // TODO: Should be package private + public ConnectionPool getConnectionPool() { + return connectionPool; + } + + private Double getClientTimeout(long serverTimeout) { + return (serverTimeout / 1000.0) + additionalTimeForClientTimeout; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java new file mode 100644 index 00000000000..4405ca2f05c --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -0,0 +1,190 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.vespa.config.protocol.Payload; + +/** + * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server + * + * @author vegardh + * @since 5.1 + */ +public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private JRTConfigRequester requester; + private TimingValues timingValues; + // Last time we got an OK JRT callback for this + private long lastOK=0; + + /** + * The queue containing either nothing or the one (newest) request that has got callback from JRT, + * but has not yet been handled. + */ + private LinkedBlockingQueue<JRTClientConfigRequest> reqQueue = new LinkedBlockingQueue<>(); + private ConfigSourceSet sources; + + public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { + super(key, subscriber); + this.timingValues=timingValues; + if (source instanceof ConfigSourceSet) { + this.sources=(ConfigSourceSet) source; + } + } + + @Override + public boolean nextConfig(long timeoutMillis) { + // These flags may have been left true from a previous call, since ConfigSubscriber's nextConfig + // not necessarily returned true and reset the flags then + boolean gotNew = isGenerationChanged() || isConfigChanged() || hasException(); + // Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check + if (getReqQueue().peek()==null && gotNew) { + return true; + } + // Otherwise poll the queue for another generation or timeout + // + // Note: since the JRT callback thread will clear the queue first when it inserts a brand new element, + // there is a race here. However: the caller will handle it no matter what it gets from the queue here, + // the important part is that local state on the subscription objects is preserved. + if (!pollQueue(timeoutMillis)) return gotNew; + gotNew = isGenerationChanged() || isConfigChanged() || hasException(); + return gotNew; + } + + /** + * Polls the callback queue and <em>maybe</em> sets the following (caller must check): generation, generation changed, config, config changed + * Important: it never <em>resets</em> those flags, we must persist that state until the {@link ConfigSubscriber} clears it + * @param timeoutMillis timeout when polling (returns after at most this time) + * @return true if it got anything off the queue and <em>maybe</em> changed any state, false if timed out taking from queue + */ + private boolean pollQueue(long timeoutMillis) { + JRTClientConfigRequest jrtReq; + try { + // Only valid responses are on queue, no need to validate + jrtReq = getReqQueue().poll(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e1) { + throw new ConfigInterruptedException(e1); + } + if (jrtReq == null) { + // timed out, we know nothing new. + return false; + } + if (jrtReq.hasUpdatedGeneration()) { + //printStatus(jrtReq, "Updated generation or config"); + setGeneration(jrtReq.getNewGeneration()); + setGenerationChanged(true); + if (jrtReq.hasUpdatedConfig()) { + // payload changed + setNewConfig(jrtReq); + setConfigChanged(true); + } + } + return true; + } + + protected void setNewConfig(JRTClientConfigRequest jrtReq) { + setConfig(toConfigInstance(jrtReq)); + } + + /** + * This method should ideally throw new MissingConfig/Configuration exceptions and let the caller + * catch them. However, this would make the code in JRT/File/RawSource uglier. + * Alternatively, it could return a SetConfigStatus object with an int and an error message. + * + * @param jrtRequest a config request + * @return an instance of a config class (subclass of ConfigInstance) + */ + T toConfigInstance(JRTClientConfigRequest jrtRequest) { + Payload payload = jrtRequest.getNewPayload(); + ConfigPayload configPayload = ConfigPayload.fromUtf8Array(payload.withCompression(CompressionType.UNCOMPRESSED).getData()); + T configInstance = configPayload.toInstance(configClass, jrtRequest.getConfigKey().getConfigId()); + configInstance.setConfigMd5(jrtRequest.getNewConfigMd5()); + return configInstance; + } + + LinkedBlockingQueue<JRTClientConfigRequest> getReqQueue() { + return reqQueue; + } + + @Override + public boolean subscribe(long timeout) { + lastOK=System.currentTimeMillis(); + requester = getRequester(); + requester.request(this); + JRTClientConfigRequest req = reqQueue.peek(); + while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + req = reqQueue.peek(); + } + return req != null; + } + + private JRTConfigRequester getRequester() { + JRTConfigRequester requester = subscriber.requesters().get(sources); + if (requester==null) { + requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); + subscriber.requesters().put(sources, requester); + } + return requester; + } + + @Override + @SuppressWarnings("serial") + public void close() { + super.close(); + reqQueue = new LinkedBlockingQueue<JRTClientConfigRequest>() { + @Override public void put(JRTClientConfigRequest e) throws InterruptedException { + // When closed, throw away all requests that callbacks try to put + } + }; + } + + /** + * The timing values of this + * @return timing values + */ + public TimingValues timingValues() { + return timingValues; + } + + // Used in integration tests + @SuppressWarnings("UnusedDeclaration") + public JRTConfigRequester requester() { + return requester; + } + + @Override + public void reload(long generation) { + log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); + } + + void setLastCallBackOKTS(long lastCallBackOKTS) { + this.lastOK = lastCallBackOKTS; + } + + // For debugging + @SuppressWarnings("UnusedDeclaration") + static void printStatus(JRTClientConfigRequest request, String message) { + final String name = request.getConfigKey().getName(); + if (name.equals("components") || name.equals("chains")) { + log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java new file mode 100644 index 00000000000..434b61db918 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.jar.JarFile; +import java.util.zip.ZipEntry; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; + +/** + * Subscription to use when config id is jar:.../foo.jar[!/pathInJar/] + * + * @author vegardh + * @author gjoranv + * @since 5.1 + * + */ +public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private final String jarName; + private final String path; + private ZipEntry zipEntry = null; + + // jar:configs/app.jar!/configs/ + JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) { + super(key, subscriber); + this.jarName=jarName; + this.path=path; + } + + @Override + public boolean nextConfig(long timeout) { + if (checkReloaded()) { + // Not supporting changing the payload for jar + return true; + } + if (zipEntry==null) { + // First time polled + JarFile jarFile = null; + try { + jarFile = new JarFile(jarName); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + zipEntry = getEntry(jarFile, path); + if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); + try { + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), "UTF-8")).split("\n"))); + config = payload.toInstance(configClass, key.getConfigId()); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } catch (IOException e) { + throw new ConfigurationRuntimeException(e); + } + setGeneration(0l); + setGenerationChanged(true); + setConfigChanged(true); + try { + jarFile.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return true; + } + // TODO: Should wait and detect changes + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + /** + * Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in + * the given JarFile. + * If the file with correct version number does not exist, returns the filename without version number. + * The file's existence is checked elsewhere. + */ + private ZipEntry getEntry(JarFile jarFile, String dir) { + if (!dir.endsWith("/")) { + dir = dir + '/'; + } + return jarFile.getEntry(dir + getConfigFilenameNoVersion(key)); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java new file mode 100644 index 00000000000..3e9047b3bfa --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.util.ConfigUtils; + +/** + * For unit testing + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.11 + */ +public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { + + private Request lastRequest; + private final ResponseHandler responseHandler; + private int numberOfRequests = 0; + + public int getNumberOfFailovers() { + return numberOfFailovers; + } + + private int numberOfFailovers = 0; + private final int numSpecs; + + public MockConnection() { + this(new OKResponseHandler()); + } + + public MockConnection(ResponseHandler responseHandler) { + this(responseHandler, 1); + } + + public MockConnection(ResponseHandler responseHandler, int numSpecs) { + this.responseHandler = responseHandler; + this.numSpecs = numSpecs; + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + numberOfRequests++; + lastRequest = request; + responseHandler.requestWaiter(requestWaiter).request(request); + Thread t = new Thread(responseHandler); + t.setDaemon(true); + t.run(); + } + + @Override + public void setError(int errorCode) { + numberOfFailovers++; + } + + @Override + public void setSuccess() { + numberOfFailovers = 0; + } + + @Override + public String getAddress() { + return null; + } + + @Override + public void close() { + + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + } + + @Override + public Connection getCurrent() { + return this; + } + + @Override + public Connection setNewCurrentConnection() { + return this; + } + + @Override + public int getSize() { + return numSpecs; + } + + public int getNumberOfRequests() { + return numberOfRequests; + } + + public Request getRequest() { + return lastRequest; + } + + static class OKResponseHandler extends AbstractResponseHandler { + protected void createResponse() { + JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); + Payload payload = Payload.from(ConfigPayload.empty()); + long generation = 1; + jrtReq.addOkResponse(payload, generation, ConfigUtils.getMd5(payload.getData())); + } + } + + + public interface ResponseHandler extends Runnable { + + RequestWaiter requestWaiter(); + + Request request(); + + ResponseHandler requestWaiter(RequestWaiter requestWaiter); + + ResponseHandler request(Request request); + } + + public abstract static class AbstractResponseHandler implements ResponseHandler { + private RequestWaiter requestWaiter; + protected Request request; + + @Override + public RequestWaiter requestWaiter() { + return requestWaiter; + } + + @Override + public Request request() { + return request; + } + + @Override + public ResponseHandler requestWaiter(RequestWaiter requestWaiter) { + this.requestWaiter = requestWaiter; + return this; + } + + @Override + public ResponseHandler request(Request request) { + this.request = request; + return this; + } + + @Override + public void run() { + createResponse(); + requestWaiter.handleRequestDone(request); + } + + protected abstract void createResponse(); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java new file mode 100644 index 00000000000..2c8470cf63c --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.Arrays; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; + +/** + * Subscription used when config id is raw:... + * + * Config is the actual text given after the config id, with newlines + * + * @author vegardh + * @since 5.1 + * + */ +public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + final String inputPayload; + String payload; + + RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) { + super(key, subscriber); + this.inputPayload=pl; + } + + @Override + public boolean nextConfig(long timeout) { + if (checkReloaded()) { + return true; + } + if (payload==null) { + payload = inputPayload; + setGeneration(0l); + setGenerationChanged(true); + setConfigChanged(true); + + ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); + config = configPayload.toInstance(configClass, key.getConfigId()); + return true; + } + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/package-info.java b/config/src/main/java/com/yahoo/config/subscription/package-info.java new file mode 100644 index 00000000000..7a77088a43e --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +/** + * Classes for subscribing to Vespa config. + */ +package com.yahoo.config.subscription; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/jrt/.gitignore b/config/src/main/java/com/yahoo/jrt/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/jrt/.gitignore diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java new file mode 100644 index 00000000000..15e67138393 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * A ConfigKey that also uses the def MD5 sum. Used for caching when def payload is user provided. + * @author vegardh + * + */ +public class ConfigCacheKey { + private final ConfigKey<?> key; + private final String defMd5; + + /** + * Constructs a new server key based on the contents of the given {@link ConfigKey} and the def md5 sum. + * @param key The key to base on + * @param defMd5 MD5 checksum of the config definition. Never null. + */ + public ConfigCacheKey(ConfigKey<?> key, String defMd5) { + this.key = key; + this.defMd5 = defMd5 == null ? "" : defMd5; + } + + /** + * Constructs new key + * + * @param name config definition name + * @param configIdString Can be null. + * @param namespace namespace for this config definition + * @param defMd5 MD5 checksum of the config definition. Never null. + */ + ConfigCacheKey(String name, String configIdString, String namespace, String defMd5) { + this(new ConfigKey<>(name, configIdString, namespace), defMd5); + } + + @Override + public int hashCode() { + return key.hashCode() + 37 * defMd5.hashCode(); + } + + @Override + public boolean equals(Object o) { + return o instanceof ConfigCacheKey && key.equals(((ConfigCacheKey) o).getKey()) + && defMd5.equals(((ConfigCacheKey)o).defMd5); + } + + /** + * The def md5 sum of this key + * @return md5 sum + */ + public String getDefMd5() { + return defMd5; + } + + public ConfigKey<?> getKey() { + return key; + } + + @Override + public String toString() { + return key + "," + defMd5; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java new file mode 100644 index 00000000000..9b4f8d12756 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java @@ -0,0 +1,1068 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.yolean.Exceptions; + +import java.util.*; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +/** + * Represents one legal def file, or (internally) one array or inner array definition in a def file. + * Definitions are comparable based on version. + * @author vegardh + * + */ +public class ConfigDefinition implements Comparable<ConfigDefinition> { + public static final Pattern namePattern = Pattern.compile("[a-zA-Z][a-zA-Z0-9-_]*"); + public static final Pattern namespacePattern = Pattern.compile("[a-zA-Z][a-zA-Z0-9-\\._]*"); + + public static Logger log = Logger.getLogger(ConfigDefinition.class.getName()); + private final String name; + private final String version; + private final String namespace; + protected ConfigDefinition parent = null; + + // TODO Strings without default are null, could be not OK. + private Map<String, StringDef> stringDefs = new LinkedHashMap<String, StringDef>(); + private Map<String, BoolDef> boolDefs = new LinkedHashMap<String, BoolDef>(); + private Map<String, IntDef> intDefs = new LinkedHashMap<String, IntDef>(); + private Map<String, LongDef> longDefs = new LinkedHashMap<String, LongDef>(); + private Map<String, DoubleDef> doubleDefs = new LinkedHashMap<String, DoubleDef>(); + private Map<String, EnumDef> enumDefs = new LinkedHashMap<String, EnumDef>(); + private Map<String, RefDef> referenceDefs = new LinkedHashMap<String, RefDef>(); + private Map<String, FileDef> fileDefs = new LinkedHashMap<String, FileDef>(); + private Map<String, PathDef> pathDefs = new LinkedHashMap<>(); + private Map<String, StructDef> structDefs = new LinkedHashMap<String, StructDef>(); + private Map<String, InnerArrayDef> innerArrayDefs = new LinkedHashMap<String, InnerArrayDef>(); + private Map<String, ArrayDef> arrayDefs = new LinkedHashMap<String, ArrayDef>(); + private Map<String, LeafMapDef> leafMapDefs = new LinkedHashMap<>(); + private Map<String, StructMapDef> structMapDefs = new LinkedHashMap<>(); + + public static final Integer INT_MIN = -0x80000000; + public static final Integer INT_MAX = 0x7fffffff; + + public static final Long LONG_MIN = -0x8000000000000000L; + public static final Long LONG_MAX = 0x7fffffffffffffffL; + + public static final Double DOUBLE_MIN = -1e308d; + public static final Double DOUBLE_MAX = 1e308d; + + public ConfigDefinition(String name, String version, String namespace) { + this.name = name; + this.version = version; + this.namespace = namespace; + } + + public ConfigDefinition(String name, String version) { + this(name, version, CNode.DEFAULT_NAMESPACE); + } + + public String getName() { + return name; + } + + public String getVersion() { + return version; + } + + public String getNamespace() { + return namespace; + } + + /** @return The parent ConfigDefinition, or null if this is the root. */ + public ConfigDefinition getParent() { + return parent; + } + + /** @return The root ConfigDefinition, might be this. */ + public ConfigDefinition getRoot() { + ConfigDefinition ancestor = this; + while (ancestor.getParent() != null) { + ancestor = ancestor.getParent(); + } + return ancestor; + } + + private static void defFail(String id, String val, String type, Exception e, List<String> warnings) { + defFail("Invalid value '" + val + "' for " + type + " '" + id + "'. "+ Exceptions.toMessageString(e), warnings); + } + + public void verify(String id, String val, List<String> warnings) { + if (stringDefs.containsKey(id)) { + verifyString(id, warnings); + } else if (enumDefs.containsKey(id)) { + verifyEnum(id ,val, warnings); + } else if (referenceDefs.containsKey(id)) { + verifyReference(id, warnings); + } else if (fileDefs.containsKey(id)) { + verifyFile(id, warnings); + } else if (pathDefs.containsKey(id)) { + verifyPath(id, warnings); + } else if (boolDefs.containsKey(id)) { + verifyBool(id, val, warnings); + } else if (intDefs.containsKey(id)) { + verifyInt(id, val, warnings); + } else if (longDefs.containsKey(id)) { + verifyLong(id, val, warnings); + } else if (doubleDefs.containsKey(id)) { + verifyDouble(id, val, warnings); + } else if (structDefs.containsKey(id)) { + verifyStruct(id, warnings); + } else if (arrayDefs.containsKey(id)) { + verifyArray(id, warnings); + } else if (innerArrayDefs.containsKey(id)) { + verifyInnerArray(id, warnings); + } else if (leafMapDefs.containsKey(id)) { + verifyLeafMap(id, warnings); + } else if (structMapDefs.containsKey(id)) { + verifyStructMap(id, warnings); + } else { + defFail("No such field in definition " + getRoot().getNamespace() + "." + getRoot().getName() + + ": " + getAncestorString() + id, warnings); + } + } + + private boolean verifyDouble(String id, String val, List<String> warnings) { + try { + return verifyDouble(id, Double.parseDouble(val), warnings); + } catch (NumberFormatException e) { + defFail(id, val, "double", e, warnings); + return false; + } + } + + private boolean verifyBool(String id, String val, List<String> warnings) { + if ("true".equalsIgnoreCase(val) || "false".equalsIgnoreCase(val)) { + return verifyBool(id, warnings); + } else { + defFail(id, val, "bool", null, warnings); + return false; + } + } + + public void verify(String id, List<String> warnings) { + verify(id, null, warnings); + } + + /** + * Compares def-versions. Examples: 2 is higher than 1, and 2-0-0 is higher than 1-2-2 but the same as 2. + */ + public static class VersionComparator implements Comparator<String> { + int[] parseVersion(String version) { + int[] result = {0, 0, 0}; + String[] v = version.split("-"); + + for (int i = 0; i < 3; i++) { + if (v.length > i) result[i] = Integer.parseInt(v[i]); + } + + return result; + } + + public int compare(String o1, String o2) throws ClassCastException { + int[] version1 = parseVersion(o1); + int[] version2 = parseVersion(o2); + + for (int i = 0; i < 3; i ++) { + int diff = version1[i] - version2[i]; + if (diff != 0) return diff; + } + + return 0; + } + } + + /** + * String based ("untyped") type specification used by parser and arrays. May have the name of the field which it describes. + * The index number is used to export data in correct order. + * @author vegardh + * + */ + public static class TypeSpec { + private String type; // TODO Class? + private Integer index; + private String name; + private Object defVal; + private Object min; + private Object max; + private List<String> enumVals; + + public TypeSpec(String name, String type, Object defVal, String enumValsCommaSep, Object min, Object max) { + this.name=name; + this.type = type; + this.defVal = defVal; + this.enumVals = getEnumVals(enumValsCommaSep); + this.min = min; + this.max = max; + } + + private List<String> getEnumVals(String commaSep) { + if (commaSep==null) { + return null; + } + List<String> in = new ArrayList<String>(); + for (String val: commaSep.split(",")) { + in.add(val.trim()); + } + return in; + } + public String getName() { + return name; + } + public String getType() { + return type; + } + /*public Class getTypeClass() { + return typeClass; + }*/ + public Object getDef() { + return defVal; + } + public Object getMin() { + return min; + } + public Object getMax() { + return max; + } + public List<String> getEnumVals() { + return enumVals; + } + + public boolean checkValue(String id, String val, int index, List<String> warnings) { + if ("int".equals(getType())) { + return checkInt(id, val, index, warnings); + } else if ("long".equals(getType())) { + return checkLong(id, val, index, warnings); + } else if ("double".equals(getType())) { + return checkDouble(id, val, index, warnings); + } else if ("enum".equals(getType())) { + return checkEnum(id, val, index, warnings); + } + return true; + } + + private boolean checkEnum(String id, String val, int index, List<String> warnings) { + if (!getEnumVals().contains(val)) { + ConfigDefinition.failInvalidEnum(val, id, id+"["+index+"]", warnings); + return false; + } + return true; + } + + private boolean checkDouble(String id, String val, int index, List<String> warnings) { + try { + return checkDouble(Double.parseDouble(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "double", e, warnings); + return false; + } + } + + private boolean checkLong(String id, String val, int index, List<String> warnings) { + try { + return checkLong(Long.parseLong(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "long", e, warnings); + return false; + } + } + + private boolean checkInt(String id, String val, int index, List<String> warnings) { + try { + return checkInt(Integer.parseInt(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "int", e, warnings); + return false; + } + } + + private boolean checkInt(Integer theVal, String id, int arrayIndex, List<String> warnings) { + if (!"int".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\"", warnings); + return false; + } + if (getMax()!=null && theVal>(Integer)getMax()) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Integer)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + private boolean checkLong(Long theVal, String id, int arrayIndex, List<String> warnings) { + if (!"long".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\"", warnings); + return false; + } + if (getMax()!=null && theVal>(Long)getMax()) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Long)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + private boolean checkDouble(Double theVal, String id, int arrayIndex, List<String> warnings) { + if (!"double".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\", array type is "+getType(), warnings); + return false; + } + if (getMax()!=null && (theVal>(Double)getMax())) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Double)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + public void setIndex(Integer index) { + this.index = index; + } + public Integer getIndex() { + return index; + } + } + + /** + * A ConfigDefinition that represents a struct, e.g. a.foo, a.bar where 'a' is the struct. Can be thought + * of as an inner array with only one element. + */ + public static class StructDef extends ConfigDefinition { + public StructDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * An InnerArray def is a ConfigDefinition with n scalar types of defs, and maybe sub-InnerArrays + * @author vegardh + * + */ + public static class InnerArrayDef extends ConfigDefinition { + public InnerArrayDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * An array def is a ConfigDefinition with only one other type of scalar def. + * @author vegardh + * + */ + public static class ArrayDef extends ConfigDefinition { + private TypeSpec typeSpec; + public ArrayDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + public TypeSpec getTypeSpec() { + return typeSpec; + } + public void setTypeSpec(TypeSpec typeSpec) { + this.typeSpec = typeSpec; + } + + public void verify(String val, int index, List<String> warnings) { + if (val != null && getTypeSpec() != null) { + TypeSpec spec = getTypeSpec(); + spec.checkValue(getName(), val, index, warnings); + } + } + } + + /** + * Def of a myMap{} int + * @author vegardh + * + */ + public static class LeafMapDef extends ConfigDefinition { + private TypeSpec typeSpec; + public LeafMapDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + public TypeSpec getTypeSpec() { + return typeSpec; + } + public void setTypeSpec(TypeSpec typeSpec) { + this.typeSpec = typeSpec; + } + } + + /** + * Def of a myMap{}.myInt int + * @author vegardh + * + */ + public static class StructMapDef extends ConfigDefinition { + public StructMapDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * A Default specification where instances _may_ have a default value + * @author vegardh + */ + public static interface DefaultValued<T> { + public T getDefVal(); + } + + public static class EnumDef implements DefaultValued<String>{ + private List<String> vals; + private String defVal; + public EnumDef(List<String> vals, String defVal) { + if (defVal!=null && !vals.contains(defVal)) { + throw new IllegalArgumentException("Def val "+defVal+" is not in given vals "+vals); + } + this.vals = vals; + this.defVal = defVal; + } + public List<String> getVals() { + return vals; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class StringDef implements DefaultValued<String> { + private String defVal; + + public StringDef(String def) { + this.defVal=def; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class BoolDef implements DefaultValued<Boolean> { + private Boolean defVal; + + public BoolDef(Boolean def) { + this.defVal=def; + } + + @Override + public Boolean getDefVal() { + return defVal; + } + } + + /** The type is called 'double' in .def files, but it is a 64-bit IEE 754 double, + * which means it must be represented as a double in Java + */ + public static class DoubleDef implements DefaultValued<Double> { + private Double defVal; + private Double min; + private Double max; + public DoubleDef(Double defVal, Double min, Double max) { + super(); + this.defVal = defVal; + if (min == null) { + this.min = DOUBLE_MIN; + } else { + this.min = min; + } + if (max == null){ + this.max = DOUBLE_MAX; + } else { + this.max = max; + } + } + + @Override + public Double getDefVal() { + return defVal; + } + public Double getMin() { + return min; + } + public Double getMax() { + return max; + } + } + + public static class IntDef implements DefaultValued<Integer>{ + private Integer defVal; + private Integer min; + private Integer max; + public IntDef(Integer def, Integer min, Integer max) { + super(); + this.defVal = def; + if (min == null) { + this.min = INT_MIN; + } else { + this.min = min; + } + if (max == null) { + this.max = INT_MAX; + } else { + this.max = max; + } + } + + @Override + public Integer getDefVal() { + return defVal; + } + public Integer getMin() { + return min; + } + public Integer getMax() { + return max; + } + } + + public static class LongDef implements DefaultValued<Long>{ + private Long defVal; + private Long min; + private Long max; + public LongDef(Long def, Long min, Long max) { + super(); + this.defVal = def; + if (min == null) { + this.min = LONG_MIN; + } else { + this.min = min; + } + if (max == null) { + this.max = LONG_MAX; + } else { + this.max = max; + } + } + + @Override + public Long getDefVal() { + return defVal; + } + public Long getMin() { + return min; + } + public Long getMax() { + return max; + } + } + + public static class RefDef implements DefaultValued<String>{ + private String defVal; + + public RefDef(String defVal) { + super(); + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class FileDef implements DefaultValued<String>{ + private String defVal; + + public FileDef(String defVal) { + super(); + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class PathDef implements DefaultValued<String>{ + private String defVal; + + public PathDef(String defVal) { + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public void addEnumDef(String id, EnumDef def) { + enumDefs.put(id, def); + } + + public void addInnerArrayDef(String id) { + innerArrayDefs.put(id, new InnerArrayDef(id, version, this)); + } + + public void addLeafMapDef(String id) { + leafMapDefs.put(id, new LeafMapDef(id, version, this)); + } + + public void addEnumDef(String id, List<String> vals, String defVal) { + List<String> in = new ArrayList<String>(); + for (String ins: vals) { + in.add(ins.trim()); + } + enumDefs.put(id, new EnumDef(in, defVal)); + } + + public void addEnumDef(String id, String valsCommaSep, String defVal) { + String[] valArr = valsCommaSep.split(","); + addEnumDef(id, Arrays.asList(valArr), defVal); + } + + public void addStringDef(String id, String defVal) { + stringDefs.put(id, new StringDef(defVal)); + } + + public void addStringDef(String id) { + stringDefs.put(id, new StringDef(null)); + } + + public void addIntDef(String id, Integer defVal, Integer min, Integer max) { + intDefs.put(id, new IntDef(defVal, min, max)); + } + + public void addIntDef(String id, Integer defVal) { + addIntDef(id, defVal, INT_MIN, INT_MAX); + } + + public void addIntDef(String id) { + addIntDef(id, null); + } + + public void addLongDef(String id, Long defVal, Long min, Long max) { + longDefs.put(id, new LongDef(defVal, min, max)); + } + + public void addLongDef(String id, Long defVal) { + addLongDef(id, defVal, LONG_MIN, LONG_MAX); + } + + public void addLongDef(String id) { + addLongDef(id, null); + } + + public void addBoolDef(String id) { + boolDefs.put(id, new BoolDef(null)); + } + + public void addBoolDef(String id, Boolean defVal) { + boolDefs.put(id, new BoolDef(defVal)); + } + + public void addDoubleDef(String id, Double defVal, Double min, Double max) { + doubleDefs.put(id, new DoubleDef(defVal, min, max)); + } + + public void addDoubleDef(String id, Double defVal) { + addDoubleDef(id, defVal, DOUBLE_MIN, DOUBLE_MAX); + } + + public void addDoubleDef(String id) { + addDoubleDef(id, null); + } + + public void addReferenceDef(String refId, String defVal) { + referenceDefs.put(refId, new RefDef(defVal)); + } + + public void addReferenceDef(String refId) { + referenceDefs.put(refId, new RefDef(null)); + } + + public void addFileDef(String refId, String defVal) { + fileDefs.put(refId, new FileDef(defVal)); + } + + public void addFileDef(String refId) { + fileDefs.put(refId, new FileDef(null)); + } + + public void addPathDef(String refId, String defVal) { + pathDefs.put(refId, new PathDef(defVal)); + } + + public void addPathDef(String refId) { + pathDefs.put(refId, new PathDef(null)); + } + + public Map<String, StringDef> getStringDefs() { + return stringDefs; + } + + public Map<String, BoolDef> getBoolDefs() { + return boolDefs; + } + + public Map<String, IntDef> getIntDefs() { + return intDefs; + } + + public Map<String, LongDef> getLongDefs() { + return longDefs; + } + + public Map<String, DoubleDef> getDoubleDefs() { + return doubleDefs; + } + + public Map<String, RefDef> getReferenceDefs() { + return referenceDefs; + } + + public Map<String, FileDef> getFileDefs() { + return fileDefs; + } + + public Map<String, PathDef> getPathDefs() { + return pathDefs; + } + + public Map<String, InnerArrayDef> getInnerArrayDefs() { + return innerArrayDefs; + } + + public Map<String, LeafMapDef> getLeafMapDefs() { + return leafMapDefs; + } + + public Map<String, StructMapDef> getStructMapDefs() { + return structMapDefs; + } + + public InnerArrayDef innerArrayDef(String name) { + InnerArrayDef ret = innerArrayDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new InnerArrayDef(name, version, this); + innerArrayDefs.put(name, ret); + return ret; + } + + public Map<String, StructDef> getStructDefs() { + return structDefs; + } + + public StructDef structDef(String name) { + StructDef ret = structDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new StructDef(name, version, this); + structDefs.put(name, ret); + return ret; + } + + public Map<String, EnumDef> getEnumDefs() { + return enumDefs; + } + + public ArrayDef arrayDef(String name) { + ArrayDef ret = arrayDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new ArrayDef(name, version, this); + arrayDefs.put(name, ret); + return ret; + } + + public Map<String, ArrayDef> getArrayDefs() { + return arrayDefs; + } + + public StructMapDef structMapDef(String name) { + StructMapDef ret = structMapDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new StructMapDef(name, version, this); + structMapDefs.put(name, ret); + return ret; + } + + public LeafMapDef leafMapDef(String name) { + LeafMapDef ret = leafMapDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new LeafMapDef(name, version, this); + leafMapDefs.put(name, ret); + return ret; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyDouble(String id, Double val, List<String> warnings) { + DoubleDef def = doubleDefs.get(id); + if (def==null) { + defFail("No such double in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), toString(), getAncestorString()+id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), toString(), getAncestorString()+id, warnings); + return false; + } + return true; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyEnum(String id, String val, List<String> warnings) { + EnumDef def = enumDefs.get(id); + if (def==null) { + defFail("No such enum in " + verifyWarning(id), warnings); + return false; + } + if (!def.getVals().contains(val)) { + defFail("Invalid enum value '"+val+"' in def "+toString()+ + " enum '"+getAncestorString()+id+"'.", warnings); + return false; + } + return true; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyInt(String id, Integer val, List<String> warnings) { + IntDef def = intDefs.get(id); + if (def==null) { + defFail("No such integer in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), name, id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), name, id, warnings); + return false; + } + return true; + } + + private boolean verifyInt(String id, String val, List<String> warnings) { + try { + return verifyInt(id, Integer.parseInt(val), warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "int", e, warnings); + return false; + } + } + + private boolean verifyLong(String id, String val, List<String> warnings) { + try { + return verifyLong(id, Long.parseLong(val), warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "long", e, warnings); + return false; + } + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyLong(String id, Long val, List<String> warnings) { + LongDef def = longDefs.get(id); + if (def==null) { + defFail("No such long in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), name, id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), name, id, warnings); + return false; + } + return true; + } + + static void failTooSmall(Object val, Object min, String defName, String valKey, List<String> warnings) { + defFail("Value \""+valKey+"\" outside range in definition \""+defName+"\": "+val+"<"+min, warnings); + } + + static void failTooBig(Object val, Object max, String defName, String valKey, List<String> warnings) { + defFail("Value \""+valKey+"\" outside range in definition \""+defName+"\": "+val+">"+max, warnings); + } + + static void failInvalidEnum(Object val, String defName, String defKey, List<String> warnings) { + defFail("Invalid enum value \""+val+"\" for \""+defKey+"\" in definition \""+defName, warnings); + } + + /** + * Adds the given log msg to list, and logs it + * @param msg failure message + * @param warnings list of warnings collected during model building. + * @return warnings list with msg added + */ + static List<String> defFail(String msg, List<String> warnings) { + throw new IllegalArgumentException(msg); + // Idea here is to store errors in list instead, and throw from model builder in vespamodel instead. But not so important. + /*warnings.add(msg); + log.log(LogLevel.WARNING, msg); + return warnings;*/ + } + + private boolean verifyString(String id, List<String> warnings) { + if (!stringDefs.containsKey(id)) { + defFail("No such string in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyReference(String id, List<String> warnings) { + if (!referenceDefs.containsKey(id)) { + defFail("No such reference in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyFile(String id, List<String> warnings) { + if (!fileDefs.containsKey(id)) { + defFail("No such file in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyPath(String id, List<String> warnings) { + if (!pathDefs.containsKey(id)) { + defFail("No such path in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyBool(String id, List<String> warnings) { + if (!boolDefs.containsKey(id)) { + defFail("No such bool in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyArray(String id, List<String> warnings) { + String failString = "No such array in " + verifyWarning(id); + if (!arrayDefs.containsKey(id)) { + if (innerArrayDefs.containsKey(id)) { + failString += ". However, the definition does contain an inner array with the same name."; + } + defFail(failString, warnings); + return false; + } + return true; + } + + private boolean verifyInnerArray(String id, List<String> warnings) { + String failString = "No such inner array in " + verifyWarning(id); + if (!innerArrayDefs.containsKey(id)) { + if (arrayDefs.containsKey(id)) { + failString += ". However, the definition does contain an array with the same name."; + } + defFail(failString, warnings); + return false; + } + return true; + } + + private boolean verifyStruct(String id, List<String> warnings) { + if (!structDefs.containsKey(id)) { + defFail("No such struct in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyLeafMap(String id, List<String> warnings) { + if (!leafMapDefs.containsKey(id)) { + defFail("No such leaf map in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyStructMap(String id, List<String> warnings) { + if (!structMapDefs.containsKey(id)) { + defFail("No such struct map in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private String verifyWarning(String id) { + return "definition '" + getRoot().toString() + "': " + getAncestorString() + id; + } + + /** + * Returns a string composed of the ancestors of this ConfigDefinition, skipping the root (which is the name + * of the .def file). For example, if this is an array called 'leafArray' and a child of 'innerArray' which + * is again a child of 'myStruct', then the returned string will be 'myStruct.innerArray.leafArray.' + * The trailing '.' is included for the caller's convenience. + * + * @return a string composed of the ancestors of this ConfigDefinition, not including the root. + */ + public String getAncestorString() { + StringBuilder ret = new StringBuilder(); + ConfigDefinition ancestor = this; + while (ancestor.getParent() != null) { + ret.insert(0, ancestor.getName() + "."); + ancestor = ancestor.getParent(); + } + return ret.toString(); + } + + @Override + public int compareTo(ConfigDefinition other) { + Objects.requireNonNull(other); + if (!getName().equals(other.getName())) { + throw new IllegalArgumentException("Different def names used to compare: "+getName()+"/"+other.getName()); + } + return new VersionComparator().compare(getVersion(),other.getVersion()); + } + + @Override + public String toString() { + return getNamespace() + "." + getName(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java new file mode 100644 index 00000000000..6e9f2de5a0d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.LeafCNode; + +import java.util.Arrays; + +/** + * Builds a ConfigDefinition from a tree of CNodes. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigDefinitionBuilder { + + /** + * Creates a ConfigDefinition based on a tree generated from parsing a config + * definition file. + * + * @param root the root node in a tree generated from parsing a config definition file. + * @return a ConfigDefinition object + */ + public static ConfigDefinition createConfigDefinition(CNode root) { + return createConfigDefinition(root, root.getNamespace()); + } + + // TODO This method should be removed when we have full namespace support + /** + * Creates a ConfigDefinition based on a tree generated from parsing a config + * definition file. The <code>namespace</code> argument overrides the namespace + * defined in the config definition file. + * + * @param root the root node in a tree generated from parsing a config definition file. + * @param namespace Override namespace in root with this namespace + * @return a ConfigDefinition object + */ + public static ConfigDefinition createConfigDefinition(CNode root, String namespace) { + ConfigDefinition def = new ConfigDefinition(root.getName(), root.getVersion(), namespace); + + for (CNode node : root.getChildren()) { + addNode(def, node); + } + return def; + } + + /** + * + * @param def a ConfigDefinition object + * @param node the node to be added to the config definition + */ + private static void addNode(ConfigDefinition def, CNode node) { + String name = node.getName(); + if (node instanceof LeafCNode) { + if (node.isArray) { + //System.out.println("Adding array node " + name); + String enumValues = null; + String type = ((LeafCNode) node).getType(); + if ("enum".equals(type)) { + enumValues = convertToEnumValueCommaSeparated(((LeafCNode.EnumLeaf) node).getLegalValues()); + } + def.arrayDef(name).setTypeSpec( + new ConfigDefinition.TypeSpec(name, ((LeafCNode) node).getType(), null, enumValues, null, null)); + + } else if (node.isMap) { + //System.out.println("Adding leaf map node " + name); + def.leafMapDef(name).setTypeSpec(new ConfigDefinition.TypeSpec(name, ((LeafCNode) node).getType(), null, null, null, null)); + } else { + //System.out.println("Adding basic node " + name); + if (node instanceof LeafCNode.IntegerLeaf) { + addNode(def, (LeafCNode.IntegerLeaf) node); + } else if (node instanceof LeafCNode.LongLeaf) { + addNode(def, (LeafCNode.LongLeaf) node); + } else if (node instanceof LeafCNode.BooleanLeaf) { + addNode(def, (LeafCNode.BooleanLeaf) node); + } else if (node instanceof LeafCNode.DoubleLeaf) { + addNode(def, (LeafCNode.DoubleLeaf) node); + // Need to come before StringLeaf, since it is a subclass of StringLeaf + } else if (node instanceof LeafCNode.ReferenceLeaf) { + addNode(def, (LeafCNode.ReferenceLeaf) node); + } else if (node instanceof LeafCNode.FileLeaf) { + addNode(def, (LeafCNode.FileLeaf) node); + } else if (node instanceof LeafCNode.PathLeaf) { + addNode(def, (LeafCNode.PathLeaf) node); + }else if (node instanceof LeafCNode.StringLeaf) { + addNode(def, (LeafCNode.StringLeaf) node); + } else if (node instanceof LeafCNode.EnumLeaf) { + addNode(def, (LeafCNode.EnumLeaf) node); + } else { + System.err.println("Unknown node type for node with name " + name); + } + } + } else { + ConfigDefinition newDef; + if (node.isArray) { + if (node.getChildren() != null && node.getChildren().length > 0) { + //System.out.println("\tAdding inner array node " + name); + newDef = def.innerArrayDef(name); + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + } else if (node.isMap) { + //System.out.println("Adding struct map node " + name); + newDef = def.structMapDef(name); + if (node.getChildren() != null && node.getChildren().length > 0) { + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + + } else { + //System.out.println("Adding struct node " + name); + newDef = def.structDef(name); + if (node.getChildren() != null && node.getChildren().length > 0) { + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + } + } + } + + + static void addNode(ConfigDefinition def, LeafCNode.IntegerLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addIntDef(leaf.getName(), new Integer(leaf.getDefaultValue().getValue())); + } else { + def.addIntDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.LongLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addLongDef(leaf.getName(), new Long(leaf.getDefaultValue().getValue())); + } else { + def.addLongDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.BooleanLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addBoolDef(leaf.getName(), Boolean.valueOf(leaf.getDefaultValue().getValue())); + } else { + def.addBoolDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.DoubleLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addDoubleDef(leaf.getName(), new Double(leaf.getDefaultValue().getValue())); + } else { + def.addDoubleDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.StringLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addStringDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addStringDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.ReferenceLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addReferenceDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addReferenceDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.FileLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addFileDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addFileDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.PathLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addPathDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addPathDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.EnumLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), leaf.getDefaultValue().getValue()); + } else { + def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), null); + } + } + + static String convertToEnumValueCommaSeparated(String[] enumValues) { + StringBuilder sb = new StringBuilder(); + for (String s : enumValues) { + sb.append(s); + sb.append(", "); + } + int length = sb.length(); + sb.delete(length - 2, length); + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java new file mode 100644 index 00000000000..29c39ef0ab0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Represents one config definition key (name, namespace) + * + * @author vegardh + */ +public class ConfigDefinitionKey { + + private final String name; + private final String namespace; + + /** + * Creates a config definition key. + * @param name config definition name + * @param namespace config definition namespace + */ + public ConfigDefinitionKey(String name, String namespace) { + this.name = name; + this.namespace = namespace; + } + + public ConfigDefinitionKey(ConfigKey<?> key) { + this(key.getName(), key.getNamespace()); + } + + public String getName() { + return name; + } + + public String getNamespace() { + return namespace; + } + + @Override + public boolean equals(Object oth) { + if (!(oth instanceof ConfigDefinitionKey)) { + return false; + } + ConfigDefinitionKey other = (ConfigDefinitionKey) oth; + return name.equals(other.getName()) && + namespace.equals(other.getNamespace()); + } + + @Override + public int hashCode() { + return namespace.hashCode() + name.hashCode(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(namespace).append(".").append(name); + return sb.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java new file mode 100644 index 00000000000..5e5f8db2711 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Class to hold config definitions and resolving requests for the correct definition + * + * @author Harald Musum <musum@yahoo-inc.com> + * @since 5.1 + */ +public class ConfigDefinitionSet { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigDefinitionSet.class.getName()); + + private final Map<ConfigDefinitionKey, ConfigDefinition> defs = new ConcurrentHashMap<ConfigDefinitionKey, ConfigDefinition>(); + + public ConfigDefinitionSet() { + + } + + public void add(ConfigDefinitionKey key, ConfigDefinition def) { + log.log(LogLevel.DEBUG, "Adding to set: " + key); + defs.put(key, def); + } + + /** + * Returns a ConfigDefinition from the set matching the given <code>key</code>. If no ConfigDefinition + * is found in the set, it will try to find a ConfigDefinition with same name in the default namespace. + * @param key a {@link ConfigDefinitionKey} + * @return a ConfigDefinition if found, else null + */ + public ConfigDefinition get(ConfigDefinitionKey key) { + log.log(LogLevel.DEBUG, "Getting from set " + defs + " for key " + key); + ConfigDefinition ret = defs.get(key); + if (ret == null) { + // Return entry if we fallback to default namespace + log.log(LogLevel.DEBUG, "Found no def for key " + key + ", trying to find def with same name in default namespace"); + for (Map.Entry<ConfigDefinitionKey, ConfigDefinition> entry : defs.entrySet()) { + if (key.getName().equals(entry.getKey().getName()) && entry.getKey().getNamespace().equals(CNode.DEFAULT_NAMESPACE)) { + return entry.getValue(); + } + } + } + return ret; + } + + public int size() { + return defs.size(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (ConfigDefinitionKey key : defs.keySet()) { + sb.append(key.toString()).append("\n"); + } + return sb.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java new file mode 100644 index 00000000000..1ce81abd57a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java @@ -0,0 +1,233 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.LeafCNode; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.Stack; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { + private final InnerCNode root; + private DataOutputStream out = null; + private Stack<Node> nodeStack; + + public ConfigFileFormat(InnerCNode root) { + this.root = root; + this.nodeStack = new Stack<>(); + } + + private void printPrefix() throws IOException { + for (Node node : nodeStack) { + CNode cnode = node.node; + if (cnode != root) { + encodeString(cnode.getName()); + if (cnode.isArray) { + encodeString("[" + node.arrayIndex + "]"); + if (!(cnode instanceof LeafCNode)) { + encodeString("."); + } + } else if (cnode.isMap) { + encodeString("{\"" + node.mapKey + "\"}"); + if (!(cnode instanceof LeafCNode)) { + encodeString("."); + } + } else if (cnode instanceof LeafCNode) { + encodeString(""); + } else { + encodeString("."); + } + } + } + encodeString(" "); + } + + private void encode(Inspector inspector, CNode node) throws IOException { + switch (inspector.type()) { + case BOOL: + encodeValue(String.valueOf(inspector.asBool()), (LeafCNode) node); + return; + case LONG: + encodeValue(String.valueOf(inspector.asLong()), (LeafCNode) node); + return; + case DOUBLE: + encodeValue(String.valueOf(inspector.asDouble()), (LeafCNode) node); + return; + case STRING: + encodeValue(inspector.asString(), (LeafCNode) node); + return; + case ARRAY: + encodeArray(inspector, node); + return; + case OBJECT: + if (node.isMap) { + encodeMap(inspector, node); + } else { + encodeObject(inspector, node); + } + return; + case NIX: + case DATA: + throw new IllegalArgumentException("Illegal config format supplied. Unknown type for field '" + node.getName() + "'"); + } + throw new RuntimeException("Should not be reached"); + } + + private void encodeMap(Inspector inspector, final CNode node) { + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + try { + nodeStack.push(new Node(node, -1, name)); + if (inspector.type().equals(Type.OBJECT)) { + encodeObject(inspector, node); + } else { + encode(inspector, node); + } + nodeStack.pop(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + private void encodeArray(Inspector inspector, final CNode node) { + inspector.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + try { + nodeStack.push(new Node(node, idx, "")); + encode(inspector, node); + nodeStack.pop(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + } + + private void encodeObject(Inspector inspector, CNode node) { + if (!node.isArray && !node.isMap) { + nodeStack.push(new Node(node)); + inspector.traverse(this); + nodeStack.pop(); + } else { + inspector.traverse(this); + } + } + + private void encodeValue(String value, LeafCNode node) throws IOException { + printPrefix(); + try { + if (node instanceof LeafCNode.StringLeaf) { + encodeStringQuoted(value); + } else if (node instanceof LeafCNode.IntegerLeaf) { + //Integer.parseInt(value); + encodeString(value); + } else if (node instanceof LeafCNode.LongLeaf) { + //Long.parseLong(value); + encodeString(value); + } else if (node instanceof LeafCNode.DoubleLeaf) { + //Double.parseDouble(value); + encodeString(value); + } else if (node instanceof LeafCNode.BooleanLeaf) { + encodeString(String.valueOf(Boolean.parseBoolean(value))); + } else if (node instanceof LeafCNode.EnumLeaf) { + // LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) node; + // TODO: Reenable this when we can return illegal config id. + // checkLegalEnumValue(enumNode, value); + encodeString(value); + } else { + encodeStringQuoted(value); + } + encodeString("\n"); + } catch (Exception e) { + throw new IllegalArgumentException("Unable to serialize field '" + node.getFullName() + "': ", e); + } + } + + private void checkLegalEnumValue(LeafCNode.EnumLeaf enumNode, String value) { + boolean found = false; + for (String legalVal : enumNode.getLegalValues()) { + if (legalVal.equals(value)) { + found = true; + } + } + if (!found) + throw new IllegalArgumentException("Illegal enum value '" + value + "'"); + } + + private void encodeStringQuoted(String s) throws IOException { + encodeString("\"" + escapeString(s) + "\""); + } + + private String escapeString(String s) { + return ConfigUtils.escapeConfigFormatValue(s); + } + + private void encodeString(String s) throws IOException { + out.write(Utf8.toBytes(s)); + } + + @Override + public void encode(OutputStream os, Slime slime) throws IOException { + encode(os, slime.get()); + } + + public void encode(OutputStream os, Inspector inspector) throws IOException { + this.out = new DataOutputStream(os); + this.nodeStack = new Stack<>(); + nodeStack.push(new Node(root)); + encode(inspector, root); + } + + @Override + public void decode(InputStream is, Slime slime) throws IOException { + throw new UnsupportedOperationException("decode is not supported"); + } + + @Override + public void field(String fieldName, Inspector inspector) { + try { + Node parent = nodeStack.peek(); + CNode child = parent.node.getChild(fieldName); + if (child == null) { + return; // Skip this field to optimistic + } + if (!child.isArray && !child.isMap && child instanceof LeafCNode) { + nodeStack.push(new Node(child)); + encode(inspector, child); + nodeStack.pop(); + } else { + encode(inspector, child); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private class Node { + final int arrayIndex; + final String mapKey; + final CNode node; + Node(CNode node, int arrayIndex, String mapKey) { + this.node = node; + this.arrayIndex = arrayIndex; + this.mapKey = mapKey; + } + + public Node(CNode node) { + this(node, -1, ""); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java b/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java new file mode 100644 index 00000000000..a1469b746dc --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; + +/** + * Helper class for config applications (currently ConfigManager and ConfigProxy). + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigHelper { + private final JRTConnectionPool jrtConnectionPool; + private TimingValues timingValues; + + /** + * @param configSourceSet The set of config sources for this helper. + */ + public ConfigHelper(ConfigSourceSet configSourceSet) { + this(configSourceSet, new TimingValues()); + } + + /** + * @param configSourceSet The set of config sources for this helper. + * @param timingValues values for timeouts and delays, see {@link TimingValues} + */ + public ConfigHelper(ConfigSourceSet configSourceSet, TimingValues timingValues) { + jrtConnectionPool = new JRTConnectionPool(configSourceSet); + this.timingValues = timingValues; + } + + /** + * @return the config sources (remote servers and/or proxies) in this helper's connection pool. + */ + public ConfigSourceSet getConfigSourceSet() { + return jrtConnectionPool.getSourceSet(); + } + + /** + * @return the connection pool for this config helper. + */ + public JRTConnectionPool getConnectionPool() { + return jrtConnectionPool; + } + + /** + * @return the timing values for this config helper. + */ + public TimingValues getTimingValues() { + return timingValues; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java new file mode 100755 index 00000000000..1e8ba43d649 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.CNode; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +/** + * Class for holding the key when doing cache look-ups and other management of config instances. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable<ConfigKey<?>> { + + @NonNull + private final String name; + @NonNull + private final String configId; + @NonNull + private final String namespace; + + // The two fields below are only set when ConfigKey is constructed from a config class. Can be null + private final Class<CONFIGCLASS> configClass; + private final String md5; // config definition md5 + + /** + * Constructs new key + * + * @param name config definition name + * @param configIdString Can be null. + * @param namespace namespace for this config definition + */ + public ConfigKey(String name, String configIdString, String namespace) { + this(name, configIdString, namespace, null, null); + } + + /** + * Creates a new instance from the given class and configId + * + * @param clazz Config class + * @param configIdString config id, can be null. + */ + public ConfigKey(Class<CONFIGCLASS> clazz, String configIdString) { + this(getFieldFromClass(clazz, "CONFIG_DEF_NAME"), + configIdString, getFieldFromClass(clazz, "CONFIG_DEF_NAMESPACE"), getFieldFromClass(clazz, "CONFIG_DEF_MD5"), clazz); + } + + public ConfigKey(String name, String configIdString, String namespace, String defMd5, Class<CONFIGCLASS> clazz) { + if (name == null) + throw new ConfigurationRuntimeException("Config name must be non-null!"); + this.name = name; + this.configId = (configIdString == null) ? "" : configIdString; + this.namespace = (namespace == null) ? CNode.DEFAULT_NAMESPACE : namespace; + this.md5 = (defMd5 == null) ? "" : defMd5; + this.configClass = clazz; + } + + /** + * Comparison sort order: namespace, name, configId. + */ + @Override + public int compareTo(ConfigKey<?> o) { + if (!o.getNamespace().equals(getNamespace())) return getNamespace().compareTo(o.getNamespace()); + if (!o.getName().equals(getName())) return getName().compareTo(o.getName()); + return getConfigId().compareTo(o.getConfigId()); + } + + private static String getFieldFromClass(Class<?> clazz, String fieldName) { + try { + return (String) clazz.getField(fieldName).get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new ConfigurationRuntimeException("No such field '" + fieldName + "' in class " + clazz + ", or could not access field.", e); + } + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof ConfigKey)) { + return false; + } + ConfigKey<?> key = (ConfigKey) o; + return (name.equals(key.name) && + configId.equals(key.configId) && + namespace.equals(key.namespace)); + } + + public int hashCode() { + int hash = 17; + hash = 37 * hash + name.hashCode(); + hash = 37 * hash + configId.hashCode(); + hash = 37 * hash + namespace.hashCode(); + return hash; + } + + @NonNull + public String getName() { + return name; + } + + @NonNull + public String getConfigId() { + return configId; + } + + @NonNull + public String getNamespace() { + return namespace; + } + + @Nullable + public Class<CONFIGCLASS> getConfigClass() { + return configClass; + } + + @Nullable + public String getMd5() { + return md5; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("name="); + sb.append(name); + sb.append(",namespace="); + sb.append(namespace); + sb.append(",configId="); + sb.append(configId); + return sb.toString(); + } + + public static ConfigKey<?> createFull(String name, String configId, String namespace, String md5) { + return new ConfigKey<>(name, configId, namespace, md5, null); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java new file mode 100644 index 00000000000..a4ac34e65aa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.subscription.ConfigInstanceSerializer; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeFormat; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A class that holds a representation of a config payload. + * + * @author lulf + * @since 5.1.6 + */ +public class ConfigPayload { + private final Slime slime; + + public ConfigPayload(Slime slime) { + this.slime = slime; + } + + public static ConfigPayload fromInstance(ConfigInstance instance) { + Slime slime = new Slime(); + ConfigInstanceSerializer serializer = new ConfigInstanceSerializer(slime); + ConfigInstance.serialize(instance, serializer); + return new ConfigPayload(slime); + } + + public static ConfigPayload fromBuilder(ConfigPayloadBuilder builder) { + Slime slime = new Slime(); + builder.resolve(slime.setObject()); + return new ConfigPayload(slime); + } + + public Slime getSlime() { + return slime; + } + + public void serialize(OutputStream os, SlimeFormat format) throws IOException { + format.encode(os, slime); + } + + @Override + public String toString() { + return toString(false); + } + + public String toString(boolean compact) { + return toUtf8Array(compact).toString(); + } + + public ConfigPayload applyDefaultsFromDef(InnerCNode clientDef) { + DefaultValueApplier defaultValueApplier = new DefaultValueApplier(); + defaultValueApplier.applyDefaults(slime, clientDef); + return this; + } + + public static ConfigPayload empty() { + Slime slime = new Slime(); + slime.setObject(); + return new ConfigPayload(slime); + } + + public static ConfigPayload fromString(String jsonString) { + return fromUtf8Array(new Utf8String(jsonString)); + } + + public boolean isEmpty() { + return !slime.get().valid() || slime.get().children() == 0; + } + + public Utf8Array toUtf8Array(boolean compact) { + ByteArrayOutputStream os = new ByteArrayOutputStream(10000); + try { + new JsonFormat(compact).encode(os, slime); + os.close(); + } catch (IOException e) { + e.printStackTrace(); + } + return new Utf8Array(os.toByteArray()); + } + + public static ConfigPayload fromUtf8Array(Utf8Array payload) { + return new ConfigPayload(new JsonDecoder().decode(new Slime(), payload.getBytes())); + } + + public <ConfigType extends ConfigInstance> ConfigType toInstance(Class<ConfigType> clazz, String configId) { + return ConfigInstanceUtil.getNewInstance(clazz, configId, this); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java new file mode 100644 index 00000000000..2b17305252f --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -0,0 +1,498 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Type; +import com.yahoo.text.Utf8; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.nio.file.Path; +import java.util.*; +import java.util.logging.Logger; + +/** + * A utility class that can be used to apply a payload to a config builder. + * + * TODO: This can be refactored a lot, since many of the reflection methods are duplicated + * + * @author lulf, musum, tonyv + * @since 5.1.6 + */ +public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { + private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName()); + + private final ConfigInstance.Builder rootBuilder; + private final ConfigTransformer.PathAcquirer pathAcquirer; + private final Stack<NamedBuilder> stack = new Stack<>(); + + public ConfigPayloadApplier(T builder) { + this(builder, new IdentityPathAcquirer()); + } + + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + this.rootBuilder = builder; + this.pathAcquirer = pathAcquirer; + debug("rootBuilder=" + rootBuilder); + } + + public void applyPayload(ConfigPayload payload) { + stack.push(new NamedBuilder(rootBuilder)); + try { + handleValue(payload.getSlime().get()); + } catch (Exception e) { + throw new RuntimeException("Not able to create config builder for payload:" + payload.toString() + + ", " + Exceptions.toMessageString(e), e); + } + } + + private void handleValue(Inspector inspector) { + switch (inspector.type()) { + case NIX: + case BOOL: + case LONG: + case DOUBLE: + case STRING: + case DATA: + handleLeafValue(inspector); + break; + case ARRAY: + handleARRAY(inspector); + break; + case OBJECT: + handleOBJECT(inspector); + break; + default: + assert false : "Should not be reached"; + } + } + + private void handleARRAY(Inspector inspector) { + trace("Array"); + inspector.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + handleArrayEntry(idx, inspector); + } + }); + } + + private void handleArrayEntry(int idx, Inspector inspector) { + try { + trace("entry, idx=" + idx); + trace("top of stack=" + stack.peek().toString()); + String name = stack.peek().nameStack().peek(); + if (inspector.type().equals(Type.OBJECT)) { + stack.push(createBuilder(stack.peek(), name)); + } + handleValue(inspector); + if (inspector.type().equals(Type.OBJECT)) { + stack.peek().nameStack().pop(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleOBJECT(Inspector inspector) { + trace("Object"); + printStack(); + + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + handleObjectEntry(name, inspector); + } + }); + + trace("Should pop a builder from stack"); + NamedBuilder builder = stack.pop(); + printStack(); + + // Need to set e.g struct(Struct.Builder) here + if (!stack.empty()) { + trace("builder= " + builder); + try { + invokeSetter(stack.peek().builder, builder.peekName(), builder.builder); + } catch (Exception e) { + throw new RuntimeException("Could not set '" + builder.peekName() + + "' for value '" + builder.builder() + "'", e); + } + } + } + + private void handleObjectEntry(String name, Inspector inspector) { + try { + trace("field, name=" + name); + NamedBuilder parentBuilder = stack.peek(); + if (inspector.type().equals(Type.OBJECT)) { + if (isMapField(parentBuilder, name)) { + parentBuilder.nameStack().push(name); + handleMap(inspector); + parentBuilder.nameStack().pop(); + return; + } else { + stack.push(createBuilder(parentBuilder, name)); + } + } else if (inspector.type().equals(Type.ARRAY)) { + for (int i = 0; i < inspector.children(); i++) { + trace("Pushing " + name); + parentBuilder.nameStack().push(name); + } + } else { // leaf + parentBuilder.nameStack().push(name); + } + handleValue(inspector); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleMap(Inspector inspector) { + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + switch (inspector.type()) { + case OBJECT: + handleInnerMap(name, inspector); + break; + case ARRAY: + throw new IllegalArgumentException("Never herd of array inside maps before"); + default: + setMapLeafValue(name, getValueFromInspector(inspector)); + break; + } + } + }); + } + + private void handleInnerMap(String name, Inspector inspector) { + NamedBuilder builder = createBuilder(stack.peek(), stack.peek().peekName()); + setMapLeafValue(name, builder.builder()); + stack.push(builder); + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + handleObjectEntry(name, inspector); + } + }); + stack.pop(); + } + + private void setMapLeafValue(String key, Object value) { + NamedBuilder parent = stack.peek(); + ConfigBuilder builder = parent.builder(); + String methodName = parent.peekName(); + //trace("class to obtain method from: " + builder.getClass().getName()); + try { + // Need to convert reference into actual path if 'path' type is used + if (isPathField(builder, methodName)) { + FileReference wrappedPath = resolvePath((String)value); + invokeSetter(builder, methodName, key, wrappedPath); + } else { + invokeSetter(builder, methodName, key, value); + } + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); + } catch (NoSuchMethodException e) { + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + } + } + + private boolean isMapField(NamedBuilder parentBuilder, String name) { + ConfigBuilder builder = parentBuilder.builder(); + try { + Field f = builder.getClass().getField(name); + return f.getType().getName().equals("java.util.Map"); + } catch (Exception e) { + return false; + } + } + + NamedBuilder createBuilder(NamedBuilder parentBuilder, String name) { + Object builder = parentBuilder.builder(); + Object newBuilder = getBuilderForStruct(findBuilderName(name), name, builder.getClass().getDeclaringClass()); + trace("New builder for " + name + "=" + newBuilder); + trace("Pushing builder for " + name + "=" + newBuilder + " onto stack"); + return new NamedBuilder((ConfigBuilder) newBuilder, name); + } + + private void handleLeafValue(Inspector value) { + trace("String "); + printStack(); + NamedBuilder peek = stack.peek(); + trace("popping name stack"); + String name = peek.nameStack().pop(); + printStack(); + ConfigBuilder builder = peek.builder(); + trace("name=" + name + ",builder=" + builder + ",value=" + value.toString()); + setValueForLeafNode(builder, name, value); + } + + // Sets values for leaf nodes (uses private accessors that take string as argument) + private void setValueForLeafNode(Object builder, String methodName, Inspector value) { + try { + // Need to convert reference into actual path if 'path' type is used + if (isPathField(builder, methodName)) { + FileReference wrappedPath = resolvePath(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, wrappedPath); + } else { + Object object = getValueFromInspector(value); + invokeSetter(builder, methodName, object); + } + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); + } catch (NoSuchMethodException e) { + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + } + } + + private FileReference resolvePath(String value) { + Path path = pathAcquirer.getPath(newFileReference(value)); + return newFileReference(path.toString()); + } + + private FileReference newFileReference(String fileReference) { + try { + Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); + constructor.setAccessible(true); + return constructor.newInstance(fileReference); + } catch (Exception e) { + throw new RuntimeException("Failed invoking FileReference constructor.", e); + } + } + + private final Map<String, Method> methodCache = new HashMap<>(); + private static String methodCacheKey(Object builder, String methodName, Object[] params) { + StringBuilder sb = new StringBuilder(); + sb.append(builder.getClass().getName()) + .append(".") + .append(methodName); + for (Object param : params) { + sb.append(".").append(param.getClass().getName()); + } + return sb.toString(); + } + + private Method lookupSetter(Object builder, String methodName, Object ... params) throws NoSuchMethodException { + Class<?>[] parameterTypes = new Class<?>[params.length]; + for (int i = 0; i < params.length; i++) { + parameterTypes[i] = params[i].getClass(); + } + Method method = builder.getClass().getDeclaredMethod(methodName, parameterTypes); + method.setAccessible(true); + trace("method=" + method + ",params=" + params); + return method; + } + + private void invokeSetter(Object builder, String methodName, Object ... params) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + // TODO: Does not work for native types. + String key = methodCacheKey(builder, methodName, params); + Method method = methodCache.get(key); + if (method == null) { + method = lookupSetter(builder, methodName, params); + methodCache.put(key, method); + } + method.invoke(builder, params); + } + + private Object getValueFromInspector(Inspector inspector) { + switch (inspector.type()) { + case STRING: + return Utf8.toString(inspector.asUtf8()); + case LONG: + return String.valueOf(inspector.asLong()); + case DOUBLE: + return String.valueOf(inspector.asDouble()); + case NIX: + return null; + case BOOL: + return String.valueOf(inspector.asBool()); + case DATA: + return String.valueOf(inspector.asData()); + } + throw new IllegalArgumentException("Unhandled type " + inspector.type()); + } + + + /** + * Checks whether or not this field is of type 'path', in which + * case some special handling might be needed. Caches the result. + */ + private Set<String> pathFieldSet = new HashSet<>(); + private boolean isPathField(Object builder, String methodName) { + String key = pathFieldKey(builder, methodName); + if (pathFieldSet.contains(key)) { + return true; + } + boolean isPath = false; + try { + Field field = builder.getClass().getDeclaredField(methodName); + //Paths are stored as FileReference in Builder. + java.lang.reflect.Type fieldType = field.getGenericType(); + if (fieldType instanceof Class<?> && fieldType == FileReference.class) { + isPath = true; + } else if (fieldType instanceof ParameterizedType) { + isPath = isParameterizedWithPath((ParameterizedType) fieldType); + } + } catch (NoSuchFieldException e) { + } + if (isPath) { + pathFieldSet.add(key); + } + return isPath; + } + + private static String pathFieldKey(Object builder, String methodName) { + return builder.getClass().getName() + "." + methodName; + } + + private boolean isParameterizedWithPath(ParameterizedType fieldType) { + int numTypeArgs = fieldType.getActualTypeArguments().length; + if (numTypeArgs > 0) + return fieldType.getActualTypeArguments()[numTypeArgs - 1] == FileReference.class; + return false; + } + + + private String findBuilderName(String name) { + StringBuilder sb = new StringBuilder(); + sb.append(name.substring(0, 1).toUpperCase()).append(name.substring(1)); + return sb.toString(); + } + + private Constructor<?> lookupBuilderForStruct(String builderName, String name, Class<?> currentClass) { + final String currentClassName = currentClass.getName(); + trace("builderName=" + builderName + ", name=" + name + ",current class=" + currentClassName); + Class<?> structClass = findClass(currentClass, currentClassName + "$" + builderName); + Class<?> structBuilderClass = findClass(structClass, currentClassName + "$" + builderName + "$Builder"); + try { + return structBuilderClass.getDeclaredConstructor(new Class<?>[]{}); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Could not create class '" + "'" + structBuilderClass.getName() + "'"); + } + } + + /** + * Finds a nested class or builder class with the given <code>name</code>name in <code>clazz</code> + * @param clazz a Class + * @param name a name + * @return class found, or throws an exception is no class is found + */ + private Class<?> findClass(Class<?> clazz, String name) { + for (Class<?> cls : clazz.getDeclaredClasses()) { + if (cls.getName().equals(name)) { + trace("Found class " + cls.getName()); + return cls; + } + } + throw new RuntimeException("could not find class representing '" + printCurrentConfigName() + "'"); + } + + private final Map<String, Constructor<?>> constructorCache = new HashMap<>(); + private static String constructorCacheKey(String builderName, String name, Class<?> currentClass) { + return builderName + "." + name + "." + currentClass.getName(); + } + + private Object getBuilderForStruct(String builderName, String name, Class<?> currentClass) { + String key = constructorCacheKey(builderName, name, currentClass); + Constructor<?> ctor = constructorCache.get(key); + if (ctor == null) { + ctor = lookupBuilderForStruct(builderName, name, currentClass); + constructorCache.put(key, ctor); + } + Object builder; + try { + builder = ctor.newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not create class '" + "'" + ctor.getDeclaringClass().getName() + "'"); + } + return builder; + } + + private String printCurrentConfigName() { + StringBuilder sb = new StringBuilder(); + ArrayList<String> stackElements = new ArrayList<>(); + Stack<String> nameStack = stack.peek().nameStack(); + while (!nameStack.empty()) { + stackElements.add(nameStack.pop()); + } + Collections.reverse(stackElements); + for (String s : stackElements) { + sb.append(s); + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); // remove last . + return sb.toString(); + } + + private void debug(String message) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, message); + } + } + + private void trace(String message) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, message); + } + } + + private void printStack() { + trace("stack=" + stack.toString()); + } + + /** + * A class that holds a builder and a stack of names + */ + private static class NamedBuilder { + private ConfigBuilder builder; + private final Stack<String> names = new Stack<>(); // if empty, the builder is the root builder + + NamedBuilder(ConfigBuilder builder) { + this.builder = builder; + } + + NamedBuilder(ConfigBuilder builder, String name) { + this(builder); + names.push(name); + } + + ConfigBuilder builder() { + return builder; + } + + String peekName() { + return names.peek(); + } + + Stack<String> nameStack() { + return names; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(builder() == null ? "null" : builder.toString()).append(" names=").append(names); + return sb.toString(); + } + } + + static class IdentityPathAcquirer implements ConfigTransformer.PathAcquirer { + @Override + public Path getPath(FileReference fileReference) { + return new File(fileReference.value()).toPath(); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java new file mode 100644 index 00000000000..409bf307a34 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java @@ -0,0 +1,527 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.*; + +import java.util.*; + +/** + * Helper class for building Slime config payloads, while supporting referring to payloads with their indices. The + * builder does not care about config field types. This is resolved by the actual config type consumer created + * from the Slime tree. + * + * TODO: Add toString + * @author lulf + * @since 5.1 + */ +public class ConfigPayloadBuilder { + private String value; + private final Map<String, ConfigPayloadBuilder> objectMap; + private final Map<String, Array> arrayMap; + private final Map<String, MapBuilder> mapBuilderMap; + private final ConfigDefinition configDefinition; + private List<String> warnings = new ArrayList<>(); + + /** + * Construct a payload builder that is not a leaf. + */ + public ConfigPayloadBuilder() { + this(null, null, null); + } + + public ConfigPayloadBuilder(ConfigDefinition configDefinition, List<String> warnings) { + this(configDefinition, null, warnings); + } + + /** + * Construct a payload builder with a leaf value + * + * @param value The value of this leaf. + */ + private ConfigPayloadBuilder(String value, List<String> warnings) { + this(null, value, warnings); + } + + private ConfigPayloadBuilder(ConfigDefinition configDefinition, String value, List<String> warnings) { + this.objectMap = new LinkedHashMap<>(); + this.arrayMap = new LinkedHashMap<>(); + this.mapBuilderMap = new LinkedHashMap<>(); + this.value = value; + this.configDefinition = configDefinition; + this.warnings=warnings; + } + + /** + * Set the value of a config field. + * + * @param name Name of the config field. + * @param value Value of the config field. + */ + public void setField(String name, String value) { + validateField(name, value, warnings); + objectMap.put(name, new ConfigPayloadBuilder(value, warnings)); + } + + private void validateField(String name, String value, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, value, warnings); + } + } + + /** + * Get a new payload builder for a config struct, which can be used to add inner values to that struct. + * + * @param name Name of the struct to create. + * @return A payload builder corresponding to the name. + */ + public ConfigPayloadBuilder getObject(String name) { + ConfigPayloadBuilder p = objectMap.get(name); + if (p == null) { + validateObject(name, warnings); + p = new ConfigPayloadBuilder(getStructDef(name), warnings); + objectMap.put(name, p); + } + return p; + } + + private ConfigDefinition getStructDef(String name) { + return (configDefinition == null ? null : configDefinition.getStructDefs().get(name)); + } + + private void validateObject(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + /** + * Create a new array where new values may be added. + * + * @param name Name of array. + * @return Array object supporting adding elements to it. + */ + public Array getArray(String name) { + Array a = arrayMap.get(name); + if (a == null) { + validateArray(name, warnings); + a = new Array(configDefinition, name); + arrayMap.put(name, a); + } + return a; + } + + private void validateArray(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + /** + * Create slime tree from this builder. + * + * @param parent the parent Cursor for this builder + */ + public void resolve(Cursor parent) { + // TODO: Fix so that names do not clash + for (Map.Entry<String, ConfigPayloadBuilder> entry : objectMap.entrySet()) { + String name = entry.getKey(); + ConfigPayloadBuilder value = entry.getValue(); + if (value.getValue() == null) { + Cursor childCursor = parent.setObject(name); + value.resolve(childCursor); + } else { + // TODO: Support giving correct type + parent.setString(name, value.getValue()); + } + } + for (Map.Entry<String, ConfigPayloadBuilder.Array> entry : arrayMap.entrySet()) { + Cursor array = parent.setArray(entry.getKey()); + entry.getValue().resolve(array); + } + for (Map.Entry<String, MapBuilder> entry : mapBuilderMap.entrySet()) { + String name = entry.getKey(); + MapBuilder map = entry.getValue(); + Cursor cursormap = parent.setObject(name); + map.resolve(cursormap); + } + } + + public ConfigPayloadBuilder override(ConfigPayloadBuilder other) { + value = other.value; + for (Map.Entry<String, ConfigPayloadBuilder> entry : other.objectMap.entrySet()) { + String key = entry.getKey(); + ConfigPayloadBuilder value = entry.getValue(); + if (objectMap.containsKey(key)) { + objectMap.put(key, objectMap.get(key).override(value)); + } else { + objectMap.put(key, new ConfigPayloadBuilder(value)); + } + } + for (Map.Entry<String, Array> entry : other.arrayMap.entrySet()) { + String key = entry.getKey(); + Array value = entry.getValue(); + if (arrayMap.containsKey(key)) { + arrayMap.put(key, arrayMap.get(key).override(value)); + } else { + arrayMap.put(key, new Array(value)); + } + } + mapBuilderMap.putAll(other.mapBuilderMap); + return this; + } + + + /** + * Get the value of this field, if any. + * + * @return value of field, null if this is not a leaf. + */ + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + /** + * Create a new map where new values may be added. + * + * @param name Name of map. + * @return Map builder supporting adding elements to it. + */ + public MapBuilder getMap(String name) { + MapBuilder a = mapBuilderMap.get(name); + if (a == null) { + validateMap(name, warnings); + a = new MapBuilder(configDefinition, name); + mapBuilderMap.put(name, a); + } + return a; + } + + /** + * The definition warnings issued for this payload + * @return list of warnings + */ + public List<String> warnings() { + return warnings; + } + + private void validateMap(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + public ConfigDefinition getConfigDefinition() { + return configDefinition; + } + + public class MapBuilder { + private final Map<String, ConfigPayloadBuilder> elements = new LinkedHashMap<>(); + private final ConfigDefinition configDefinition; + private final String name; + public MapBuilder(ConfigDefinition configDefinition, String name) { + this.configDefinition = configDefinition; + this.name = name; + } + + public void put(String key, String value) { + elements.put(key, new ConfigPayloadBuilder(getLeafMapDef(name), value, warnings)); + } + + public ConfigPayloadBuilder put(String key) { + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getStructMapDef(name), warnings); + elements.put(key, p); + return p; + } + + public ConfigPayloadBuilder get(String key) { + ConfigPayloadBuilder builder = elements.get(key); + if (builder == null) { + builder = put(key); + } + return builder; + } + + public void resolve(Cursor parent) { + for (Map.Entry<String, ConfigPayloadBuilder> entry : elements.entrySet()) { + ConfigPayloadBuilder child = entry.getValue(); + String childVal = child.getValue(); + if (childVal != null) { + parent.setString(entry.getKey(), childVal); + } else { + Cursor childCursor = parent.setObject(entry.getKey()); + child.resolve(childCursor); + } + } + } + + private ConfigDefinition.LeafMapDef getLeafMapDef(String name) { + return (configDefinition == null ? null : configDefinition.getLeafMapDefs().get(name)); + } + + private ConfigDefinition getStructMapDef(String name) { + return (configDefinition == null ? null : configDefinition.getStructMapDefs().get(name)); + } + + public Collection<ConfigPayloadBuilder> getElements() { + return elements.values(); + } + } + + /** + * Array modes. + */ + private enum ArrayMode { + INDEX, APPEND + } + + /** + * Representation of a config array, which supports both INDEX and APPEND modes. + */ + public class Array { + private final Map<Integer, ConfigPayloadBuilder> elements = new LinkedHashMap<>(); + private ArrayMode mode = ArrayMode.INDEX; + private final String name; + private final ConfigDefinition configDefinition; + + public Array(ConfigDefinition configDefinition, String name) { + this.configDefinition = configDefinition; + this.name = name; + } + + public Array(Array other) { + this.elements.putAll(other.elements); + this.mode = other.mode; + this.name = other.name; + this.configDefinition = other.configDefinition; + } + + /** + * Append a value to this array. + * + * @param value Value to append. + */ + public void append(String value) { + setAppend(); + validateArrayElement(getArrayDef(name), value, elements.size()); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getArrayDef(name), value, warnings); + elements.put(elements.size(), p); + } + + private void validateArrayElement(ConfigDefinition.ArrayDef arrayDef, String value, int index) { + if (arrayDef != null) { + arrayDef.verify(value, index, warnings); + } + } + + private ConfigDefinition.ArrayDef getArrayDef(String name) { + return (configDefinition == null ? null : configDefinition.getArrayDefs().get(name)); + } + + private ConfigDefinition getInnerArrayDef(String name) { + return (configDefinition == null ? null : configDefinition.getInnerArrayDefs().get(name)); + } + + public Collection<ConfigPayloadBuilder> getElements() { + return elements.values(); + } + + /** + * Create a new slime object and returns its payload builder. Append the element after all other elements + * in the array. + * + * @return a payload builder for the new slime object. + */ + public ConfigPayloadBuilder append() { + setAppend(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getInnerArrayDef(name), warnings); + elements.put(elements.size(), p); + return p; + } + + /** + * Set the value of array element index to value + * + * @param index Index of array element to set. + * @param value Value that the element should point to. + */ + public void set(int index, String value) { + verifyIndex(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(value, warnings); + elements.put(index, p); + } + + /** + * Set Create a payload object for the given index and return it. Any previously stored version will be + * overwritten. + * + * @param index Index of new element. + * @return The payload builder for the newly created slime object. + */ + public ConfigPayloadBuilder set(int index) { + verifyIndex(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getInnerArrayDef(name), warnings); + elements.put(index, p); + return p; + } + + /** + * Get payload builder in this array corresponding to index. If it does not exist, create a new one. + * + * @param index of element to get. + * @return The corresponding ConfigPayloadBuilder. + */ + public ConfigPayloadBuilder get(int index) { + ConfigPayloadBuilder builder = elements.get(index); + if (builder == null) { + if (mode == ArrayMode.APPEND) + builder = append(); + else + builder = set(index); + } + return builder; + } + + /** + * Try to set append mode, but do some checking if indexed mode has been used first. + */ + private void setAppend() { + if (mode == ArrayMode.INDEX && elements.size() > 0) { + throw new IllegalStateException("Cannot append elements to an array in index mode with more than one element"); + } + mode = ArrayMode.APPEND; + } + + /** + * Try and verify that index mode is possible. + */ + private void verifyIndex() { + if (mode == ArrayMode.APPEND) + throw new IllegalStateException("Cannot reference array elements with index once append is done"); + } + + public void resolve(Cursor parent) { + for (Map.Entry<Integer, ConfigPayloadBuilder> entry : elements.entrySet()) { + ConfigPayloadBuilder child = entry.getValue(); + String childVal = child.getValue(); + if (childVal != null) { + parent.addString(childVal); + } else { + Cursor childCursor = parent.addObject(); + child.resolve(childCursor); + } + } + } + + public Array override(Array superior) { + if (mode == ArrayMode.INDEX && superior.mode == ArrayMode.INDEX) { + elements.putAll(superior.elements); + } else { + for (ConfigPayloadBuilder builder : superior.elements.values()) { + append().override(builder); + } + } + return this; + } + } + + private ConfigPayloadBuilder(ConfigPayloadBuilder other) { + this.arrayMap = other.arrayMap; + this.mapBuilderMap = other.mapBuilderMap; + this.value = other.value; + this.objectMap = other.objectMap; + this.configDefinition = other.configDefinition; + this.warnings = other.warnings; + } + + public ConfigPayloadBuilder(ConfigPayload payload) { + this(new BuilderDecoder(payload.getSlime()).decode(payload.getSlime().get())); + } + + private static class BuilderDecoder { + + private final Slime slime; + public BuilderDecoder(Slime slime) { + this.slime = slime; + } + + ConfigPayloadBuilder decode(Inspector element) { + ConfigPayloadBuilder root = new ConfigPayloadBuilder(); + decodeObject(slime, root, element); + return root; + } + + private static void decodeObject(Slime slime, ConfigPayloadBuilder builder, Inspector element) { + BuilderObjectTraverser traverser = new BuilderObjectTraverser(slime, builder); + element.traverse(traverser); + } + + private static void decode(Slime slime, String name, Inspector inspector, ConfigPayloadBuilder builder) { + switch (inspector.type()) { + case STRING: + builder.setField(name, inspector.asString()); + break; + case LONG: + builder.setField(name, String.valueOf(inspector.asLong())); + break; + case DOUBLE: + builder.setField(name, String.valueOf(inspector.asDouble())); + break; + case BOOL: + builder.setField(name, String.valueOf(inspector.asBool())); + break; + case OBJECT: + ConfigPayloadBuilder objectBuilder = builder.getObject(name); + decodeObject(slime, objectBuilder, inspector); + break; + case ARRAY: + ConfigPayloadBuilder.Array array = builder.getArray(name); + decodeArray(slime, array, inspector); + break; + } + } + + private static void decodeArray(Slime slime, Array array, Inspector inspector) { + BuilderArrayTraverser traverser = new BuilderArrayTraverser(slime, array); + inspector.traverse(traverser); + } + + private static class BuilderObjectTraverser implements ObjectTraverser { + private final ConfigPayloadBuilder builder; + private final Slime slime; + public BuilderObjectTraverser(Slime slime, ConfigPayloadBuilder builder) { + this.slime = slime; + this.builder = builder; + } + + @Override + public void field(String name, Inspector inspector) { + decode(slime, name, inspector, builder); + } + } + + private static class BuilderArrayTraverser implements ArrayTraverser { + private final Array array; + private final Slime slime; + public BuilderArrayTraverser(Slime slime, Array array) { + this.array = array; + this.slime = slime; + } + + @Override + public void entry(int idx, Inspector inspector) { + switch (inspector.type()) { + case STRING: + array.append(inspector.asString()); + break; + case OBJECT: + decodeObject(slime, array.append(), inspector); + break; + } + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java new file mode 100644 index 00000000000..3d15804e14a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; + +import java.nio.file.Path; + +import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer; + +/** + * A utility class that can be used to transform config from one format to another. + * + * @author lulf, musum, tonyv + * @since 5.1.6 + */ +public class ConfigTransformer<T extends ConfigInstance> { + /** + * Workaround since FileAcquirer is in a separate module that depends on config. + * Consider moving FileAcquirer into config instead. + */ + public interface PathAcquirer { + Path getPath(FileReference fileReference); + } + + private final Class<T> clazz; + + private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + + /** + * For internal use only * + */ + public static void setPathAcquirer(PathAcquirer pathAcquirer) { + ConfigTransformer.pathAcquirer = (pathAcquirer == null) ? + new IdentityPathAcquirer() : + pathAcquirer; + } + + /** + * Create a transformer capable of converting payloads to clazz + * + * @param clazz a Class for the config instance which this config payload should create a builder for + */ + public ConfigTransformer(Class<T> clazz) { + this.clazz = clazz; + } + + /** + * Create a ConfigBuilder from a payload, based on the <code>clazz</code> supplied. + * + * @param payload a Payload to be transformed to builder. + * @return a ConfigBuilder + */ + public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { + ConfigInstance.Builder builder = getRootBuilder(); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + creator.applyPayload(payload); + return builder; + } + + private ConfigInstance.Builder getRootBuilder() { + ConfigInstance.Builder builder = null; + Class<?>[] classes = clazz.getDeclaredClasses(); + for (Class<?> c : classes) { + if (c.getName().endsWith("Builder")) { + try { + builder = (ConfigInstance.Builder) c.getConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not instantiate builder for " + clazz.getName(), e); + } + } + } + if (builder == null) { + throw new RuntimeException("Could not find builder for " + clazz.getName()); + } else { + return builder; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java new file mode 100644 index 00000000000..eeda2eefdfa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.*; + +/** + * Tool to verify that configs across multiple config servers are the same. + * + * @author lulf + * @since 5.12 + */ +public class ConfigVerification { + private final static int port = 19071; + private final static String prefix = "http://"; + + public static void main(String [] args) throws IOException { + List<String> configservers = new ArrayList<>(); + String tenant = "default"; + String appName = "default"; + String environment = "prod"; + String region = "default"; + String instance= "default"; + for (String arg : args) { + configservers.add(prefix + arg + ":" + port + "/config/v2/tenant/" + tenant + "/application/" + appName + "/environment/" + environment + "/region/" + region + "/instance/" + instance + "/?recursive=true"); + } + System.exit(compareConfigs(listConfigs(configservers))); + } + + private static Map<String, Stack<String>> listConfigs(List<String> urls) throws IOException { + Map<String, String> outputs = performRequests(urls); + + Map<String, Stack<String>> recurseMappings = new LinkedHashMap<>(); + for (Map.Entry<String, String> entry : outputs.entrySet()) { + Slime slime = new JsonDecoder().decode(new Slime(), Utf8.toBytes(entry.getValue())); + final List<String> list = new ArrayList<>(); + slime.get().field("configs").traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + list.add(inspector.asString()); + } + }); + Stack<String> stack = new Stack<>(); + Collections.sort(list); + stack.addAll(list); + recurseMappings.put(entry.getKey(), stack); + } + return recurseMappings; + } + + private static Map<String, String> performRequests(List<String> urls) throws IOException { + Map<String, String> outputs = new LinkedHashMap<>(); + for (String url : urls) { + outputs.put(url, performRequest(url)); + } + return outputs; + } + + private static int compareConfigs(Map<String, Stack<String>> mappings) throws IOException { + for (int n = 0; n < mappings.values().iterator().next().size(); n++) { + List<String> recurseUrls = new ArrayList<>(); + for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { + recurseUrls.add(entry.getValue().pop()); + } + int ret = compareOutputs(performRequests(recurseUrls)); + if (ret != 0) { + return ret; + } + } + return 0; + } + + private static int compareOutputs(Map<String, String> outputs) { + Map.Entry<String, String> firstEntry = outputs.entrySet().iterator().next(); + for (Map.Entry<String, String> entry : outputs.entrySet()) { + if (!entry.getValue().equals(firstEntry.getValue())) { + System.out.println("output from '" + entry.getKey() + "' did not equal output from '" + firstEntry.getKey() + "'"); + return -1; + } + } + return 0; + } + + private static String performRequest(String url) throws IOException { + URLConnection connection = new URL(url).openConnection(); + InputStream response = connection.getInputStream(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int ch; + while ((ch = response.read()) > -1) { + baos.write(ch); + } + return Utf8.toString(baos.toByteArray()); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/Connection.java b/config/src/main/java/com/yahoo/vespa/config/Connection.java new file mode 100644 index 00000000000..5ba9f2b598b --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/Connection.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; + +/** + * @author musum + */ +public interface Connection { + + void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter); + + void setError(int errorCode); + + void setSuccess(); + + String getAddress(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java new file mode 100644 index 00000000000..db21acc3d6e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author musum + */ +public interface ConnectionPool { + + void close(); + + void setError(Connection connection, int i); + + Connection getCurrent(); + + Connection setNewCurrentConnection(); + + int getSize(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java new file mode 100644 index 00000000000..187a0f0199b --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.LeafCNode; +import com.yahoo.slime.*; + +/** + * Applies default values of a given config definition to a slime payload. + * TODO: Support giving correct type of default values + * + * @author lulf + * @since 5.1 + */ +public class DefaultValueApplier { + + public Slime applyDefaults(Slime slime, InnerCNode def) { + applyDefaultsRecursive(slime.get(), def); + return slime; + } + + private void applyDefaultsRecursive(Cursor cursor, InnerCNode def) { + if (def.isArray) { + applyDefaultsToArray(cursor, def); + } else if (def.isMap) { + applyDefaultsToMap(cursor, def); + } else { + applyDefaultsToObject(cursor, def); + } + } + + private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) { + cursor.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + applyDefaultsToObject(cursor.field(name), def); + } + }); + } + + private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) { + cursor.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + applyDefaultsToObject(cursor.entry(idx), def); + } + }); + } + + private void applyDefaultsToObject(Cursor cursor, InnerCNode def) { + for (CNode child : def.getChildren()) { + Cursor childCursor = cursor.field(child.getName()); + if (isLeafNode(child) && canApplyDefault(childCursor, child)) { + applyDefaultToLeaf(cursor, child); + } else if (isInnerNode(child)) { + if (!childCursor.valid()) { + if (child.isArray) { + childCursor = cursor.setArray(child.getName()); + } else { + childCursor = cursor.setObject(child.getName()); + } + } + applyDefaultsRecursive(childCursor, (InnerCNode) child); + } + } + } + + private boolean isInnerNode(CNode child) { + return child instanceof InnerCNode; + } + + private boolean isLeafNode(CNode child) { + return child instanceof LeafCNode; + } + + private void applyDefaultToLeaf(Cursor cursor, CNode child) { + cursor.setString(child.getName(), ((LeafCNode) child).getDefaultValue().getValue()); + } + + private boolean canApplyDefault(Cursor cursor, CNode child) { + return !cursor.valid() && ((LeafCNode) child).getDefaultValue() != null; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java new file mode 100644 index 00000000000..50fbe2170a2 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public final class ErrorCode { + // Cannot find a config with this name, version and config md5sum + public static final int UNKNOWN_CONFIG = 100000; + // No config def with that name or version number + public static final int UNKNOWN_DEFINITION = UNKNOWN_CONFIG + 1; + public static final int UNKNOWN_DEF_MD5 = UNKNOWN_CONFIG + 4; + public static final int UNKNOWN_VESPA_VERSION = UNKNOWN_CONFIG + 5; + + public static final int ILLEGAL_NAME = UNKNOWN_CONFIG + 100; + // Version is not a number + public static final int ILLEGAL_VERSION = UNKNOWN_CONFIG + 101; + public static final int ILLEGAL_CONFIGID = UNKNOWN_CONFIG + 102; + public static final int ILLEGAL_DEF_MD5 = UNKNOWN_CONFIG + 103; + public static final int ILLEGAL_CONFIG_MD5 = UNKNOWN_CONFIG + 104; + // I don't think this will actually happen ... + public static final int ILLEGAL_TIMEOUT = UNKNOWN_CONFIG + 105; + public static final int ILLEGAL_GENERATION = UNKNOWN_CONFIG + 106; + public static final int ILLEGAL_SUB_FLAG = UNKNOWN_CONFIG + 107; + public static final int ILLEGAL_NAME_SPACE = UNKNOWN_CONFIG + 108; + public static final int ILLEGAL_PROTOCOL_VERSION = UNKNOWN_CONFIG + 109; + public static final int ILLEGAL_CLIENT_HOSTNAME = UNKNOWN_CONFIG + 110; + + // hasUpdatedConfig() is true, but generation says the config is older than previous config. + public static final int OUTDATED_CONFIG = UNKNOWN_CONFIG + 150; + + public static final int INTERNAL_ERROR = UNKNOWN_CONFIG + 200; + + public static final int APPLICATION_NOT_LOADED = UNKNOWN_CONFIG + 300; + + public static final int INCONSISTENT_CONFIG_MD5 = UNKNOWN_CONFIG + 400; + + private ErrorCode() { + } + + public static String getName(int error) { + switch(error) { + case UNKNOWN_CONFIG: return "UNKNOWN_CONFIG"; + case UNKNOWN_DEFINITION: return "UNKNOWN_DEFINITION"; + case UNKNOWN_DEF_MD5: return "UNKNOWN_DEF_MD5"; + case ILLEGAL_NAME: return "ILLEGAL_NAME"; + case ILLEGAL_VERSION: return "ILLEGAL_VERSION"; + case ILLEGAL_CONFIGID: return "ILLEGAL_CONFIGID"; + case ILLEGAL_DEF_MD5: return "ILLEGAL_DEF_MD5"; + case ILLEGAL_CONFIG_MD5: return "ILLEGAL_CONFIG_MD5"; + case ILLEGAL_TIMEOUT: return "ILLEGAL_TIMEOUT"; + case ILLEGAL_GENERATION: return "ILLEGAL_GENERATION"; + case ILLEGAL_SUB_FLAG: return "ILLEGAL_SUBSCRIBE_FLAG"; + case ILLEGAL_NAME_SPACE: return "ILLEGAL_NAME_SPACE"; + case ILLEGAL_CLIENT_HOSTNAME: return "ILLEGAL_CLIENT_HOSTNAME"; + case OUTDATED_CONFIG: return "OUTDATED_CONFIG"; + case INTERNAL_ERROR: return "INTERNAL_ERROR"; + case APPLICATION_NOT_LOADED: return "APPLICATION_NOT_LOADED"; + case ILLEGAL_PROTOCOL_VERSION: return "ILLEGAL_PROTOCOL_VERSION"; + case INCONSISTENT_CONFIG_MD5: return "INCONSISTENT_CONFIG_MD5"; + case UNKNOWN_VESPA_VERSION: return "UNKNOWN_VESPA_VERSION"; + default: return "Unknown error"; + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ErrorType.java b/config/src/main/java/com/yahoo/vespa/config/ErrorType.java new file mode 100644 index 00000000000..1371f0e93cc --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ErrorType.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author musum + */ +public enum ErrorType { + TRANSIENT, FATAL; + + public static ErrorType getErrorType(int errorCode) { + switch (errorCode) { + case com.yahoo.jrt.ErrorCode.CONNECTION: + case com.yahoo.jrt.ErrorCode.TIMEOUT: + return ErrorType.TRANSIENT; + case ErrorCode.UNKNOWN_CONFIG: + case ErrorCode.UNKNOWN_DEFINITION: + case ErrorCode.UNKNOWN_DEF_MD5: + case ErrorCode.ILLEGAL_NAME: + case ErrorCode.ILLEGAL_VERSION: + case ErrorCode.ILLEGAL_CONFIGID: + case ErrorCode.ILLEGAL_DEF_MD5: + case ErrorCode.ILLEGAL_CONFIG_MD5: + case ErrorCode.ILLEGAL_TIMEOUT: + case ErrorCode.OUTDATED_CONFIG: + case ErrorCode.INTERNAL_ERROR: + case ErrorCode.APPLICATION_NOT_LOADED: + case ErrorCode.UNKNOWN_VESPA_VERSION: + case ErrorCode.ILLEGAL_PROTOCOL_VERSION: + case ErrorCode.INCONSISTENT_CONFIG_MD5: + return ErrorType.FATAL; + default: + return ErrorType.FATAL; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java new file mode 100644 index 00000000000..904c1d29818 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Interface for counters. + * + * @author lulf + * @since 5.9 + */ +public interface GenerationCounter { + /** + * Increment counter and return new value. + * + * @return incremented counter value. + */ + public long increment(); + + /** + * @return current counter value. + */ + public long get(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java new file mode 100644 index 00000000000..549a51383b1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; + +/** + * + /** + * A generic config with an internal generic builder that mimics a real config builder in order to support builders + * when we don't have the schema. + * + * @author lulf + * @since 5.1 + */ +public class GenericConfig { + public static class GenericConfigBuilder implements ConfigInstance.Builder { + private final ConfigPayloadBuilder payloadBuilder; + private final ConfigDefinitionKey defKey; + public GenericConfigBuilder(ConfigDefinitionKey defKey, ConfigPayloadBuilder payloadBuilder) { + this.defKey = defKey; + this.payloadBuilder = payloadBuilder; + } + private ConfigBuilder override(GenericConfigBuilder superior) { + ConfigPayloadBuilder superiorPayload = superior.payloadBuilder; + payloadBuilder.override(superiorPayload); + return this; + } + + public ConfigPayload getPayload() { return ConfigPayload.fromBuilder(payloadBuilder); } + + @Override + public boolean dispatchGetConfig(ConfigInstance.Producer producer) { + return false; + } + + @Override + public String getDefName() { + return defKey.getName(); + } + + @Override + public String getDefNamespace() { + return defKey.getNamespace(); + } + + @Override + public String getDefMd5() { + return ""; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java new file mode 100644 index 00000000000..2a12a659538 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.protocol.DefContent; + +import java.util.Optional; + +/** + * Interface for getConfig requests. + * @author lulf + * @since 5.3 + */ + +public interface GetConfigRequest { + + /** + * Returns the ConfigKey for this request. + * + * @return the ConfigKey for this config request + */ + public ConfigKey<?> getConfigKey(); + + /** + * The def file contents in the request, or empty array if not sent/not supported + * @return the contents (payload) of the def schema + */ + public DefContent getDefContent(); + + /** + * Get Vespa version for this GetConfigRequest + */ + public Optional<com.yahoo.vespa.config.protocol.VespaVersion> getVespaVersion(); + + /** + * Whether or not the config can be retrieved from or stored in a cache. + * @return true if content should _not_ be cached, false if it should. + */ + public boolean noCache(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java new file mode 100644 index 00000000000..2ddb810981d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.*; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; +import java.util.logging.Logger; + +/** + * A JRT connection to a config server or config proxy. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class JRTConnection implements Connection { + + private final String address; + private final Supervisor supervisor; + private Target target; + + private long lastConnectionAttempt = 0; // Timestamp for last connection attempt + private long lastSuccess = 0; + private long lastFailure = 0; + + private static final long delayBetweenConnectionMessage = 30000; //ms + + private static SimpleDateFormat yyyyMMddz; + static { + yyyyMMddz = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + yyyyMMddz.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + getTarget().invokeAsync(request, jrtTimeout, requestWaiter); + } + + public final static Logger logger = Logger.getLogger(JRTConnection.class.getPackage().getName()); + + + public JRTConnection(String address, Supervisor supervisor) { + this.address = address; + this.supervisor = supervisor; + } + + public String getAddress() { + return address; + } + + /** + * This is synchronized to avoid multiple ConfigInstances creating new targets simultaneously, if + * the existing target is null, invalid or has not yet been initialized. + * + * @return The existing target, or a new one if invalid or null. + */ + public synchronized Target getTarget() { + if (target == null || !target.isValid()) { + if ((System.currentTimeMillis() - lastConnectionAttempt) > delayBetweenConnectionMessage) { + logger.fine("Connecting to " + address); + } + lastConnectionAttempt = System.currentTimeMillis(); + target = supervisor.connect(new Spec(address)); + } + return target; + } + + @Override + public synchronized void setError(int errorCode) { + lastFailure = System.currentTimeMillis(); + } + + @Override + public synchronized void setSuccess() { + lastSuccess = System.currentTimeMillis(); + } + + public void setLastSuccess() { + lastSuccess = System.currentTimeMillis(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Address: "); + sb.append(address); + if (lastSuccess > 0) { + sb.append("\n"); + sb.append("Last success: "); + sb.append(yyyyMMddz.format(lastSuccess)); + } + if (lastFailure > 0) { + sb.append("\n"); + sb.append("Last failure: "); + sb.append(yyyyMMddz.format(lastFailure)); + } + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java new file mode 100644 index 00000000000..5e52dfc5e2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.logging.Logger; + +/** + * A pool of JRT connections to a config source (either a config server or a config proxy). + * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} + * (since the connection is chosen randomly, it might end up using the same connection again, + * and it will always do so if there is only one source). + * The current connection is available with {@link #getCurrent()}. + * When calling {@link #setError(Connection, int)}, {#link #setNewCurrentConnection} will always be called. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author musum + */ +public class JRTConnectionPool implements ConnectionPool { + private static final Logger log = Logger.getLogger(JRTConnectionPool.class.getName()); + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Map<String, JRTConnection> connections = new LinkedHashMap<>(); + + // The config sources used by this connection pool. + private ConfigSourceSet sourceSet = null; + + // The current connection used by this connection pool. + private volatile JRTConnection currentConnection; + + public JRTConnectionPool(ConfigSourceSet sourceSet) { + addSources(sourceSet); + } + + public JRTConnectionPool(List<String> addresses) { + this(new ConfigSourceSet(addresses)); + } + + public void addSources(ConfigSourceSet sourceSet) { + this.sourceSet = sourceSet; + synchronized (connections) { + for (String address : sourceSet.getSources()) { + connections.put(address, new JRTConnection(address, supervisor)); + } + } + setNewCurrentConnection(); + } + + /** + * Returns the current JRTConnection instance + * + * @return a JRTConnection + */ + public synchronized JRTConnection getCurrent() { + return currentConnection; + } + + /** + * Returns and set the current JRTConnection instance by randomly choosing + * from the available sources (this means that you might end up using + * the same connection). + * + * @return a JRTConnection + */ + public synchronized JRTConnection setNewCurrentConnection() { + List<JRTConnection> sources = getSources(); + currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Choosing new connection: " + currentConnection); + } + return currentConnection; + } + + List<JRTConnection> getSources() { + List<JRTConnection> ret = new ArrayList<>(); + synchronized (connections) { + for (JRTConnection source : connections.values()) { + ret.add(source); + } + } + return ret; + } + + ConfigSourceSet getSourceSet() { + return sourceSet; + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + setNewCurrentConnection(); + } + + public JRTConnectionPool updateSources(List<String> addresses) { + ConfigSourceSet newSources = new ConfigSourceSet(addresses); + return updateSources(newSources); + } + + public JRTConnectionPool updateSources(ConfigSourceSet sourceSet) { + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + conn.getTarget().close(); + } + connections.clear(); + addSources(sourceSet); + } + return this; + } + + public String getAllSourceAddresses() { + StringBuilder sb = new StringBuilder(); + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + sb.append(conn.getAddress()); + sb.append(","); + } + } + // Remove trailing "," + sb.deleteCharAt(sb.length() - 1); + return sb.toString(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + sb.append(conn.toString()); + sb.append("\n"); + } + } + return sb.toString(); + } + + public void close() { + supervisor.transport().shutdown().join(); + } + + @Override + public int getSize() { + synchronized (connections) { + return connections.size(); + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java b/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java new file mode 100644 index 00000000000..22c57413bc1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; + +/** + * Defines methods used for RPC config requests. + */ +public class JRTMethods { + + static final String getConfigMethodName = "getConfig"; + private static final String getConfigRequestTypes = "sssssll"; + static final String getConfigResponseTypes = "sssssilS"; + + static final String configV1getConfigMethodName = "config.v1.getConfig"; + private static final String configV1GetConfigRequestTypes = "sssssllsSi"; + static final String configV1GetConfigResponseTypes = "sssssilSs"; + + /** + * Creates a Method object for the RPC method getConfig. + * + * @param handler the object that will handle the method call + * @param handlerMethod the method belonging to the handler that will handle the method call + * @return a Method + */ + public static Method createGetConfigMethod(Object handler, String handlerMethod) { + return new Method(getConfigMethodName, getConfigRequestTypes, getConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config") + .paramDesc(0, "defName", "config class definition name") + .paramDesc(1, "defVersion", "config class definition version") + .paramDesc(2, "defMD5", "md5sum for config class definition") + .paramDesc(3, "configid", "config id") + .paramDesc(4, "configMD5", "md5sum for last got config, empty string if unknown") + .paramDesc(5, "timestamp", + "timestamp for last got config, only relevant to the server if useTimestamp != 0") + .paramDesc(6, "timeout", "timeout (milliseconds) before answering request if config is unchanged") + .returnDesc(0, "defName", "config name") + .returnDesc(1, "defVersion", "config version") + .returnDesc(2, "defMD5", "md5sum for config class definition") + .returnDesc(3, "configid", "requested config id") + .returnDesc(4, "configMD5", "md5sum for this config") + .returnDesc(5, "changed", "changed flag (1 if config changed, 0 otherwise") + .returnDesc(6, "timestamp", "timestamp when config was last changed") + .returnDesc(7, "payload", "config payload for the requested config"); + } + + /** + * Creates a Method object for the RPC method config.v1.getConfig. Use both for + * getting config and subscribing to config + * + * @param handler the object that will handle the method call + * @param handlerMethod the method belonging to the handler that will handle the method call + * @return a Method + */ + public static Method createConfigV1GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV1getConfigMethodName, configV1GetConfigRequestTypes, configV1GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v1") + .paramDesc(0, "defName", "config class definition name") + .paramDesc(1, "defVersion", "config class definition version") + .paramDesc(2, "defMD5", "md5sum for config class definition") + .paramDesc(3, "configid", "config id") + .paramDesc(4, "configMD5", "md5sum for last got config, empty string if unknown") + .paramDesc(5, "generation", + "generation for last got config, only relevant to the server if generation != 0") + .paramDesc(6, "timeout", "timeout (milliseconds) before answering request if config is unchanged") + .paramDesc(7, "namespace", "namespace for defName") + .paramDesc(8, "defContent", "config definition content") + .paramDesc(9, "subscribe", "subscribe to config (1) or not (0)") + .returnDesc(0, "defName", "config name") + .returnDesc(1, "defVersion", "config version") + .returnDesc(2, "defMD5", "md5sum for config class definition") + .returnDesc(3, "configid", "requested config id") + .returnDesc(4, "configMD5", "md5sum for this config") + .returnDesc(5, "changed", "changed flag (1 if config changed, 0 otherwise") // TODO Maybe remove? + .returnDesc(6, "generation", "generation of config") + .returnDesc(7, "payload", "config payload for the requested config") + .returnDesc(8, "namespace", "namespace for defName"); + } + + public static final String configV2getConfigMethodName = "config.v2.getConfig"; + private static final String configV2GetConfigRequestTypes = "s"; + private static final String configV2GetConfigResponseTypes = "s"; + public static Method createConfigV2GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV2getConfigMethodName, configV2GetConfigRequestTypes, configV2GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v2") + .paramDesc(0, "request", "config request") + .returnDesc(0, "response", "config response"); + } + + public static boolean checkV2ReturnTypes(Request request) { + return request.checkReturnTypes(JRTMethods.configV2GetConfigResponseTypes); + } + + public static final String configV3getConfigMethodName = "config.v3.getConfig"; + private static final String configV3GetConfigRequestTypes = "s"; + private static final String configV3GetConfigResponseTypes = "sx"; + public static Method createConfigV3GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV3getConfigMethodName, configV3GetConfigRequestTypes, configV3GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v3") + .paramDesc(0, "request", "config request") + .returnDesc(0, "response", "config response") + .returnDesc(1, "payload", "config response payload"); + + } + + public static boolean checkV3ReturnTypes(Request request) { + return request.checkReturnTypes(JRTMethods.configV3GetConfigResponseTypes); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java new file mode 100644 index 00000000000..4c51afa3afe --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.util.ConfigUtils; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; + +/** + * Wrapper for LZ4 compression that selects compression level based on properties. + * + * @author lulf + * @since 5.19 + */ +public class LZ4PayloadCompressor { + private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance(); + private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL"; + private static final int compressionLevel = getCompressionLevel(); + + private static int getCompressionLevel() { + return Integer.parseInt(ConfigUtils.getEnvValue("0", + System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL), + System.getenv("services__config_protocol_compression_level"), + System.getProperty(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL))); + } + + public byte[] compress(byte[] input) { + return getCompressor().compress(input); + } + + public void decompress(byte[] input, byte[] outputbuffer) { + if (input.length > 0) { + lz4Factory.safeDecompressor().decompress(input, outputbuffer); + } + } + + private LZ4Compressor getCompressor() { + return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java new file mode 100755 index 00000000000..a7c4f4bf788 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.vespa.config.protocol.JRTConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Encapsulates config, usually associated with a {@link JRTConfigRequest}. An instance of this class can represent + * either a config that is not yet resolved, a successfully resolved config, or an error. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class RawConfig { + + private final ConfigKey<?> key; + private final String defMd5; + private final List<String> defContent; + private final Payload payload; + private final int errorCode; + private final String configMd5; + private final Optional<VespaVersion> vespaVersion; + private long generation; + + /** + * Constructor for an empty config (not yet resolved). + * @param key The ConfigKey + * @param defMd5 The md5 sum of the .def-file. + */ + public RawConfig(ConfigKey<?> key, String defMd5) { + this(key, defMd5, null, "", 0L, 0, Collections.<String>emptyList(), Optional.empty()); + } + + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this(key, defMd5, payload, configMd5, generation, 0, defContent, vespaVersion); + } + + /** Copy constructor */ + public RawConfig(RawConfig rawConfig) { + this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5, + rawConfig.generation, rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion()); + } + + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, + String configMd5, long generation, int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this.key = key; + this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent); + this.payload = payload; + this.configMd5 = configMd5; + this.generation = generation; + this.errorCode = errorCode; + this.defContent = defContent; + this.vespaVersion = vespaVersion; + } + + /** + * Creates a new Config from the given request, with the values in the response parameters. + * @param req a {@link JRTClientConfigRequest} + */ + public static RawConfig createFromResponseParameters(JRTClientConfigRequest req) { + return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5(), req.getNewPayload(), req.getNewConfigMd5(), + req.getNewGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + } + + /** + * Creates a new Config from the given request, with the values in the response parameters. + * @param req a {@link JRTClientConfigRequest} + */ + public static RawConfig createFromServerRequest(JRTServerConfigRequest req) { + return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5() , Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), req.getRequestConfigMd5(), + req.getRequestGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + } + + + public ConfigKey<?> getKey() { + return key; + } + + public String getName() { + return key.getName(); + } + + public String getNamespace() { + return key.getNamespace(); + } + + public String getConfigId() { + return key.getConfigId(); + } + + public String getConfigMd5() { + return configMd5; + } + + public String getDefMd5() { + return defMd5; + } + + public long getGeneration() { + return generation; + } + + public void setGeneration(long generation) { + this.generation = generation; + } + + public Payload getPayload() { + return payload; + } + + public int errorCode() { + return errorCode; + } + + public String getDefNamespace() { + return key.getNamespace(); + } + + public Optional<VespaVersion> getVespaVersion() { + return vespaVersion; + } + + /** + * Returns true if this config is equal to the config (same payload md5) in the given request. + * + * @param req The request for which to compare config payload with this config. + * @return true if this config is equal to the config in the given request. + */ + public boolean hasEqualConfig(JRTServerConfigRequest req) { + return (getConfigMd5().equals(req.getRequestConfigMd5())); + } + + /** + * Returns true if this config has a more recent generation than the config in the given request. + * + * @param req The request for which to compare generation with this config. + * @return true if this config has a more recent generation than the config in the given request. + */ + public boolean hasNewerGeneration(JRTServerConfigRequest req) { + return (getGeneration() > req.getRequestGeneration()); + } + + /** + * Convenience method. + * @return true if errorCode() returns 0, false otherwise. + */ + public boolean isError() { + return (errorCode() != 0); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof RawConfig)) { + return false; + } + RawConfig other = (RawConfig) o; + if (! (key.equals(other.key) && + defMd5.equals(other.defMd5) && + (errorCode == other.errorCode)) ) { + return false; + } + // Need to check error codes before isError, since unequal error codes always means unequal requests, + // while non-zero and equal error codes means configs are equal. + if (isError()) + return true; + if (generation != other.generation) + return false; + if (configMd5 != null) { + return configMd5.equals(other.configMd5); + } else { + return (other.configMd5 == null); + } + } + + public int hashCode() { + int hash = 17; + if (key != null) { + hash = 31 * hash + key.hashCode(); + } + if (defMd5 != null) { + hash = 31 * hash + defMd5.hashCode(); + } + hash = 31 * hash + errorCode; + if (! isError()) { + // configMd5 and generation only matter when the RawConfig is not an error. + hash = 31 * hash + (int)(generation ^(generation >>>32)); + if (configMd5 != null) { + hash = 31 * hash + configMd5.hashCode(); + } + } + return hash; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(key.getNamespace()).append(".").append(key.getName()); + sb.append(","); + sb.append(getDefMd5()); + sb.append(","); + sb.append(key.getConfigId()); + sb.append(","); + sb.append(getConfigMd5()); + sb.append(","); + sb.append(getGeneration()); + sb.append(","); + sb.append(getPayload()); + return sb.toString(); + } + + public List<String> getDefContent() { + return defContent; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java new file mode 100644 index 00000000000..6a5052b66cf --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; + +/** + * Extra utilities/operations on slime trees that we would like to have as part of slime in the future, but + * which resides here until we have a better place to put it. + * + * @author lulf + * @since 5.8 + */ +public class SlimeUtils { + public static void copyObject(Inspector from, final Cursor to) { + if (from.type() != Type.OBJECT) { + throw new IllegalArgumentException("Cannot copy object: " + from); + } + from.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + setObjectEntry(inspector, name, to); + } + }); + + } + + private static void setObjectEntry(Inspector from, String name, Cursor to) { + switch (from.type()) { + case NIX: + to.setNix(name); + break; + case BOOL: + to.setBool(name, from.asBool()); + break; + case LONG: + to.setLong(name, from.asLong()); + break; + case DOUBLE: + to.setDouble(name, from.asDouble()); + break; + case STRING: + to.setString(name, from.asString()); + break; + case DATA: + to.setData(name, from.asData()); + break; + case ARRAY: + Cursor array = to.setArray(name); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.setObject(name); + copyObject(from, object); + break; + } + } + + private static void copyArray(Inspector from, final Cursor to) { + from.traverse(new ArrayTraverser() { + @Override + public void entry(int i, Inspector inspector) { + addValue(inspector, to); + } + }); + + } + + private static void addValue(Inspector from, Cursor to) { + switch (from.type()) { + case NIX: + to.addNix(); + break; + case BOOL: + to.addBool(from.asBool()); + break; + case LONG: + to.addLong(from.asLong()); + break; + case DOUBLE: + to.addDouble(from.asDouble()); + break; + case STRING: + to.addString(from.asString()); + break; + case DATA: + to.addData(from.asData()); + break; + case ARRAY: + Cursor array = to.addArray(); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.addObject(); + copyObject(from, object); + break; + } + + } + + public static byte[] toJsonBytes(Slime slime) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new JsonFormat(true).encode(baos, slime); + return baos.toByteArray(); + } + + public static Slime jsonToSlime(byte[] json) { + Slime slime = new Slime(); + new JsonDecoder().decode(slime, json); + return slime; + } + + public static Optional<String> optionalString(Inspector inspector) { + return Optional.of(inspector.asString()).filter(s -> !s.isEmpty()); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/Source.java b/config/src/main/java/com/yahoo/vespa/config/Source.java new file mode 100644 index 00000000000..3ec17038506 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/Source.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigurationRuntimeException; + +import java.util.logging.Logger; + +/** + * A general config source that retrieves config for its SourceConfig. + * + * This class and its subclasses are thread safe. + * + * Note that it is the responsibility of the user to set a source's state to OPEN and CANCELLED, and + * that the READY state can be set by the user as a mark of progress e.g. when waiting for a monitor/lock. + * All other states are set by this class or one of its subclasses. + * + * Originally designed for re-use by closing and reopening, but this caused problems related to + * synchronization between this class and ConfigInstance.subscribeLock. Currently (2008-05-08) + * a source cannot be reopened once it has been cancelled. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public abstract class Source { + + public enum State { NEW, OPEN_PENDING, READY, OPEN, CANCEL_REQUESTED, CANCELLED } + + protected volatile SourceConfig config; + protected volatile State state = State.NEW; + protected long openTimestamp = 0; + public final static Logger logger = Logger.getLogger(Source.class.getPackage().getName()); + + public Source(SourceConfig sourceConfig) { + this.config = sourceConfig; + } + + /** + * Opens this config source. + * Typically called when the first subscriber subscribes to our ConfigInstance. + */ + public final synchronized void open() { + if ((state == State.OPEN) || (state == State.OPEN_PENDING)) { + return; + } else if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + throw new ConfigurationRuntimeException("Subscription with config ID: " + config.getConfigId() + ": Trying to reopen a cancelled source, should not happen.", null); + } + state = State.OPEN_PENDING; + openTimestamp = System.currentTimeMillis(); + myOpen(); + getConfig(); + } + + /** + * Optional subclass hook for the open() method. + */ + protected void myOpen() { } + + /** + * Gets config from this config source. + */ + public final synchronized void getConfig() { + if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + logger.info("Trying to retrieve config from source " + this + " in state: " + state); + return; + } + myGetConfig(); + } + + /** + * Mandatory subclass hook for the getConfig() method. + */ + protected abstract void myGetConfig(); + + /** + * Cancels this config source. Typically called when our ConfigInstance has no more subscribers. + * + * Irreversible. Reopening a cancelled source would cause problems with multiple threads accessing the source + * simultaneously. With better synchronization mechanisms it _should_ be possible to close and reopen a source. + */ + public final void cancel() { + logger.fine("Closing source " + this + " from state " + state); + if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + return; + } + state = State.CANCEL_REQUESTED; + myCancel(); + } + + /** + * Optional subclass hook for the cancel() method. + * Should typically free all the subclass' resources, i.e. requests, threads etc.. + */ + protected void myCancel() { } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java b/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java new file mode 100644 index 00000000000..49484e16ff0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import java.util.List; + +/** + * Interface for config instances that use a {@link Source} to retrieve config values. + * + * @author <a href="gv@yahoo-inc.com">Gj\u00F8ran Voldengen</a> + */ +public interface SourceConfig { + + /** + * Notify subscribers that this config has been initialized by the {@link Source}. + */ + public void notifyInitMonitor(); + + /** + * Sets the fields in the config object from the payload in the given request and updates subscribers + * with the new config. The given request can be an error response with an error code and no payload. + * + * @param req Config request containing return values, or an error response. + */ + public void setConfig(com.yahoo.vespa.config.protocol.JRTClientConfigRequest req); + + /** + * Sets this config's generation. + * + * @param generation The new generation (usually from the source). + */ + public void setGeneration(long generation); + + public String getDefName(); + public String getDefNamespace(); + public String getDefVersion(); + public List<String> getDefContent(); + public String getDefMd5(); + + public String getConfigId(); + public ConfigKey<?> getKey(); + + public String getConfigMd5(); + + public long getGeneration(); + + public RawConfig getConfig(); + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java new file mode 100644 index 00000000000..46f0854084c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -0,0 +1,262 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import java.util.Random; + +/** + * Timeouts, delays and retries used in RPC config protocol. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class TimingValues { + public static final long defaultNextConfigTimeout = 1000; + // See getters below for an explanation of how these values are used and interpreted + // All time values in milliseconds. + private long successTimeout = 600000; + private long errorTimeout = 20000; + private long initialTimeout = 15000; + private long subscribeTimeout = 55000; + private long configuredErrorTimeout = -1; // Don't ever timeout (and do not use error response) when we are already configured + private long nextConfigTimeout = defaultNextConfigTimeout; + + private long fixedDelay = 5000; + private long unconfiguredDelay = 1000; + private long configuredErrorDelay = 15000; + private int maxDelayMultiplier = 10; + private final Random rand; + + public TimingValues() { + this.rand = new Random(System.currentTimeMillis()); + } + + // TODO Should add nextConfigTimeout in all constructors + public TimingValues(long successTimeout, + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier) { + + this.successTimeout = successTimeout; + this.errorTimeout = errorTimeout; + this.initialTimeout = initialTimeout; + this.subscribeTimeout = subscribeTimeout; + this.unconfiguredDelay = unconfiguredDelay; + this.configuredErrorDelay = configuredErrorDelay; + this.fixedDelay = fixedDelay; + this.maxDelayMultiplier = maxDelayMultiplier; + this.rand = new Random(System.currentTimeMillis()); + } + + private TimingValues(long successTimeout, + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier, + Random rand) { + + this.successTimeout = successTimeout; + this.errorTimeout = errorTimeout; + this.initialTimeout = initialTimeout; + this.subscribeTimeout = subscribeTimeout; + this.unconfiguredDelay = unconfiguredDelay; + this.configuredErrorDelay = configuredErrorDelay; + this.fixedDelay = fixedDelay; + this.maxDelayMultiplier = maxDelayMultiplier; + this.rand = rand; + } + + public TimingValues(TimingValues tv) { + this(tv.successTimeout, + tv.errorTimeout, + tv.initialTimeout, + tv.subscribeTimeout, + tv.unconfiguredDelay, + tv.configuredErrorDelay, + tv.fixedDelay, + tv.maxDelayMultiplier, + tv.getRandom()); + } + + public TimingValues(TimingValues tv, Random random) { + this(tv.successTimeout, + tv.errorTimeout, + tv.initialTimeout, + tv.subscribeTimeout, + tv.unconfiguredDelay, + tv.configuredErrorDelay, + tv.fixedDelay, + tv.maxDelayMultiplier, + random); + } + + /** + * Returns timeout to use as server timeout when previous config request was a success. + * + * @return timeout in milliseconds. + */ + public long getSuccessTimeout() { + return successTimeout; + } + + /** + * Returns timeout to use as server timeout when we got an error with the previous config request. + * + * @return timeout in milliseconds. + */ + public long getErrorTimeout() { + return errorTimeout; + } + + /** + * Returns initial timeout to use as server timeout when a config is requested for the first time. + * + * @return timeout in milliseconds. + */ + public long getInitialTimeout() { + return initialTimeout; + } + + public TimingValues setInitialTimeout(long t) { + initialTimeout = t; + return this; + } + + /** + * Returns timeout to use as server timeout when subscribing for the first time. + * + * @return timeout in milliseconds. + */ + public long getSubscribeTimeout() { + return subscribeTimeout; + } + + public TimingValues setSubscribeTimeout(long t) { + subscribeTimeout = t; + return this; + } + + /** + * Returns the time to retry getting config from the remote sources, until the next error response will + * be set as config. Counted from the last ok request was received. A negative value means that + * we will always retry getting config and never set an error response as config. + * + * @return timeout in milliseconds. + */ + public long getConfiguredErrorTimeout() { + return configuredErrorTimeout; + } + + public TimingValues setConfiguredErrorTimeout(long t) { + configuredErrorTimeout = t; + return this; + } + + /** + * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or + * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()} + * + * @return timeout in milliseconds. + */ + public long getNextConfigTimeout() { + return nextConfigTimeout; + } + + public TimingValues setNextConfigTimeout(long t) { + nextConfigTimeout = t; + return this; + } + + /** + * Returns time to wait until next attempt to get config after a failed request when the client has not + * gotten a successful response to a config subscription (i.e, the client has not been configured). + * A negative value means that there will never be a next attempt. If a negative value is set, the + * user must also setSubscribeTimeout(0) to prevent a deadlock while subscribing. + * + * @return delay in milliseconds, a negative value means never. + */ + public long getUnconfiguredDelay() { + return unconfiguredDelay; + } + + public TimingValues setUnconfiguredDelay(long d) { + unconfiguredDelay = d; + return this; + } + + /** + * Returns time to wait until next attempt to get config after a failed request when the client has + * previously gotten a successful response to a config subscription (i.e, the client is configured). + * A negative value means that there will never be a next attempt. + * + * @return delay in milliseconds, a negative value means never. + */ + public long getConfiguredErrorDelay() { + return configuredErrorDelay; + } + + public TimingValues setConfiguredErrorDelay(long d) { + configuredErrorDelay = d; + return this; + } + + /** + * Returns maximum multiplier to use when calculating delay (the delay is multiplied by the number of + * failed requests, unless that number is this maximum multiplier). + * + * @return timeout in milliseconds. + */ + public int getMaxDelayMultiplier() { + return maxDelayMultiplier; + } + + + public TimingValues setSuccessTimeout(long successTimeout) { + this.successTimeout = successTimeout; + return this; + } + + /** + * Returns fixed delay that is used when retrying getting config no matter if it was a success or an error + * and independent of number of retries. + * + * @return timeout in milliseconds. + */ + public long getFixedDelay() { + return fixedDelay; + } + + /** + * Returns a number +/- a random component + * + * @param val input + * @param fraction for instance 0.1 for +/- 10% + * @return a number + */ + public long getPlusMinusFractionRandom(long val, float fraction) { + return Math.round(val - (val * fraction) + (rand.nextFloat() * 2l * val * fraction)); + } + + Random getRandom() { + return rand; + } + + @Override + public String toString() { + return "TimingValues [successTimeout=" + successTimeout + + ", errorTimeout=" + errorTimeout + ", initialTimeout=" + + initialTimeout + ", subscribeTimeout=" + subscribeTimeout + + ", configuredErrorTimeout=" + configuredErrorTimeout + + ", fixedDelay=" + fixedDelay + ", unconfiguredDelay=" + + unconfiguredDelay + ", configuredErrorDelay=" + + configuredErrorDelay + ", maxDelayMultiplier=" + + maxDelayMultiplier + ", rand=" + rand + "]"; + } + + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java b/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java new file mode 100644 index 00000000000..d78eebe3ed4 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Used when a config model does not recognize a config id + * @author vegardh + * + */ +@SuppressWarnings("serial") +public class UnknownConfigIdException extends IllegalArgumentException { + + public UnknownConfigIdException(String msg) { + super(msg); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java new file mode 100644 index 00000000000..07f2911cc69 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -0,0 +1,259 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import com.yahoo.collections.Tuple2; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.*; +import com.yahoo.system.CommandLineParser; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.protocol.*; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.*; + +/** + * A load client for a config server or proxy. + * + * Log messages from a run will have a # first in the line, the end result will not. + * + * @author vegardh + */ +public class LoadTester { + + private static boolean debug = false; + private Transport transport = new Transport(); + protected Supervisor supervisor = new Supervisor(transport); + private List<ConfigKey<?>> configs = new ArrayList<>(); + private Random random = new Random(System.currentTimeMillis()); + private Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs = new HashMap<>(); + private long protocolVersion = Long.parseLong(JRTConfigRequestFactory.getProtocolVersion()); + private CompressionType compressionType = JRTConfigRequestFactory.getCompressionType(); + + /** + * @param args command-line arguments + */ + public static void main(String[] args) throws IOException, InterruptedException { + CommandLineParser parser = new CommandLineParser("LoadTester", args); + parser.addLegalUnarySwitch("-d", "debug"); + parser.addRequiredBinarySwitch("-c", "host (config proxy or server)"); + parser.addRequiredBinarySwitch("-p", "port"); + parser.addRequiredBinarySwitch("-i", "iterations per thread"); + parser.addRequiredBinarySwitch("-t", "threads"); + parser.addLegalBinarySwitch("-l", "configs file, on form name,configid. (To get list: configproxy-cmd -m cache | cut -d ',' -f1-2)"); + parser.addLegalBinarySwitch("-dd", "dir with def files, must be of form name.def"); + parser.parse(); + String host = parser.getBinarySwitches().get("-c"); + int port = Integer.parseInt(parser.getBinarySwitches().get("-p")); + int iterations = Integer.parseInt(parser.getBinarySwitches().get("-i")); + int threads = Integer.parseInt(parser.getBinarySwitches().get("-t")); + String configsList = parser.getBinarySwitches().get("-l"); + String defPath = parser.getBinarySwitches().get("-dd"); + debug = parser.getUnarySwitches().contains("-d"); + LoadTester loadTester = new LoadTester(); + loadTester.runLoad(host, port, iterations, threads, configsList, defPath); + } + + private void runLoad(String host, int port, int iterations, int threads, + String configsList, String defPath) throws IOException, InterruptedException { + configs = readConfigs(configsList); + defs = readDefs(defPath); + List<LoadThread> threadList = new ArrayList<>(); + long start = System.currentTimeMillis(); + Metrics m = new Metrics(); + + for (int i = 0; i < threads; i++) { + LoadThread lt = new LoadThread(iterations, host, port); + threadList.add(lt); + lt.start(); + } + + for (LoadThread lt : threadList) { + lt.join(); + m.merge(lt.metrics); + } + printOutput(start, threads, iterations, m); + } + + private Map<ConfigDefinitionKey, Tuple2<String, String[]>> readDefs(String defPath) throws IOException { + Map<ConfigDefinitionKey, Tuple2<String, String[]>> ret = new HashMap<>(); + if (defPath==null) return ret; + File defDir = new File(defPath); + if (!defDir.isDirectory()) { + System.out.println("# Given def file dir is not a directory: "+defDir.getPath()+" , will not send def contents in requests."); + return ret; + } + final File[] files = defDir.listFiles(); + if (files == null) { + System.out.println("# Given def file dir has no files: "+defDir.getPath()+" , will not send def contents in requests."); + return ret; + } + for (File f : files) { + String name = f.getName(); + if (!name.endsWith(".def")) continue; + String[] splitted = name.split("\\."); + if (splitted.length<2) continue; + String nam = splitted[splitted.length - 2]; + String contents = IOUtils.readFile(f); + ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefContent(nam, Utf8.toBytes(contents)); + ret.put(key, new Tuple2<>(ConfigUtils.getDefMd5(Arrays.asList(contents.split("\n"))), contents.split("\n"))); + } + System.out.println("# Read "+ret.size()+" def files from "+defDir.getPath()); + return ret; + } + + private void printOutput(long start, long threads, long iterations, Metrics metrics) { + long stop = System.currentTimeMillis(); + float durSec = (float) (stop - start) / 1000f; + StringBuilder sb = new StringBuilder(); + sb.append("#reqs/sec #bytes/sec #avglatency #minlatency #maxlatency #failedrequests\n"); + sb.append(((float) (iterations * threads)) / durSec).append(","); + sb.append((metrics.totBytes / durSec)).append(","); + sb.append((metrics.totLatency / threads / iterations)).append(","); + sb.append((metrics.minLatency)).append(","); + sb.append((metrics.maxLatency)).append(","); + sb.append((metrics.failedRequests)); + sb.append("\n"); + System.out.println(sb.toString()); + } + + private List<ConfigKey<?>> readConfigs(String configsList) throws IOException { + List<ConfigKey<?>> ret = new ArrayList<>(); + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(configsList), "UTF-8")); + String str = br.readLine(); + while (str != null) { + String[] nameAndId = str.split(","); + Tuple2<String, String> nameAndNamespace = ConfigUtils.getNameAndNamespaceFromString(nameAndId[0]); + ConfigKey<?> key = new ConfigKey<>(nameAndNamespace.first, nameAndId[1], nameAndNamespace.second); + ret.add(key); + str = br.readLine(); + } + br.close(); + return ret; + } + + private class Metrics { + public long totBytes = 0; + public long totLatency = 0; + public long failedRequests = 0; + public long maxLatency = Long.MIN_VALUE; + public long minLatency = Long.MAX_VALUE; + + public void merge(Metrics m) { + this.totBytes += m.totBytes; + this.totLatency += m.totLatency; + this.failedRequests += m.failedRequests; + updateMin(m.minLatency); + updateMax(m.maxLatency); + } + + + public void update(long bytes, long latency) { + this.totBytes += bytes; + this.totLatency += latency; + updateMin(latency); + updateMax(latency); + } + + private void updateMin(long latency) { + if (latency < minLatency) + minLatency = latency; + } + + private void updateMax(long latency) { + if (latency > maxLatency) + maxLatency = latency; + } + + private void incFailedRequests() { + failedRequests++; + } + } + + private class LoadThread extends Thread { + int iterations = 0; + String host = ""; + int port = 0; + Metrics metrics = new Metrics(); + + public LoadThread(int iterations, String host, int port) { + this.iterations = iterations; + this.host = host; + this.port = port; + } + + @Override + public void run() { + Spec spec = new Spec(host, port); + Target target = connect(spec); + ConfigKey<?> reqKey; + JRTClientConfigRequest request; + int totConfs = configs.size(); + boolean reconnCycle = false; // to log reconn message only once, for instance at restart + for (int i = 0; i < iterations; i++) { + reqKey = configs.get(random.nextInt(totConfs)); + ConfigDefinitionKey dKey = new ConfigDefinitionKey(reqKey); + Tuple2<String, String[]> defContent = defs.get(dKey); + if (defContent==null && defs.size()>0) { // Only complain if we actually did run with a def dir + System.out.println("# No def found for "+dKey+", not sending in request."); + }/* else { + System.out.println("# FOUND: "+dKey+" : "+ StringUtilities.implode(defContent, "\n")); + }*/ + request = getRequest(ConfigKey.createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first), defContent.second); + if (debug) System.out.println("# Requesting: " + reqKey); + long start = System.currentTimeMillis(); + target.invokeSync(request.getRequest(), 10.0); + long end = System.currentTimeMillis(); + if (request.isError()) { + if ("Connection lost".equals(request.errorMessage()) || "Connection down".equals(request.errorMessage())) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (!reconnCycle) { + System.out.println("# Connection lost, reconnecting..."); + reconnCycle = true; + } + target = connect(spec); + } else { + System.err.println(request.errorMessage()); + } + metrics.incFailedRequests(); + } else { + if (reconnCycle) { + reconnCycle = false; + System.out.println("# Connection OK"); + } + long duration = end - start; + + if (debug) { + String payload = request.getNewPayload().toString(); + metrics.update(payload.length(), duration); // assume 8 bit... + System.out.println("# Ret: " + payload); + } else { + metrics.update(0, duration); + } + } + } + } + + private JRTClientConfigRequest getRequest(ConfigKey<?> reqKey, String[] defContent) { + if (defContent==null) defContent=new String[0]; + final long serverTimeout = 1000; + if (protocolVersion == 3) { + return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)), + "unknown", "", 0, serverTimeout, Trace.createDummy(), + compressionType, Optional.empty()); + } else { + throw new RuntimeException("Unsupported protocol version" + protocolVersion); + } + } + + private Target connect(Spec spec) { + return supervisor.connectSync(spec); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java new file mode 100644 index 00000000000..3f2cd9ae2fa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java @@ -0,0 +1,277 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import com.yahoo.jrt.*; +import com.yahoo.system.CommandLineParser; + +import java.io.*; +import java.util.*; + +/** + * /** + * A class for stress-testing config server and config proxy. + * Includes an RPC server interface for communicating + * with test classes that implement the {@link Tester} interface. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.5 + */ +public class StressTester { + private static boolean debug = false; + private final String testClassName; + private final List<Thread> threadList = new ArrayList<>(); + private final List<TestRunner> testRunners = new ArrayList<>(); + + public StressTester(String testClass) { + this.testClassName = testClass; + } + + /** + * @param args command-line arguments + */ + public static void main(String[] args) { + CommandLineParser parser = new CommandLineParser("StressTester", args); + parser.addLegalUnarySwitch("-d", "debug"); + parser.addRequiredBinarySwitch("-c", "host (config proxy or server)"); + parser.addRequiredBinarySwitch("-p", "port"); + parser.addLegalBinarySwitch("-class", "Use class with this name from test bundle (must be given in class path)"); + parser.addLegalBinarySwitch("-serverport", "port for rpc server"); + parser.parse(); + // TODO Handle other hosts and ports + String host = parser.getBinarySwitches().get("-c"); + int port = Integer.parseInt(parser.getBinarySwitches().get("-p")); + debug = parser.getUnarySwitches().contains("-d"); + String classNameInBundle = parser.getBinarySwitches().get("-class"); + int serverPort = Integer.parseInt(parser.getBinarySwitches().get("-serverport")); + RpcServer rpcServer = new RpcServer(null, serverPort, new StressTester(classNameInBundle)); + new Thread(rpcServer).start(); + } + + static class TestRunner implements Runnable { + private final Tester tester; + private volatile boolean stop = false; + + TestRunner(Tester tester) { + this.tester = tester; + } + + @Override + public void run() { + tester.subscribe(); + while (!stop) { + tester.fetch(); + } + tester.close(); + } + + public void stop() { + stop = true; + } + } + + private Map<String, Map<String, String>> getVerificationMap(String verificationFile) { + // Read verification file into a map that test stubs should verify against + Map<String, Map<String, String>> verificationMap = new HashMap<>(); + if (verificationFile != null) { + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(verificationFile)); + String l; + while ((l = reader.readLine()) != null) { + String[] line = l.split(","); + String defFile = line[0]; + String fieldName = line[1]; + String expectedValue = line[2]; + Map<String, String> defExpected = verificationMap.get(defFile); + if (defExpected == null) + defExpected = new HashMap<>(); + defExpected.put(fieldName, expectedValue); + verificationMap.put(defFile, defExpected); + } + } catch (Exception e) { + throw new IllegalArgumentException("Unable to load verification file " + verificationFile); + } finally { + if (reader != null) try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return verificationMap; + } + + private void startTesters(int threads) { + // Load and run actual test stub + Class<?> testClass; + try { + testClass = Class.forName(testClassName); + threadList.clear(); + testRunners.clear(); + for (int i = 0; i < threads; i++) { + Tester tester = (Tester) testClass.newInstance(); + TestRunner testRunner = new TestRunner(tester); + testRunners.add(testRunner); + Thread t = new Thread(testRunner); + threadList.add(t); + } + debug("Starting testers"); + // Now that all testers have been created, start them + for (Thread t : threadList) { + debug("Starting thread"); + t.start(); + } + } catch (Exception e) { + debug("error in startTesters"); + throw new IllegalArgumentException("Unable to load class with name " + testClassName, e); + } + debug("After starting testers"); + } + + public boolean verify(long generation, long timeout, String verificationFile) throws InterruptedException { + Map<String, Map<String, String>> verificationMap = getVerificationMap(verificationFile); + for (TestRunner testRunner : testRunners) { + long start = System.currentTimeMillis(); + boolean ok = false; + do { + if (testRunner.tester.verify(verificationMap, generation)) { + ok = true; + } + Thread.sleep(10); + } while (!ok && (System.currentTimeMillis() - start < timeout)); + if (!ok) { + return false; + } + } + return true; + } + + public void stop() { + debug("Stopping test runners"); + for (TestRunner testRunner : testRunners) { + testRunner.stop(); + } + debug("Stopping threads"); + for (Thread t : threadList) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + debug("End of stop"); + } + + private static void debug(String s) { + if (debug) { + System.out.println(s); + } + } + + public static class RpcServer implements Runnable { + private Transport transport = new Transport(); + protected Supervisor supervisor = new Supervisor(transport); + private final Spec spec; + private final StressTester tester; + + RpcServer(String host, int port, StressTester tester) { + this.tester = tester; + setUp(this); + spec = new Spec(host, port); + } + + public void run() { + try { + Acceptor acceptor = supervisor.listen(spec); + supervisor.transport().join(); + acceptor.shutdown().join(); + } catch (ListenFailedException e) { + throw new RuntimeException("Could not listen to " + spec); + } + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + public final void start(Request request) { + debug("start: Got " + request); + int ret = 1; + int clients = request.parameters().get(0).asInt32(); + debug("start: starting testers"); + try { + tester.startTesters(clients); + ret = 0; + } catch (Exception e) { + debug("start: error: " + e.getMessage()); + e.printStackTrace(); + } + debug("start: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + } + + public final void verify(Request request) { + debug("verify: Got " + request); + long generation = request.parameters().get(0).asInt64(); + String verificationFile = request.parameters().get(1).asString(); + long timeout = request.parameters().get(2).asInt64(); + int ret = 0; + String errorMessage = ""; + try { + if (!tester.verify(generation, timeout, verificationFile)) { + ret = 1; + errorMessage = "Unable to get generation " + generation + " within timeout " + timeout; + } + } catch (Exception e) { + ret = 1; + errorMessage = e.getMessage(); + e.printStackTrace(); + } catch (AssertionError e) { + ret = 1; + errorMessage = e.getMessage(); + } + debug("verify: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + request.returnValues().add(new StringValue(errorMessage)); + } + + public final void stop(Request request) { + debug("stop: Got " + request); + int ret = 1; + try { + tester.stop(); + ret = 0; + } catch (Exception e) { + e.printStackTrace(); + } + debug("stop: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + } + + /** + * Set up RPC method handlers. + * + * @param handler a MethodHandler that will handle the RPC methods + */ + + protected void setUp(Object handler) { + supervisor.addMethod(new Method("start", "i", "i", + handler, "start") + .methodDesc("start") + .paramDesc(0, "clients", "number of clients") + .returnDesc(0, "ret code", "return code, 0 is OK")); + supervisor.addMethod(new Method("verify", "lsl", "is", + handler, "verify") + .methodDesc("verify") + .paramDesc(0, "generation", "config generation") + .paramDesc(1, "verification file", "name of verification file") + .paramDesc(2, "timeout", "timeout when verifying") + .returnDesc(0, "ret code", "return code, 0 is OK") + .returnDesc(1, "error message", "error message, if non zero return code")); + supervisor.addMethod(new Method("stop", "", "i", + handler, "stop") + .methodDesc("stop") + .returnDesc(0, "ret code", "return code, 0 is OK")); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java new file mode 100644 index 00000000000..68d20f7a75f --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import java.util.Map; + +/** + * Tester interface for loadable test runners. + */ +public interface Tester { + public void subscribe(); + public boolean fetch(); + public boolean verify(Map<String, Map<String, String>> expected, long generation); + public void close(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java new file mode 100644 index 00000000000..04799d47494 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; + +/** + * Represents a compilation task that can be run and also collects diagnostic messages from the compilation. + * TODO: Assumes that diagnostics is the same as given to the task, not ideal. + * + * @author lulf + * @since 5.2 + */ +class CompilationTask { + private final JavaCompiler.CompilationTask task; + private final DiagnosticCollector<JavaFileObject> diagnostics; + + CompilationTask(JavaCompiler.CompilationTask task, DiagnosticCollector<JavaFileObject> diagnostics) { + this.task = task; + this.diagnostics = diagnostics; + } + + void call() { + boolean success = task.call(); + if (!success) { + throw new IllegalArgumentException("Compilation diagnostics: " + getDiagnosticMessage()); + } + } + + private String getDiagnosticMessage() { + StringBuilder diagnosticMessages = new StringBuilder(); + for (Diagnostic<?> diagnostic : diagnostics.getDiagnostics()) { + diagnosticMessages.append(diagnostic.getCode()).append("\n"); + diagnosticMessages.append(diagnostic.getKind()).append("\n"); + diagnosticMessages.append(diagnostic.getPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getStartPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getEndPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getSource()).append("\n"); + diagnosticMessages.append(diagnostic.getMessage(null)).append("\n"); + } + return diagnosticMessages.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java new file mode 100644 index 00000000000..403ac61b872 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.config.ConfigInstance; + +/** + * Represents a builder that can be instantiated. + * + * @author lulf + * @since 5.2 + */ +public interface CompiledBuilder { + <BUILDER extends ConfigInstance.Builder> BUILDER newInstance(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java new file mode 100644 index 00000000000..24b102321d2 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +/** + * Interface towards compilers for compiling builders from a config class definition. + * + * @author lulf + * @since 5.2 + */ +public interface ConfigCompiler { + CompiledBuilder compile(ConfigDefinitionClass defClass); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java new file mode 100644 index 00000000000..42e7ebe29b9 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.google.common.io.Files; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.JavaClassBuilder; +import com.yahoo.text.StringUtilities; + +import java.io.File; +import java.io.StringReader; + +/** + * Represents a higher level functionality on a config definition to (in the future) hide the InnerCNode class. + * @author lulf + */ +public class ConfigDefinition { + private final String name; + private final String[] defSchema; + private final InnerCNode cnode; + + public ConfigDefinition(String name, String[] defSchema) { + this.name = name; + this.defSchema = defSchema; + this.cnode = new DefParser(name, new StringReader(StringUtilities.implode(defSchema, "\n"))).getTree(); + } + + // TODO: Remove once no fat bundles are using this. + public ConfigDefinition(InnerCNode targetDef) { + this.name = null; + this.defSchema = null; + this.cnode = targetDef; + } + + public InnerCNode getCNode() { + return cnode; + } + + public ConfigDefinitionClass generateClass() { + File tempDir = Files.createTempDir(); + DefParser parser = new DefParser(name, new StringReader(StringUtilities.implode(defSchema, "\n"))); + JavaClassBuilder builder = new JavaClassBuilder(parser.getTree(), parser.getNormalizedDefinition(), tempDir); + String className = builder.className(); + return new ConfigDefinitionClass(className, builder.javaPackage(), builder.getConfigClass(className)); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java new file mode 100644 index 00000000000..0820d77612e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +/** + * @author lulf + */ +public class ConfigDefinitionClass { + private final String name; + private final String pkg; + private final String definition; + + ConfigDefinitionClass(String name, String pkg, String definition) { + this.name = name; + this.pkg = pkg; + this.definition = definition; + } + + String getDefinition() { + return definition; + } + + String getSimpleName() { + return name; + + } + + String getName() { + return pkg + "." + name; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java new file mode 100644 index 00000000000..957b4a3d3e0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.config.ConfigInstance; + +import javax.tools.*; +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.Locale; + +/** + * Represents a compiler that waits performing the compilation until the requested builder is requested from the + * {@link CompiledBuilder}. + * + * @author lulf + * @since 5.2 + */ +public class LazyConfigCompiler implements ConfigCompiler { + private final File outputDirectory; + private final ClassLoader classLoader; + private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + public LazyConfigCompiler(File outputDirectory) { + this.outputDirectory = outputDirectory; + try { + this.classLoader = new URLClassLoader(new URL[]{outputDirectory.toURI().toURL()}); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Unable to create class loader for directory '" + outputDirectory.getAbsolutePath() + "'", e); + } + } + + @Override + public CompiledBuilder compile(ConfigDefinitionClass defClass) { + Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(new StringSourceObject(defClass.getName(), defClass.getDefinition())); + DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>(); + + StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, Locale.ENGLISH, null); + Iterable<String> options = Arrays.asList("-d", outputDirectory.getAbsolutePath()); + JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, compilationUnits); + return new LazyCompiledBuilder(classLoader, defClass.getName(), new CompilationTask(task, diagnostics)); + } + + /** + * Lazy implementation of compiled builder that defers compilation until class is requested. + */ + private static class LazyCompiledBuilder implements CompiledBuilder { + private final ClassLoader classLoader; + private final String classUrl; + private final CompilationTask compilationTask; + private LazyCompiledBuilder(ClassLoader classLoader, String classUrl, CompilationTask compilationTask) { + this.classLoader = classLoader; + this.classUrl = classUrl; + this.compilationTask = compilationTask; + } + + @Override + public <BUILDER extends ConfigInstance.Builder> BUILDER newInstance() { + compileBuilder(); + String builderClassUrl = classUrl + "$Builder"; + return loadBuilder(builderClassUrl); + + } + + private void compileBuilder() { + try { + compilationTask.call(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Error compiling '" + classUrl + "'", e); + } + } + + @SuppressWarnings("unchecked") + private <BUILDER extends ConfigInstance.Builder> BUILDER loadBuilder(String builderClassUrl) { + try { + Class<BUILDER> clazz = (Class<BUILDER>) classLoader.<BUILDER>loadClass(builderClassUrl); + return clazz.newInstance(); + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { + throw new RuntimeException("Error creating new instance of '" + builderClassUrl + "'", e); + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java new file mode 100644 index 00000000000..01fc0691d2e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import javax.tools.SimpleJavaFileObject; +import java.net.URI; + +/** + * Represents an in memory source object that can be compiled. + * + * @author lulf + * @since 5.2 + */ +class StringSourceObject extends SimpleJavaFileObject { + private final String code; + StringSourceObject(String name, String code) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension),Kind.SOURCE); + this.code = code; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java new file mode 100644 index 00000000000..0a0e85ff47c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/package-info.java b/config/src/main/java/com/yahoo/vespa/config/package-info.java new file mode 100644 index 00000000000..9bb8f40a806 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java b/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java new file mode 100644 index 00000000000..4835464c7f7 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.parser; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java new file mode 100644 index 00000000000..22320ecfa28 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.slime.Inspector; + +import java.io.IOException; + +/** + * Contains info relevant for compression and decompression. + * + * @author lulf + * @since 5.19 + */ +public class CompressionInfo { + private static final String COMPRESSION_TYPE = "compressionType"; + private static final String UNCOMPRESSED_SIZE = "uncompressedSize"; + + public CompressionType getCompressionType() { + return compressionType; + } + public int getUncompressedSize() { + return uncompressedSize; + } + + private final CompressionType compressionType; + private final int uncompressedSize; + + private CompressionInfo(CompressionType compressionType, int uncompressedSize) { + this.compressionType = compressionType; + this.uncompressedSize = uncompressedSize; + } + + public static CompressionInfo uncompressed() { + return new CompressionInfo(CompressionType.UNCOMPRESSED, 0); + } + + public static CompressionInfo create(CompressionType type, int uncompressedSize) { + return new CompressionInfo(type, uncompressedSize); + } + + public static CompressionInfo fromSlime(Inspector field) { + CompressionType type = CompressionType.parse(field.field(COMPRESSION_TYPE).asString()); + int uncompressedSize = (int) field.field(UNCOMPRESSED_SIZE).asLong(); + return new CompressionInfo(type, uncompressedSize); + } + + public void serialize(JsonGenerator jsonGenerator) throws IOException { + jsonGenerator.writeStringField(COMPRESSION_TYPE, compressionType.name()); + jsonGenerator.writeNumberField(UNCOMPRESSED_SIZE, uncompressedSize); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CompressionInfo that = (CompressionInfo) o; + + if (uncompressedSize != that.uncompressedSize) return false; + if (compressionType != that.compressionType) return false; + + return true; + } + + @Override + public int hashCode() { + int result = compressionType.hashCode(); + result = 31 * result + uncompressedSize; + return result; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java new file mode 100644 index 00000000000..bf7e79121ea --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * @author lulf + * @since 5.18 + */ +public enum CompressionType { + UNCOMPRESSED, LZ4; + public static CompressionType parse(String value) { + for (CompressionType type : CompressionType.values()) { + if (type.name().equals(value)) { + return type; + } + } + return CompressionType.UNCOMPRESSED; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java new file mode 100644 index 00000000000..5e6918c1e88 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.text.Utf8Array; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * A config response encapsulates the payload and some meta information. This makes it possible to + * represent the payload in different formats all up to when rendering it to the client. A subclass + * of this must be thread safe, because a response may be cached and, the methods below should be callable + * from multiple request handler threads. + * + * @author lulf + * @since 5.1.14 + */ +public interface ConfigResponse { + + Utf8Array getPayload(); + + List<String> getLegacyPayload(); + + long getGeneration(); + + String getConfigMd5(); + + void serialize(OutputStream os, CompressionType uncompressed) throws IOException; + + default boolean hasEqualConfig(JRTServerConfigRequest request) { + return (getConfigMd5().equals(request.getRequestConfigMd5())); + } + + default boolean hasNewerGeneration(JRTServerConfigRequest request) { + return (getGeneration() > request.getRequestGeneration()); + } + + CompressionInfo getCompressionInfo(); + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java new file mode 100644 index 00000000000..d22628b7b4a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.slime.*; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** +* @author lulf +* @since 5.3 +*/ +public class DefContent { + private final List<String> data; + + private DefContent(List<String> data) { + this.data = data; + } + + public String[] asStringArray() { + return data.toArray(new String[data.size()]); + } + + public List<String> asList() { + return data; + } + + public String asString() { + return com.yahoo.text.StringUtilities.implode(asStringArray(), "\n"); + } + + static DefContent fromSlime(Inspector data) { + final List<String> lst = new ArrayList<>(); + data.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + lst.add(inspector.asString()); + } + }); + return new DefContent(lst); + } + + public static DefContent fromClass(Class<? extends ConfigInstance> clazz) { + return fromArray(defSchema(clazz)); + } + + public static DefContent fromList(List<String> def) { + return new DefContent(def); + } + + public static DefContent fromArray(String[] schema) { + return fromList(Arrays.asList(schema)); + } + + /** + * The def file payload of the actual class of the given config + * @param configClass the class of a generated config instance + * @return a String array with the config definition (one line per element) + */ + private static String[] defSchema(Class<? extends ConfigInstance> configClass) { + if (configClass==null) return new String[0]; + try { + Field f = configClass.getField("CONFIG_DEF_SCHEMA"); + return (String[]) f.get(configClass); + } catch (NoSuchFieldException e) { + return new String[0]; + } catch (Exception e) { + throw new ConfigurationRuntimeException(e); + } + } + + public void serialize(final Cursor cursor) { + for (String line : data) { + cursor.addString(line); + } + } + + public boolean isEmpty() { + return data.isEmpty(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java new file mode 100644 index 00000000000..8997173a479 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * Interface for config requests used by clients. + * + * @author lulf + * @since 5.3 + */ +public interface JRTClientConfigRequest extends JRTConfigRequest { + /** + * Validate config response given by the server. If none is given, or an error occurred, this should return false. + * @return true if valid response, false if not. + */ + boolean validateResponse(); + + /** + * Test whether ot not the returned config has an updated generation. This should return false if no response have + * been given. + * @return true if generation is updated, false if not. + */ + boolean hasUpdatedGeneration(); + + /** + * Return the payload in the response given by the server. The payload will be empty if no response was given. + * @return the config payload. + */ + Payload getNewPayload(); + + /** + * Create a new {@link JRTClientConfigRequest} based on this request based on the same request parameters, but having + * the timeout changed. + * @param timeout server timeout of the new request. + * @return a new {@link JRTClientConfigRequest} instance. + */ + JRTClientConfigRequest nextRequest(long timeout); + + /** + * Test whether or not the returned request is an error. + * @return true if error, false if not. + */ + boolean isError(); + + /** + * Get the generation of the newly provided config. If none has been given, 0 should be returned. + * @return the new generation. + */ + long getNewGeneration(); + + /** + * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned. + * @return a config md5. + */ + String getNewConfigMd5(); + + /** + * For protocols that perform an optimization when no new config has been given, this method will provide the + * payload and hasUpdatedConfig state of the previous request. + * @param payload a config payload of the previous request. + * @param hasUpdatedConfig the hasUpdatedConfig flag of the previous request. + */ + void updateRequestPayload(Payload payload, boolean hasUpdatedConfig); + + /** + * Test whether or not the payload is contained in this response or not. Should return false for error responses as well. + * @return true if empty, false if not. + */ + boolean containsPayload(); + + /** + * Test whether or not the response contains an updated config or not. False if no response has been returned. + * @return true if config is updated, false if not. + */ + boolean hasUpdatedConfig(); + + /** + * Get the {@link Trace} given in the response by the server. The {@link Trace} can be used to add further tracing + * and later printed to provide useful debug info. + * @return a {@link Trace}. + */ + Trace getResponseTrace(); + + /** + * Get config definition content. + * @return def as lines. + */ + DefContent getDefContent(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java new file mode 100644 index 00000000000..3f97f066829 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.jrt.Request; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.JRTMethods; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Optional; + +/** + * Represents version 3 config request for config clients. Provides methods for inspecting request and response + * values. + * + * See {@link JRTServerConfigRequestV3} for protocol details. + * + * @author lulf + * @since 5.19 + */ +public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { + + protected JRTClientConfigRequestV3(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + super(key, hostname, defSchema, configMd5, generation, timeout, trace, compressionType, vespaVersion); + } + + @Override + protected String getJRTMethodName() { + return JRTMethods.configV3getConfigMethodName; + } + + @Override + protected boolean checkReturnTypes(Request request) { + return JRTMethods.checkV3ReturnTypes(request); + } + + @Override + public Payload getNewPayload() { + CompressionInfo compressionInfo = getResponseData().getCompressionInfo(); + Utf8Array payload = new Utf8Array(request.returnValues().get(1).asData()); + return Payload.from(payload, compressionInfo); + } + + @Override + public long getProtocolVersion() { + return 3; + } + + @Override + public JRTClientConfigRequest nextRequest(long timeout) { + return new JRTClientConfigRequestV3(getConfigKey(), + getClientHostName(), + getDefContent(), + isError() ? getRequestConfigMd5() : newConfMd5(), + isError() ? getRequestGeneration() : newGen(), + timeout, + Trace.createNew(), + requestData.getCompressionType(), + requestData.getVespaVersion()); + } + + public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + String hostname = ConfigUtils.getCanonicalHostName(); + ConfigKey<T> key = sub.getKey(); + T i = sub.getConfig(); + return createWithParams(key, + sub.getDefContent(), + hostname, + i != null ? i.getConfigMd5() : "", + sub.getGeneration() != null ? sub.getGeneration() : 0L, + sub.timingValues().getSubscribeTimeout(), + trace, + compressionType, + vespaVersion); + } + + + public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + String hostname = ConfigUtils.getCanonicalHostName(); + return createWithParams(config.getKey(), + DefContent.fromList(config.getDefContent()), + hostname, + config.getConfigMd5(), + config.getGeneration(), + serverTimeout, + trace, + compressionType, + vespaVersion); + } + + + public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey, + DefContent defContent, + String hostname, + String configMd5, + long generation, + long serverTimeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + return new JRTClientConfigRequestV3(reqKey, + hostname, + defContent, + configMd5, + generation, + serverTimeout, + trace, + compressionType, + vespaVersion); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java new file mode 100644 index 00000000000..98a4ddbf7f1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Optional; + +/** + * Common interface for jrt config requests available both at server and client. + * @author lulf + * @since 5.3 + */ +public interface JRTConfigRequest { + /** + * Get the config key of the config request. + * @return a {@link ConfigKey}. + */ + ConfigKey<?> getConfigKey(); + + /** + * Perform request parameter validation of this config request. This method should be called before fetching + * any kind of config protocol-specific parameter. + * @return true if valid, false if not. + */ + boolean validateParameters(); + + /** + * Get the config md5 of the config request. Return an empty string if no response has been returned. + * @return a config md5. + */ + String getRequestConfigMd5(); + + /** + * Get the generation of the requested config. If none has been given, 0 should be returned. + * @return the generation in the request. + */ + long getRequestGeneration(); + + /** + * Get the JRT request object for this config request. + * TODO: This method leaks the internal jrt stuff :( + * @return a {@link Request} object. + */ + Request getRequest(); + + /** + * Get a short hand description of this request. + * @return a short description + */ + String getShortDescription(); + + /** + * Get the error code of this request + * @return the error code as defined in {@link com.yahoo.vespa.config.ErrorCode}. + */ + int errorCode(); + + /** + * Return the error message of this request, mostly corresponding to the {@link com.yahoo.vespa.config.ErrorCode}. + * @return the error message. + */ + String errorMessage(); + + /** + * Get the server timeout of this request. + * @return the timeout given to the server + */ + long getTimeout(); + + /** + * Get the config protocol version + * @return a protocol version number. + */ + long getProtocolVersion(); + + /** + * Get the wanted generation for this request. + * @return a generation that client would like. + */ + long getWantedGeneration(); + + /** + * Get the host name of the client that is requesting config. + * @return hostname of the client. + */ + String getClientHostName(); + + /** + * Get the Vespa version of the client that initiated the request + * @return Vespa version of the client + */ + Optional<VespaVersion> getVespaVersion(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java new file mode 100644 index 00000000000..583e4ba39f5 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.*; +import java.util.logging.Logger; + +/** + * To hide JRT implementations. + * + * @author lulf + * @since 5.3 + */ +public class JRTConfigRequestFactory { + + public static final String VESPA_CONFIG_PROTOCOL_VERSION = "VESPA_CONFIG_PROTOCOL_VERSION"; + private final static Logger log = Logger.getLogger(JRTConfigRequestFactory.class.getName()); + private static final CompressionType compressionType = getCompressionType(); + private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION = "VESPA_CONFIG_PROTOCOL_COMPRESSION"; + public static final String VESPA_VERSION = "VESPA_VERSION"; + + public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub) { + // TODO: Get trace from caller + return JRTClientConfigRequestV3.createFromSub(sub, Trace.createNew(), compressionType, getVespaVersion()); + } + + public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout) { + // TODO: Get trace from caller + return JRTClientConfigRequestV3.createFromRaw(config, serverTimeout, Trace.createNew(), compressionType, getVespaVersion()); + } + + public static String getProtocolVersion() { + return "3"; + } + + static String getProtocolVersion(String env, String yinstEnv, String property) { + return ConfigUtils.getEnvValue("3", env, yinstEnv, property); + } + + public static Set<Long> supportedProtocolVersions() { + return Collections.singleton(3l); + } + + public static CompressionType getCompressionType() { + return getCompressionType(System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION), + System.getenv("services__config_protocol_compression"), + System.getProperty(VESPA_CONFIG_PROTOCOL_COMPRESSION)); + } + + static CompressionType getCompressionType(String env, String yinstEnv, String property) { + return CompressionType.valueOf(ConfigUtils.getEnvValue("LZ4", env, yinstEnv, property)); + } + + static Optional<VespaVersion> getVespaVersion() { + final String envValue = ConfigUtils.getEnvValue("", System.getenv(VESPA_VERSION), System.getProperty(VESPA_VERSION)); + if (envValue != null && !envValue.isEmpty()) { + return Optional.of(VespaVersion.fromString(envValue)); + } + return Optional.of(getCompiledVespaVersion()); + } + + static VespaVersion getCompiledVespaVersion() { + return VespaVersion.fromString(String.format("%d.%d.%d", + com.yahoo.vespa.config.VespaVersion.major, + com.yahoo.vespa.config.VespaVersion.minor, + com.yahoo.vespa.config.VespaVersion.micro)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java new file mode 100644 index 00000000000..e201f1f08ef --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.vespa.config.GetConfigRequest; + +/** + * Interface for config requests at the server end point. + * + * @author lulf + * @since 5.3 + */ +public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigRequest { + /** + * Notify this request that its delayed due to no new config being available at this point. The value + * provided in this function should be returned when calling {@link #isDelayedResponse()}. + * @param delayedResponse true if response is delayed, false if not. + */ + void setDelayedResponse(boolean delayedResponse); + + /** + * Signal error when handling this request. The error should be reflected in the request state and propagated + * back to the client. + * @param errorCode error code, as described in {@link com.yahoo.vespa.config.ErrorCode}. + * @param message message to display for this error, typically printed by client. + */ + void addErrorResponse(int errorCode, String message); + + /** + * Signal that the request was handled and provide return values typically needed by a client. + * @param payload The config payload that the client should receive. + * @param generation The config generation of the given payload. + * @param configMd5 The md5sum of the given payload. + */ + void addOkResponse(Payload payload, long generation, String configMd5); + + /** + * Get the current config md5 of the client config. + * @return a config md5. + */ + String getRequestConfigMd5(); + + /** + * Get the current config generation of the client config. + * @return the current config generation. + */ + long getRequestGeneration(); + + /** + * Check whether or not this request is delayed. + * @return true if delayed, false if not. + */ + boolean isDelayedResponse(); + + /** + * Get the request trace for this request. The trace can be used to trace config execution to provide useful + * debug info in production environments. + * @return a {@link Trace} instance. + */ + Trace getRequestTrace(); + + /** + * Extract the appropriate payload for this request type for a given config response. + * + * @param response {@link ConfigResponse} to get payload from. + * @return A {@link Payload} that satisfies this request format. + */ + Payload payloadFromResponse(ConfigResponse response); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java new file mode 100644 index 00000000000..54dcd649d69 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Request; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields JRT + * + * * A metadata field containing json data describing config generation, md5 and compression info + * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload + * has not changed since last request, triggering an optimization at the client where the previous payload is used instead. + * + * The implementation of addOkResponse is optimized for doing as little copying of payload data as possible, ensuring + * that we get a lower memory footprint. + * + * @author lulf + * @since 5.19 + */ +public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { + + protected JRTServerConfigRequestV3(Request request) { + super(request); + } + + @Override + public void addOkResponse(Payload payload, long generation, String configMd5) { + boolean changedConfig = !configMd5.equals(getRequestConfigMd5()); + boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); + Payload responsePayload = payload.withCompression(getCompressionType()); + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096); + try { + JsonGenerator jsonGenerator = createJsonGenerator(byteArrayOutputStream); + jsonGenerator.writeStartObject(); + addCommonReturnValues(jsonGenerator); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation); + jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO); + if (responsePayload == null) { + throw new RuntimeException("Payload is null for ' " + this + ", not able to create response"); + } + CompressionInfo compressionInfo = responsePayload.getCompressionInfo(); + // If payload is not being sent, we must adjust compression info to avoid client confusion. + if (!changedConfigAndNewGeneration) { + compressionInfo = CompressionInfo.create(compressionInfo.getCompressionType(), 0); + } + compressionInfo.serialize(jsonGenerator); + jsonGenerator.writeEndObject(); + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, getConfigKey() + ": response dataXXXXX" + payload.withCompression(CompressionType.UNCOMPRESSED) + "XXXXX"); + } + jsonGenerator.writeEndObject(); + jsonGenerator.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add OK response for " + this); + } + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + if (changedConfigAndNewGeneration) { + request.returnValues().add(new DataValue(responsePayload.getData().getBytes())); + } else { + request.returnValues().add(new DataValue(new byte[0])); + } + } + + @Override + public long getProtocolVersion() { + return 3; + } + + public static JRTServerConfigRequestV3 createFromRequest(Request req) { + return new JRTServerConfigRequestV3(req); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java new file mode 100644 index 00000000000..d73d0e9a14c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import java.io.ByteArrayOutputStream; + +/** + * Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method. + * + * @author lulf + * @since 5.19 + */ +class NoCopyByteArrayOutputStream extends ByteArrayOutputStream { + public NoCopyByteArrayOutputStream() { + super(); + } + + public NoCopyByteArrayOutputStream(int initialSize) { + super(initialSize); + } + + @Override + public byte[] toByteArray() { + return buf; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java new file mode 100644 index 00000000000..aa0bf374363 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.LZ4PayloadCompressor; + +import java.util.Objects; + +/** + * An immutable config payload + * + * @author musum + * @author bratseth + */ +public class Payload { + + private final Utf8Array data; + private final CompressionInfo compressionInfo; + private final static LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); + + private Payload(ConfigPayload payload) { + this.data = payload.toUtf8Array(true); + this.compressionInfo = CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength()); + } + + private Payload(Utf8Array payload, CompressionInfo compressionInfo) { + Objects.requireNonNull(payload, "Payload"); + Objects.requireNonNull(compressionInfo, "CompressionInfo"); + this.data = payload; + this.compressionInfo = compressionInfo; + } + + public static Payload from(ConfigPayload payload) { + return new Payload(payload); + } + + /** Creates an uncompressed payload from a string */ + public static Payload from(String payload) { + return new Payload(new Utf8String(payload), CompressionInfo.uncompressed()); + } + + public static Payload from(String payload, CompressionInfo compressionInfo) { + return new Payload(new Utf8String(payload), compressionInfo); + } + + /** Creates an uncompressed payload from an Utf8Array */ + public static Payload from(Utf8Array payload) { + return new Payload(payload, CompressionInfo.uncompressed()); + } + + public static Payload from(Utf8Array payload, CompressionInfo compressionInfo) { + return new Payload(payload, compressionInfo); + } + + public Utf8Array getData() { return data; } + + /** Returns a copy of this payload where the data is compressed using the given compression */ + public Payload withCompression(CompressionType requestedCompression) { + CompressionType responseCompression = compressionInfo.getCompressionType(); + if (requestedCompression == CompressionType.UNCOMPRESSED && responseCompression == CompressionType.LZ4) { + byte[] buffer = new byte[compressionInfo.getUncompressedSize()]; + compressor.decompress(data.getBytes(), buffer); + Utf8Array data = new Utf8Array(buffer); + CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, compressionInfo.getUncompressedSize()); + return Payload.from(data, info); + } else if (requestedCompression == CompressionType.LZ4 && responseCompression == CompressionType.UNCOMPRESSED) { + Utf8Array data = new Utf8Array(compressor.compress(this.data.getBytes())); + CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, this.data.getByteLength()); + return Payload.from(data, info); + } else { + return Payload.from(data, compressionInfo); + } + } + + public CompressionInfo getCompressionInfo() { return compressionInfo; } + + @Override + public String toString() { + if (compressionInfo.getCompressionType() == CompressionType.UNCOMPRESSED) + return data.toString(); + else + return withCompression(CompressionType.UNCOMPRESSED).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Payload other = (Payload) o; + return this.compressionInfo.equals(other.compressionInfo) && this.data.equals(other.data); + } + + @Override + public int hashCode() { + return data.hashCode() + 31 * compressionInfo.hashCode(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java new file mode 100644 index 00000000000..d44f9190b42 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; + +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Static utility methods for verifying common request properties. + * + * @author lulf + * @since 5.3 + */ +public class RequestValidation { + private static final Logger log = Logger.getLogger(RequestValidation.class.getName()); + + private static final Pattern md5Pattern = Pattern.compile("[0-9a-zA-Z]+"); + + public static int validateRequest(JRTConfigRequest request) { + ConfigKey<?> key = request.getConfigKey(); + if (!RequestValidation.verifyName(key.getName())) { + log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); + return ErrorCode.ILLEGAL_NAME; + } + if (!RequestValidation.verifyNamespace(key.getNamespace())) { + log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); + return ErrorCode.ILLEGAL_NAME_SPACE; + } + if (!RequestValidation.verifyMd5(key.getMd5())) { + log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + return ErrorCode.ILLEGAL_DEF_MD5; + } + if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { + log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + return ErrorCode.ILLEGAL_CONFIG_MD5; + } + if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { + log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + return ErrorCode.ILLEGAL_GENERATION; + } + if (!RequestValidation.verifyGeneration(request.getWantedGeneration())) { + log.log(LogLevel.INFO, "Illegal wanted generation '" + request.getWantedGeneration() + "'"); + return ErrorCode.ILLEGAL_GENERATION; + } + if (!RequestValidation.verifyTimeout(request.getTimeout())) { + log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); + return ErrorCode.ILLEGAL_TIMEOUT; + } + if (!RequestValidation.verifyHostname(request.getClientHostName())) { + log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; + } + return 0; + } + + public static boolean verifyName(String name) { + Matcher m = ConfigDefinition.namePattern.matcher(name); + return m.matches(); + } + + public static boolean verifyMd5(String md5) { + if (md5.equals("")) { + return true; // Empty md5 is ok (e.g. upon getconfig from command line tools) + } else if (md5.length() != 32) { + return false; + } + Matcher m = md5Pattern.matcher(md5); + return m.matches(); + } + + public static boolean verifyTimeout(Long timeout) { + return (timeout > 0); + } + + public static boolean verifyGeneration(Long generation) { + return (generation >= 0); + } + + private static boolean verifyNamespace(String namespace) { + Matcher m = ConfigDefinition.namespacePattern.matcher(namespace); + return m.matches(); + } + + private static boolean verifyHostname(String clientHostName) { + return !("".equals(clientHostName)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java new file mode 100644 index 00000000000..2cb61cac427 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java @@ -0,0 +1,230 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.*; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of + * payload encoding and decoding, as well as adding extra request/response fields. + * + * @author lulf + * @since 5.18 + */ +public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest { + + protected static final Logger log = Logger.getLogger(SlimeClientConfigRequest.class.getName()); + + protected final SlimeRequestData requestData; + private final SlimeResponseData responseData; + + protected final Request request; + + protected SlimeClientConfigRequest(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + Slime data = SlimeRequestData.encodeRequest(key, + hostname, + defSchema, + configMd5, + generation, + timeout, + trace, + getProtocolVersion(), + compressionType, + vespaVersion); + Request jrtReq = new Request(getJRTMethodName()); + jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true))); + + this.requestData = new SlimeRequestData(jrtReq, data); + this.responseData = new SlimeResponseData(jrtReq); + this.request = jrtReq; + } + + protected abstract String getJRTMethodName(); + + protected static String encodeAsUtf8String(Slime data, boolean compact) { + ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + try { + new JsonFormat(compact).encode(baos, data); + } catch (IOException e) { + throw new RuntimeException("Unable to encode config request", e); + } + return Utf8.toString(baos.toByteArray()); + } + + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()) + .append(",").append(getVespaVersion()).append("'\n"); + sb.append("response='").append(getNewConfigMd5()) + .append(",").append(getNewGeneration()) + .append("'\n"); + return sb.toString(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + @Override + public long getWantedGeneration() { + return requestData.getWantedGeneration(); + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + @Override + public boolean hasUpdatedGeneration() { + long prevGen = getRequestGeneration(); + long newGen = getNewGeneration(); + return ConfigUtils.isGenerationNewer(newGen, prevGen); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + protected String newConfMd5() { + String newMd5 = getNewConfigMd5(); + if ("".equals(newMd5)) return getRequestConfigMd5(); + return newMd5; + } + + protected long newGen() { + long newGen = getNewGeneration(); + if (newGen==0) return getRequestGeneration(); + return newGen; + } + + @Override + public DefContent getDefContent() { + return requestData.getSchema(); + } + + @Override + public boolean isError() { + return request.isError(); + } + + @Override + public void updateRequestPayload(Payload payload, boolean hasUpdatedConfig) { + // This protocol sends payload in all cases, so ignore this. + } + + @Override + public boolean containsPayload() { + return false; + } + + @Override + public boolean hasUpdatedConfig() { + String respMd5 = getNewConfigMd5(); + return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); + } + + @Override + public Trace getResponseTrace() { + return responseData.getResponseTrace(); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + @Override + public boolean validateResponse() { + if (request.isError()) { + return false; + } else if (request.returnValues().size() == 0) { + return false; + } else if (!checkReturnTypes(request)) { + log.warning("Invalid return types for config response: " + errorMessage()); + return false; + } + if (hasUpdatedConfig() && ! hasUpdatedGeneration()) { + request.setError(ErrorCode.OUTDATED_CONFIG, "Config payload has changed (old config md5:" + + getRequestConfigMd5() + ", new config md5: " + getNewConfigMd5() +"), but new generation " + + getNewGeneration() + " is not newer than current generation " + getRequestGeneration() + "."); + return false; + } + return true; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + return (errorCode == 0); + } + + protected abstract boolean checkReturnTypes(Request request); + + @Override + public String getNewConfigMd5() { + return responseData.getResponseConfigMd5(); + } + + @Override + public long getNewGeneration() { + return responseData.getResponseConfigGeneration(); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + protected SlimeResponseData getResponseData() { + return responseData; + } + + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java new file mode 100644 index 00000000000..d5c4fc638ee --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigFileFormat; +import com.yahoo.vespa.config.ConfigPayload; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; + +/** + * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. + * + * @author lulf + * @since 5.1 + */ +public class SlimeConfigResponse implements ConfigResponse { + + private final Utf8Array payload; + private final CompressionInfo compressionInfo; + private final InnerCNode targetDef; + private final long generation; + private final String configMd5; + + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, String configMd5) { + Utf8Array data = payload.toUtf8Array(true); + return new SlimeConfigResponse(data, targetDef, generation, configMd5, CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); + } + + public SlimeConfigResponse(Utf8Array payload, InnerCNode targetDef, long generation, String configMd5, CompressionInfo compressionInfo) { + this.payload = payload; + this.targetDef = targetDef; + this.generation = generation; + this.configMd5 = configMd5; + this.compressionInfo = compressionInfo; + } + + @Override + public Utf8Array getPayload() { + return payload; + } + + @Override + public List<String> getLegacyPayload() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ConfigFileFormat format = new ConfigFileFormat(targetDef); + Payload v1payload = Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); + try { + ConfigPayload.fromUtf8Array(v1payload.getData()).serialize(baos, format); + return Arrays.asList(baos.toString("UTF-8").split("\\n")); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long getGeneration() { + return generation; + } + + @Override + public String getConfigMd5() { + return configMd5; + } + + @Override + public void serialize(OutputStream os, CompressionType type) throws IOException { + os.write(Payload.from(payload, compressionInfo).withCompression(type).getData().getBytes()); + } + + @Override + public String toString() { + return "generation=" + generation + "\n" + + "configmd5=" + configMd5 + "\n" + + Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); + } + + @Override + public CompressionInfo getCompressionInfo() { return compressionInfo; } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java new file mode 100644 index 00000000000..80c050b9b61 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Optional; + +/** + * Contains slime request data objects. Provides methods for reading various fields from slime request data. All + * data is read lazily. + * +* @author lulf +* @since 5.18 +*/ +class SlimeRequestData { + + private static final String REQUEST_VERSION = "version"; + private static final String REQUEST_DEF_NAME = "defName"; + private static final String REQUEST_DEF_NAMESPACE = "defNamespace"; + private static final String REQUEST_DEF_CONTENT = "defContent"; + private static final String REQUEST_CLIENT_CONFIGID = "configId"; + private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname"; + private static final String REQUEST_CURRENT_GENERATION = "currentGeneration"; + private static final String REQUEST_WANTED_GENERATION = "wantedGeneration"; + private static final String REQUEST_CONFIG_MD5 = "configMD5"; + private static final String REQUEST_TRACE = "trace"; + private static final String REQUEST_TIMEOUT = "timeout"; + private static final String REQUEST_DEF_MD5 = "defMD5"; + private static final String REQUEST_COMPRESSION_TYPE = "compressionType"; + private static final String REQUEST_VESPA_VERSION = "vespaVersion"; + + private final Request request; + private Slime data = null; + + SlimeRequestData(Request request) { + this.request = request; + } + + SlimeRequestData(Request request, Slime data) { + this.request = request; + this.data = data; + } + + private Slime getData() { + if (data == null) { + data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(request.parameters().get(0).asString())); + } + return data; + } + + Inspector getRequestField(String requestField) { + return getData().get().field(requestField); + } + + ConfigKey<?> getConfigKey() { + return ConfigKey.createFull(getRequestField(REQUEST_DEF_NAME).asString(), + getRequestField(REQUEST_CLIENT_CONFIGID).asString(), + getRequestField(REQUEST_DEF_NAMESPACE).asString(), + getRequestField(REQUEST_DEF_MD5).asString()); + } + + DefContent getSchema() { + Inspector content = getRequestField(REQUEST_DEF_CONTENT); + return DefContent.fromSlime(content); + } + + String getClientHostName() { + return getRequestField(REQUEST_CLIENT_HOSTNAME).asString(); + } + + long getWantedGeneration() { + return getRequestField(REQUEST_WANTED_GENERATION).asLong(); + } + + long getTimeout() { + return getRequestField(REQUEST_TIMEOUT).asLong(); + } + + String getRequestConfigMd5() { + return getRequestField(REQUEST_CONFIG_MD5).asString(); + } + + long getRequestGeneration() { + return getRequestField(REQUEST_CURRENT_GENERATION).asLong(); + } + + static Slime encodeRequest(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + long protocolVersion, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + Slime data = new Slime(); + Cursor request = data.setObject(); + request.setLong(REQUEST_VERSION, protocolVersion); + request.setString(REQUEST_DEF_NAME, key.getName()); + request.setString(REQUEST_DEF_NAMESPACE, key.getNamespace()); + request.setString(REQUEST_DEF_MD5, key.getMd5()); + request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId()); + request.setString(REQUEST_CLIENT_HOSTNAME, hostname); + defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); + request.setString(REQUEST_CONFIG_MD5, configMd5); + request.setLong(REQUEST_CURRENT_GENERATION, generation); + request.setLong(REQUEST_WANTED_GENERATION, 0l); + request.setLong(REQUEST_TIMEOUT, timeout); + request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name()); + if (vespaVersion.isPresent()) { + request.setString(REQUEST_VESPA_VERSION, vespaVersion.get().toString()); + } + trace.serialize(request.setObject(REQUEST_TRACE)); + return data; + } + + Trace getRequestTrace() { + return Trace.fromSlime(getRequestField(REQUEST_TRACE)); + } + + public CompressionType getCompressionType() { + Inspector field = getRequestField(REQUEST_COMPRESSION_TYPE); + return field.valid() ? CompressionType.parse(field.asString()) : CompressionType.UNCOMPRESSED; + } + + public Optional<VespaVersion> getVespaVersion() { + String versionString = getRequestField(REQUEST_VESPA_VERSION).asString(); // will be "" if not set, never null + return versionString.isEmpty() ? Optional.<VespaVersion>empty() : Optional.of(VespaVersion.fromString(versionString)); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java new file mode 100644 index 00000000000..8899658730c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; + +/** + * Contains response data for a slime response and methods for decoding the response data that + * are common to all {@link Slime} based config requests. + * + * @author lulf + * @since 5.18 + */ +class SlimeResponseData { + static final String RESPONSE_VERSION = "version"; + static final String RESPONSE_DEF_NAME = "defName"; + static final String RESPONSE_DEF_NAMESPACE = "defNamespace"; + static final String RESPONSE_DEF_MD5 = "defMD5"; + static final String RESPONSE_CONFIGID = "configId"; + static final String RESPONSE_CLIENT_HOSTNAME = "clientHostname"; + static final String RESPONSE_TRACE = "trace"; + static final String RESPONSE_CONFIG_MD5 = "configMD5"; + static final String RESPONSE_CONFIG_GENERATION = "generation"; + static final String RESPONSE_COMPRESSION_INFO = "compressionInfo"; + + private final Request request; + private Slime data = null; + + SlimeResponseData(Request request) { + this.request = request; + } + + private Slime getData() { + if (request.returnValues().size() > 0) { + if (data == null) { + data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(request.returnValues().get(0).asString())); + } + return data; + } else { + return new Slime(); + } + } + + Inspector getResponseField(String responseTrace) { + return getData().get().field(responseTrace); + } + + long getResponseConfigGeneration() { + Inspector inspector = getResponseField(RESPONSE_CONFIG_GENERATION); + return inspector.valid() ? inspector.asLong() : -1; + } + + Trace getResponseTrace() { + Inspector trace = getResponseField(RESPONSE_TRACE); + return trace.valid() ? Trace.fromSlime(trace) : Trace.createDummy(); + } + + String getResponseConfigMd5() { + Inspector inspector = getResponseField(RESPONSE_CONFIG_MD5); + return inspector.valid() ? inspector.asString() : ""; + } + + CompressionInfo getCompressionInfo() { + return CompressionInfo.fromSlime(getResponseField(RESPONSE_COMPRESSION_INFO)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java new file mode 100644 index 00000000000..8eabcd7eb26 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java @@ -0,0 +1,206 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.jrt.*; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ErrorCode; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of + * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3 + * config protocol. + * + * @author lulf + * @since 5.18 + */ +abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { + + protected static final Logger log = Logger.getLogger(SlimeServerConfigRequest.class.getName()); + + private static final JsonFactory jsonFactory = new JsonFactory(); + + private final SlimeRequestData requestData; + + // Response values + private boolean isDelayed = false; + private Trace requestTrace = null; + protected final Request request; + + protected SlimeServerConfigRequest(Request request) { + this.requestData = new SlimeRequestData(request); + this.request = request; + } + + protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { + return jsonFactory.createGenerator(byteArrayOutputStream); + } + + @Override + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public DefContent getDefContent() { + return getSchema(); + } + + @Override + public boolean noCache() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()).append("'\n"); + return sb.toString(); + } + + @Override + public Payload payloadFromResponse(ConfigResponse response) { + return Payload.from(response.getPayload(), response.getCompressionInfo()); + } + + private DefContent getSchema() { + return requestData.getSchema(); + } + + @Override + public long getWantedGeneration() { + return requestData.getWantedGeneration(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + public Trace getRequestTrace() { + if (requestTrace == null) { + requestTrace = requestData.getRequestTrace(); + } + return requestTrace; + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + if (errorCode != 0) { + addErrorResponse(errorCode); + } + return (errorCode == 0); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + private void addErrorResponse(int errorCode) { + addErrorResponse(errorCode, ErrorCode.getName(errorCode)); + } + + @Override + public void setDelayedResponse(boolean delayedResponse) { + this.isDelayed = delayedResponse; + } + + @Override + public void addErrorResponse(int errorCode, String name) { + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); + try { + JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); + jsonWriter.writeStartObject(); + addCommonReturnValues(jsonWriter); + jsonWriter.writeEndObject(); + jsonWriter.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add error response for " + this); + } + request.setError(errorCode, name); + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + } + + protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { + return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); + } + + protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { + ConfigKey<?> key = requestData.getConfigKey(); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); + jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); + jsonGenerator.writeRawValue(getRequestTrace().toString(true)); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { + jsonGenerator.writeStringField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { + jsonGenerator.writeNumberField(fieldName, value); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + @Override + public boolean isDelayedResponse() { + return isDelayed; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + protected CompressionType getCompressionType() { + return requestData.getCompressionType(); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java new file mode 100644 index 00000000000..a4074a43a81 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.yolean.trace.TraceNode; + +/** + * Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode} + * + * @author lulf + * @since 5.5 + */ +public class SlimeTraceDeserializer { + private final Inspector entry; + public SlimeTraceDeserializer(Inspector inspector) { + this.entry = inspector; + } + + public TraceNode deserialize() { + return deserialize(entry); + } + + private static TraceNode deserialize(Inspector entry) { + Object payload = decodePayload(entry.field(SlimeTraceSerializer.PAYLOAD)); + long timestamp = decodeTimestamp(entry.field(SlimeTraceSerializer.TIMESTAMP)); + final TraceNode node = new TraceNode(payload, timestamp); + Inspector children = entry.field(SlimeTraceSerializer.CHILDREN); + children.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + node.add(deserialize(inspector)); + } + }); + return node; + } + + private static long decodeTimestamp(Inspector entry) { + return entry.asLong(); + } + + private static Object decodePayload(Inspector entry) { + switch (entry.type()) { + case STRING: + return entry.asString(); + case LONG: + return entry.asLong(); + case BOOL: + return entry.asBool(); + case DOUBLE: + return entry.asDouble(); + case DATA: + return entry.asData(); + default: + return null; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java new file mode 100644 index 00000000000..ea4151e9e36 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.Cursor; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import java.util.Iterator; +import java.util.Stack; + +/** + * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. + * + * @author lulf + * @since 5.5 + */ +public class SlimeTraceSerializer extends TraceVisitor { + static final String TIMESTAMP = "timestamp"; + static final String PAYLOAD = "payload"; + static final String CHILDREN = "children"; + final Stack<Cursor> cursors = new Stack<>(); + + public SlimeTraceSerializer(Cursor cursor) { + cursors.push(cursor); + } + + @Override + public void visit(TraceNode node) { + Cursor current = cursors.pop(); + current.setLong(TIMESTAMP, node.timestamp()); + encodePayload(current, node.payload()); + addChildrenCursors(current, node); + + } + + private void encodePayload(Cursor current, Object payload) { + if (payload instanceof String) { + current.setString(PAYLOAD, (String)payload); + } else if (payload instanceof Long) { + current.setLong(PAYLOAD, (Long) payload); + } else if (payload instanceof Boolean) { + current.setBool(PAYLOAD, (Boolean) payload); + } else if (payload instanceof Double) { + current.setDouble(PAYLOAD, (Double) payload); + } else if (payload instanceof byte[]) { + current.setData(PAYLOAD, (byte[]) payload); + } + } + + private void addChildrenCursors(Cursor current, TraceNode node) { + Iterator<TraceNode> it = node.children().iterator(); + if (it.hasNext()) { + Cursor childrenArray = current.setArray(CHILDREN); + while (it.hasNext()) { + cursors.push(childrenArray.addObject()); + it.next(); + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java new file mode 100644 index 00000000000..58ec5024cbb --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.trace.TraceNode; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Clock; + +/** + * A trace utility that can serialize/deserialize to/from {@link Slime} + * + * @author lulf + * @since 5.3 + */ +public class Trace { + private static final String TRACE_TRACELOG = "traceLog"; + private static final String TRACE_TRACELEVEL = "traceLevel"; + private final int traceLevel; + private final TraceNode traceNode; + private final Clock clock; + + private Trace(int traceLevel, TraceNode traceNode, Clock clock) { + this.traceLevel = traceLevel; + this.traceNode = traceNode; + this.clock = clock; + } + + + public void trace(int level, String message) { + if (shouldTrace(level)) { + addTrace(message); + } + } + + private void addTrace(String message) { + traceNode.add(new TraceNode(message, clock.millis())); + } + + public static Trace createNew(int traceLevel, Clock clock) { + return new Trace(traceLevel, new TraceNode(null, clock.millis()), clock); + } + + public static Trace createNew(int traceLevel) { + return createNew(traceLevel, Clock.systemUTC()); + } + + public static Trace fromSlime(Inspector inspector) { + int traceLevel = deserializeTraceLevel(inspector); + Clock clock = Clock.systemUTC(); + SlimeTraceDeserializer deserializer = new SlimeTraceDeserializer(inspector.field(TRACE_TRACELOG)); + return new Trace(traceLevel, deserializer.deserialize(), clock); + } + + private static int deserializeTraceLevel(Inspector inspector) { + return (int) inspector.field(TRACE_TRACELEVEL).asLong(); + } + + public void serialize(Cursor cursor) { + cursor.setLong(TRACE_TRACELEVEL, traceLevel); + SlimeTraceSerializer serializer = new SlimeTraceSerializer(cursor.setObject(TRACE_TRACELOG)); + traceNode.accept(serializer); + } + + public static Trace createDummy() { + return Trace.createNew(0); + } + + public int getTraceLevel() { + return traceLevel; + } + + public boolean shouldTrace(int level) { + return level <= traceLevel; + } + + public String toString(boolean compact) { + Slime slime = new Slime(); + serialize(slime.setObject()); + JsonFormat format = new JsonFormat(compact); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + format.encode(baos, slime); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to encode trace as JSON", e); + } + return Utf8.toString(baos.toByteArray()); + } + + + @Override + public String toString() { + return toString(false); + } + + private final static int systemTraceLevel = Integer.getInteger("config.protocol.traceLevel", 0); + public static Trace createNew() { + return createNew(systemTraceLevel); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java new file mode 100644 index 00000000000..b715792f05e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.SerializableString; +import com.yahoo.text.Utf8Array; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * Wraps utf8array as a {@link com.fasterxml.jackson.core.SerializableString} to avoid extra copy. + * + * @author lulf + * @since 5.17 + */ +public class Utf8SerializedString implements SerializableString { + private final Utf8Array value; + public Utf8SerializedString(Utf8Array value) { + this.value = value; + } + + @Override + public String getValue() { + return value.toString(); + } + + @Override + public int charLength() { + return value.getByteLength(); + } + + @Override + public char[] asQuotedChars() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asUnquotedUTF8() { + return value.getBytes(); + } + + @Override + public byte[] asQuotedUTF8() { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuotedUTF8(byte[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendUnquotedUTF8(byte[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendUnquoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int writeQuotedUTF8(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int writeUnquotedUTF8(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int putQuotedUTF8(ByteBuffer buffer) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int putUnquotedUTF8(ByteBuffer out) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java b/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java new file mode 100644 index 00000000000..748cdce4e25 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * A wrapper class for Vespa version + * + * @author musum + * @since 5.39 + */ +public class VespaVersion { + private final String version; + + public static VespaVersion fromString(String version) { + return new VespaVersion(version); + } + + private VespaVersion(String version) { + this.version = version; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + VespaVersion that = (VespaVersion) o; + + if (!version.equals(that.version)) return false; + + return true; + } + + @Override + public int hashCode() { + return version.hashCode(); + } + + @Override + public String toString() { + return version; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java b/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java new file mode 100644 index 00000000000..5a6398eda96 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java new file mode 100644 index 00000000000..b79ff278e57 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -0,0 +1,454 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.util; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.codegen.CNode; +import com.yahoo.io.HexDump; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.JsonFormat; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.*; + +import java.io.*; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.DecimalFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utilities for mangling config text, finding md5sums, version numbers in .def files etc. + */ +public class ConfigUtils { + /* Patterns used for finding ranges in config definitions */ + private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); + private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); + private static final Pattern spaceBeforeCommaPatter = Pattern.compile("\\s,"); + public static final String intFormattedMax = new DecimalFormat("#.#").format(0x7fffffff); + public static final String intFormattedMin = new DecimalFormat("#.#").format(-0x80000000); + public static final String doubleFormattedMax = new DecimalFormat("#.#").format(1e308); + public static final String doubleFormattedMin = new DecimalFormat("#.#").format(-1e308); + + /** + * Computes Md5 hash of a list of strings. The only change to input lines before + * computing md5 is to skip empty lines. + * + * @param payload a config payload + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getMd5(ConfigPayload payload) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + payload.serialize(baos, new JsonFormat(true)); + } catch (IOException e) { + throw new RuntimeException(e); + } + MessageDigest md5 = getMd5Instance(); + md5.update(baos.toByteArray()); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Computes Md5 hash of a list of strings. The only change to input lines before + * computing md5 is to skip empty lines. + * + * @param lines A list of lines + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getMd5(List<String> lines) { + StringBuilder sb = new StringBuilder(); + for (String line : lines) { + // Remove empty lines + line = line.trim(); + if (line.length() > 0) { + sb.append(line).append("\n"); + } + } + MessageDigest md5 = getMd5Instance(); + md5.update(Utf8.toBytes(sb.toString())); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Computes Md5 hash of a string. + * + * @param input the input String + * @return the Md5 hash of the input, with lowercase letters + */ + public static String getMd5(String input) { + MessageDigest md5 = getMd5Instance(); + md5.update(IOUtils.utf8ByteBuffer(input)); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + public static String getMd5(Utf8Array input) { + MessageDigest md5 = getMd5Instance(); + md5.update(input.getBytes()); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + private static MessageDigest getMd5Instance() { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + return null; + } + } + + /** + * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * @param str String to strip spaces from + * @return String with spaces stripped + */ + public static String stripSpaces(String str) { + StringBuilder ret = new StringBuilder(""); + boolean inQuotes = false; + boolean inSpaceSequence = false; + for (char c : str.toCharArray()) { + if (Character.isWhitespace(c)) { + if (inQuotes) { + ret.append(c); + continue; + } + if (!inSpaceSequence) { + // start of space sequence + inSpaceSequence=true; + ret.append(" "); + } + } else { + if (inSpaceSequence) { + inSpaceSequence=false; + } + if (c=='\"') { + inQuotes=!inQuotes; + } + ret.append(c); + } + } + return ret.toString(); + } + + /** + * Computes Md5 hash of a list of strings with the contents of a def-file. + * + * Each string is normalized according to the + * rules of Vespa config definition files before they are used: + * <ol> + * <li>Remove trailing space.<li> + * <li>Remove comment lines.</li> + * <li>Remove trailing comments, and spaces before trailing comments.</li> + * <li>Remove empty lines</li> + * <li>Remove 'version=<version-number>'</li> + * </ol> + * + * @param lines A list of lines constituting a def-file + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getDefMd5(List<String> lines) { + List<String> linesCopy = new ArrayList<>(lines); + for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) { + String line = it.next().trim(); + if (! line.startsWith("#") && ! line.equals("")) { + if (line.startsWith("version")) { + it.remove(); + } + // Quit upon 'version', or first line with real content since 'version' cannot occur after that + break; + } + } + + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (String line : linesCopy) { + // Normalize line, like it's done in make-config-preproc.pl + line = line.trim(); + // The perl script does stuff like this: + Matcher m = intPattern.matcher(line); + if (m.matches()) { + line = line.replaceFirst("\\[,", "[" + intFormattedMin + ","); + line = line.replaceFirst(",\\]", "," + intFormattedMax + "]"); + } + m = doublePattern.matcher(line); + if (m.matches()) { + line = line.replaceFirst("\\[,", "[" + doubleFormattedMin + ","); + line = line.replaceFirst(",\\]", "," + doubleFormattedMax + "]"); + } + if (line.contains("#")) { + line = line.substring(0, line.indexOf("#")); + line = line.trim(); // Remove space between "real" end of line and a trailing comment + } + if (line.length() > 0) { + line = stripSpaces(line); + m = spaceBeforeCommaPatter.matcher(line); + line = m.replaceAll(","); // Remove space before comma (for enums) + sb.append(line).append("\n"); + } + } + md5.update(Utf8.toBytes(sb.toString())); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Finds the def version from a reader for a def-file. Returns "" (empty string) + * if no version was found. + * + * @param in A reader to a def-file + * @return version of the def-file, or "" (empty string) if no version was found + */ + public static String getDefVersion(Reader in) { + return getDefKeyword(in, "version"); + } + + /** + * Finds the def namespace from a reader for a def-file. Returns "" (empty string) + * if no namespace was found. + * + * @param in A reader to a def-file + * @return namespace of the def-file, or "" (empty string) if no namespace was found + */ + public static String getDefNamespace(Reader in) { + return getDefKeyword(in, "namespace"); + } + + /** + * Finds the value of the keyword in <code>keyword</code> from a reader for a def-file. + * Returns "" (empty string) if no value for keyword was found. + * + * @param in A reader to a def-file + * @return value of keyword, or "" (empty string) if no line matching keyword was found + */ + public static String getDefKeyword(Reader in, String keyword) { + if (null == in) { + throw new IllegalArgumentException("Null reader."); + } + LineNumberReader reader; + try { + if (in instanceof LineNumberReader) { + reader = (LineNumberReader) in; + } else { + reader = new LineNumberReader(in); + } + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (!line.startsWith("#") && !line.equals("")) { + if (line.startsWith(keyword)) { + String[] v = line.split("="); + return v[1].trim(); + } + } + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException("IOException", e); + } + return ""; + } + + /** + * Finds the name and version from a string with "name,version". + * If no name is given, the first part of the tuple will be the empty string + * If no version is given, the second part of the tuple will be the empty string + * + * @param nameCommaVersion A string consisting of "name,version" + * @return a Tuple2 with first item being name and second item being version + */ + public static Tuple2<String, String> getNameAndVersionFromString(String nameCommaVersion) { + String[] av = nameCommaVersion.split(","); + return new Tuple2<>(av[0], av.length >= 2 ? av[1] : ""); + } + + /** + * Finds the name and namespace from a string with "namespace.name". + * namespace may contain dots. + * If no namespace is given (".name" or just "name"), the second part of the tuple will be the empty string + * If no name is given, the first part of the tuple will be the empty string + * + * @param nameDotNamespace A string consisting of "namespace.name" + * @return a Tuple2 with first item being name and second item being namespace + */ + public static Tuple2<String, String> getNameAndNamespaceFromString(String nameDotNamespace) { + if (!nameDotNamespace.contains(".")) { + return new Tuple2<>(nameDotNamespace, ""); + } + String name = nameDotNamespace.substring(nameDotNamespace.lastIndexOf(".") + 1); + String namespace = nameDotNamespace.substring(0, nameDotNamespace.lastIndexOf(".")); + return new Tuple2<>(name, namespace); + } + + /** + * Creates a ConfigDefinitionKey based on a string with namespace, name and version + * (e.g. Vespa's own config definitions in $VESPA_HOME/var/db/vespa/configserver/serverdb/classes) + * + * @param input A string consisting of "namespace.name.version" + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey getConfigDefinitionKeyFromString(String input) { + final String name; + final String namespace; + if (!input.contains(".")) { + name = input; + namespace = ""; + } else if (input.lastIndexOf(".") == input.indexOf(".")) { + Tuple2<String, String> tuple = ConfigUtils.getNameAndNamespaceFromString(input); + boolean containsVersion = false; + for (int i=0; i < tuple.first.length(); i++) { + if (Character.isDigit(tuple.first.charAt(i))) { + containsVersion = true; + break; + } + } + if (containsVersion) { + name = tuple.second; + namespace = ""; + } else { + name = tuple.first; + namespace = tuple.second; + } + } else { + Tuple2<String, String> tuple = ConfigUtils.getNameAndNamespaceFromString(input); + + String tempName = tuple.second; + tuple = ConfigUtils.getNameAndNamespaceFromString(tempName); + name = tuple.first; + namespace = tuple.second; + } + return new ConfigDefinitionKey(name, namespace); + } + + /** + * Creates a ConfigDefinitionKey from a string for the name of a node in ZooKeeper + * that holds a config definition + * + * @param nodeName name of a node in ZooKeeper that holds a config definition + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey createConfigDefinitionKeyFromZKString(String nodeName) { + final String name; + final String namespace; + if (nodeName.contains(".")) { + Tuple2<String, String> tuple = ConfigUtils.getNameAndVersionFromString(nodeName); + String tempName = tuple.first; // includes namespace + tuple = ConfigUtils.getNameAndNamespaceFromString(tempName); + name = tuple.first; + namespace = tuple.second; + } else { + Tuple2<String, String> tuple = ConfigUtils.getNameAndVersionFromString(nodeName); + name = tuple.first; + namespace = ""; + } + return new ConfigDefinitionKey(name, namespace); + } + + + /** + * Creates a ConfigDefinitionKey from a file by reading the file and parsing + * contents for namespace. Name and from filename, but the filename may be prefixed + * with the namespace (if two def files has the same name for instance). + * + * @param file a config definition file + * @return a ConfigDefinitionKey + */ + public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException { + String[] fileName = file.getName().split("\\."); + assert(fileName.length >= 2); + String name = fileName[fileName.length - 2]; + byte[] content = IOUtils.readFileBytes(file); + + return createConfigDefinitionKeyFromDefContent(name, content); + } + + /** + * Creates a ConfigDefinitionKey from a name and the content of a config definition + * + * @param name the name of the config definition + * @param content content of a config definition + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey createConfigDefinitionKeyFromDefContent(String name, byte[] content) { + String namespace = ConfigUtils.getDefNamespace(new StringReader(Utf8.toString(content))); + if (namespace.isEmpty()) { + namespace = CNode.DEFAULT_NAMESPACE; + } + return new ConfigDefinitionKey(name, namespace); + } + + + /** + * Escapes a config value according to the cfg format. + * @param input the string to escape + * @return the escaped string + */ + public static String escapeConfigFormatValue(String input) { + if (input == null) { + return "null"; + } + StringBuilder outputBuf = new StringBuilder(input.length()); + for (int i = 0; i < input.length(); i++) { + if (input.charAt(i) == '\\') { + outputBuf.append("\\\\"); // backslash is escaped as: \\ + } else if (input.charAt(i) == '"') { + outputBuf.append("\\\""); // double quote is escaped as: \" + } else if (input.charAt(i) == '\n') { + outputBuf.append("\\n"); // newline is escaped as: \n + } else if (input.charAt(i) == 0) { + // XXX null byte is probably not a good idea anyway + System.err.println("WARNING: null byte in config value"); + outputBuf.append("\\x00"); + } else { + // all other characters are output as-is + outputBuf.append(input.charAt(i)); + } + } + return outputBuf.toString(); + } + + + public static String getDefMd5FromRequest(String defMd5, List<String> defContent) { + if ((defMd5 == null || defMd5.isEmpty()) && defContent != null) { + return ConfigUtils.getDefMd5(defContent); + } else { + return defMd5; + } + } + + public static String getCanonicalHostName() { + try { + return com.yahoo.net.LinuxInetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + + /** + * Loop through values and return the first one that is set and non-empty. + * + * @param defaultValue The default value to use if no environment variables are set. + * @param envVars one or more environment variable strings + * @return a String with the value of the environment variable + */ + public static String getEnvValue(String defaultValue, String ... envVars) { + String value = null; + for (String envVar : envVars) { + if (value == null || value.isEmpty()) { + value = envVar; + } + } + return (value == null || value.isEmpty()) ? defaultValue : value; + } + + public static boolean isGenerationNewer(long newGen, long oldGen) { + return (oldGen < newGen) || (newGen == 0); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/util/package-info.java b/config/src/main/java/com/yahoo/vespa/config/util/package-info.java new file mode 100644 index 00000000000..30f76c1ecbf --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/util/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.util; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore b/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore diff --git a/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore b/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore |