aboutsummaryrefslogtreecommitdiffstats
path: root/configgen/src/main/java/com/yahoo/config/codegen
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /configgen/src/main/java/com/yahoo/config/codegen
Publish
Diffstat (limited to 'configgen/src/main/java/com/yahoo/config/codegen')
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/CNode.java161
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java13
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java21
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java1117
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/DefLine.java274
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/DefParser.java216
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java68
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java112
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java267
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java129
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java91
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java200
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java8
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java72
14 files changed, 2749 insertions, 0 deletions
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CNode.java b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java
new file mode 100644
index 00000000000..02fec548b40
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java
@@ -0,0 +1,161 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.util.StringTokenizer;
+
+/**
+ * Abstract superclass for all nodes representing a definition file.
+ *
+ * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
+ */
+public abstract class CNode {
+
+ public static final String DEFAULT_NAMESPACE = "config";
+
+ // TODO: replace by "type" enum
+ public final boolean isArray;
+ public final boolean isMap;
+ final String name;
+ final InnerCNode parent;
+
+ // TODO: remove! Only set for the root node, and root.getName() returns the same thing!
+ String defName = null;
+ String defVersion = "";
+ String defNamespace = null;
+ String defMd5 = "MISSING MD5";
+ String comment = "";
+
+
+ protected CNode(InnerCNode parent, String name) {
+ this.parent = parent;
+ int bracketIdx = name.indexOf('[');
+ int curlyIdx = name.indexOf('{');
+ if (bracketIdx != -1) {
+ this.name = name.substring(0, bracketIdx);
+ isArray = true;
+ isMap = false;
+ } else if (curlyIdx != -1) {
+ this.name = name.substring(0, curlyIdx);
+ isMap = true;
+ isArray = false;
+ } else {
+ this.name = name;
+ isMap = false;
+ isArray = false;
+ }
+ }
+
+ /**
+ * Returns the simple name of this node.
+ * @return the simple name of this node
+ */
+ public String getName() {
+ return name;
+ }
+
+ public InnerCNode getParent() {
+ return parent;
+ }
+
+ public abstract CNode[] getChildren();
+
+ public abstract CNode getChild(String name);
+
+ public String getMd5() {
+ return defMd5;
+ }
+
+ void setMd5(String md5) {
+ defMd5 = md5;
+ }
+
+ public String getVersion() {
+ return defVersion;
+ }
+
+ void setVersion(String version) {
+ defVersion = version;
+ }
+
+ public String getNamespace() {
+ return defNamespace;
+ }
+
+ void setNamespace(String namespace) {
+ defNamespace = namespace;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ protected abstract void setLeaf(String name, DefLine defLine, String comment)
+ throws IllegalArgumentException;
+
+ public abstract boolean needRestart();
+
+ protected void checkMyName(String myName) throws IllegalArgumentException {
+ if (isArray) {
+ int n1 = myName.indexOf('[');
+ int n2 = myName.indexOf(']');
+ if (n1 == -1 || n2 < n1)
+ throw new IllegalArgumentException("Invalid array syntax: " + myName);
+ myName = myName.substring(0, n1);
+ } else if (isMap) {
+ int n1 = myName.indexOf('{');
+ int n2 = myName.indexOf('}');
+ if (n1 == -1 || n2 < n1)
+ throw new IllegalArgumentException("Invalid map syntax: " + myName);
+ myName = myName.substring(0, n1);
+ } else if (myName.contains("[]")) {
+ throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-array type.");
+ } else if (myName.contains("{}")) {
+ throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-map type.");
+ }
+ if (!myName.equals(getName()))
+ throw new IllegalArgumentException(myName + " does not match " + getName() + ".");
+ }
+
+ /**
+ * @return the full name as a config path of this node.
+ */
+ public String getFullName() {
+ StringBuilder buf = new StringBuilder();
+ if (parent != null)
+ buf.append(parent.getFullName());
+ if (buf.length() > 0)
+ buf.append('.');
+ StringBuilder theName = new StringBuilder(this.getName());
+
+ if (isArray) theName.append("[]");
+ else if (isMap) theName.append("{}");
+
+ return buf.append(theName).toString();
+ }
+
+ /**
+ * @param prefix The prefix to use, usually an indent (spaces) followed by either '*' or "//"
+ * @return a comment block where each line is prefixed, but the caller must close it if using '*'.
+ */
+ public String getCommentBlock(String prefix) {
+ prefix = prefix + " ";
+ StringBuilder ret = new StringBuilder();
+ if (getComment().length() > 0) {
+ StringTokenizer st = new StringTokenizer(getComment(), "\n");
+ while (st.hasMoreTokens()) {
+ ret.append(prefix).append(st.nextToken()).append("\n");
+ }
+ }
+ return ret.toString();
+ }
+
+ @Override
+ public String toString() {
+ return getNamespace()+"."+getName()+","+getVersion();
+ }
+
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java
new file mode 100644
index 00000000000..dab2ce98510
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+/**
+ * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
+ */
+public interface ClassBuilder {
+
+ /**
+ * Generate config class file(s).
+ */
+ public void createConfigClasses();
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java
new file mode 100644
index 00000000000..56a87c0506e
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+/**
+ * This exception is thrown on internal errors.
+ *
+ * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
+ */
+public class CodegenRuntimeException extends RuntimeException {
+ public CodegenRuntimeException(String s, Throwable cause) {
+ super(s, cause);
+ }
+
+ public CodegenRuntimeException(Throwable cause) {
+ super(cause);
+ }
+
+ public CodegenRuntimeException(String s) {
+ super(s);
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
new file mode 100644
index 00000000000..e6d7f29ad36
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
@@ -0,0 +1,1117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.StringTokenizer;
+
+
+/**
+ * This class autogenerates C++ code for the C++ config, based on a CNode tree given.
+ */
+public class CppClassBuilder implements ClassBuilder {
+ private final CNode root;
+ private final NormalizedDefinition nd;
+ private final File rootDir;
+ private final String relativePathUnderRoot;
+ private static final Map<String, String> vectorTypeDefs;
+ static {
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("bool", "BoolVector");
+ map.put("int32_t", "IntVector");
+ map.put("int64_t", "LongVector");
+ map.put("double", "DoubleVector");
+ map.put("vespalib::string", "StringVector");
+ vectorTypeDefs = Collections.unmodifiableMap(map);
+ }
+ private static final Map<String, String> mapTypeDefs;
+ static {
+ Map<String, String> map = new HashMap<>();
+ map.put("bool", "BoolMap");
+ map.put("int32_t", "IntMap");
+ map.put("int64_t", "LongMap");
+ map.put("double", "DoubleMap");
+ map.put("vespalib::string", "StringMap");
+ mapTypeDefs = Collections.unmodifiableMap(map);
+ }
+ private static final Map<String, String> slimeTypeMap;
+ static {
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("bool", "Bool");
+ map.put("int", "Long");
+ map.put("long", "Long");
+ map.put("double", "Double");
+ map.put("string", "String");
+ map.put("enum", "String");
+ map.put("file", "String");
+ map.put("reference", "String");
+ slimeTypeMap = Collections.unmodifiableMap(map);
+ }
+
+ public CppClassBuilder(CNode root, NormalizedDefinition nd, File rootDir, String relativePathUnderRoot) {
+ this.root = root;
+ this.nd = nd;
+ this.rootDir = rootDir;
+ this.relativePathUnderRoot = relativePathUnderRoot;
+ }
+
+ public void createConfigClasses() {
+ generateConfig(root, nd);
+ }
+
+ String readFile(File f) throws IOException {
+ if (!f.isFile()) return null;
+ StringBuilder sb = new StringBuilder();
+ BufferedReader sr = new BufferedReader(new FileReader(f));
+ while (true) {
+ String line = sr.readLine();
+ if (line == null) break;
+ sb.append(line).append("\n");
+ }
+ return sb.toString();
+ }
+
+ void writeFile(File f, String content) throws IOException {
+ FileWriter fw = new FileWriter(f);
+ fw.write(content);
+ fw.close();
+ }
+
+ void generateConfig(CNode root, NormalizedDefinition nd) {
+ try{
+ StringWriter headerWriter = new StringWriter();
+ StringWriter bodyWriter = new StringWriter();
+ writeHeaderFile(headerWriter, root);
+ writeBodyFile(bodyWriter, root, relativePathUnderRoot, nd);
+
+ String newHeader = headerWriter.toString();
+ String newBody = bodyWriter.toString();
+
+ File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h"));
+ File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp"));
+
+ String oldHeader = readFile(headerFile);
+ String oldBody = readFile(bodyFile);
+
+ if (oldHeader == null || !oldHeader.equals(newHeader)) {
+ writeFile(headerFile, newHeader);
+ }
+ if (oldBody == null || !oldBody.equals(newBody)) {
+ writeFile(bodyFile, newBody);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ String getFileName(CNode node, String extension) {
+ return "config-" + node.getName() + "." + extension;
+ }
+
+ static String removeDashesAndUpperCaseAllFirstChars(String source, boolean capitalizeFirst) {
+ // Create upper case chars after each dash
+ String parts[] = source.split("[-_]");
+ StringBuilder sb = new StringBuilder();
+ for (String s : parts) {
+ sb.append(s.substring(0, 1).toUpperCase()).append(s.substring(1));
+ }
+ String result = sb.toString();
+ if (!capitalizeFirst) {
+ result = result.substring(0,1).toLowerCase() + result.substring(1);
+ }
+ return result;
+ }
+
+ /** Convert name of type to the name we want to use in macro ifdefs in file. */
+ String getDefineName(String name) {
+ return name.toUpperCase().replace("-", "");
+ }
+
+ /** Convert name of type to the name we want to use as type name in the generated code. */
+ static String getTypeName(String name) {
+ return removeDashesAndUpperCaseAllFirstChars(name, true);
+ }
+
+ /** Convert name of an identifier from value in def file to name to use in C++ file. */
+ String getIdentifier(String name) {
+ return removeDashesAndUpperCaseAllFirstChars(name, false);
+ }
+
+ void writeHeaderFile(Writer w, CNode root) throws IOException {
+ writeHeaderHeader(w, root);
+ writeHeaderPublic(w, root);
+ writeHeaderFooter(w, root);
+ }
+
+ void writeHeaderPublic(Writer w, CNode root) throws IOException {
+ w.write("public:\n");
+ writeHeaderTypeDefs(w, root, " ");
+ writeTypeDeclarations(w, root, " ");
+ writeHeaderFunctionDeclarations(w, getTypeName(root, false), root, " ");
+ writeStaticMemberDeclarations(w, " ");
+ writeMembers(w, root, " ");
+ }
+
+ String [] generateCppNameSpace(CNode root) {
+ String namespace = root.getNamespace();
+ if (namespace.contains(".")) {
+ return namespace.split("\\.");
+ }
+ return new String[]{namespace};
+ }
+
+ String generateCppNameSpaceString(String[] namespaceList) {
+ StringBuilder str = new StringBuilder();
+ for (int i = 0; i < namespaceList.length - 1; i++) {
+ str.append(namespaceList[i]);
+ str.append("::");
+ }
+ str.append(namespaceList[namespaceList.length - 1]);
+ return str.toString();
+ }
+
+ String generateCppNameSpaceDefine(String[] namespaceList) {
+ StringBuilder str = new StringBuilder();
+ for (int i = 0; i < namespaceList.length - 1; i++) {
+ str.append(namespaceList[i].toUpperCase());
+ str.append("_");
+ }
+ str.append(namespaceList[namespaceList.length - 1].toUpperCase());
+ return str.toString();
+ }
+
+ void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException {
+ for (int i = 0; i < namespaceList.length; i++) {
+ w.write("namespace " + namespaceList[i] + " {\n\n");
+ }
+ }
+
+ void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException {
+ for (int i = 0; i < namespaceList.length; i++) {
+ w.write("} // namespace " + namespaceList[i] + "\n\n");
+ }
+ }
+
+ void writeHeaderHeader(Writer w, CNode root) throws IOException {
+ String [] namespaceList = generateCppNameSpace(root);
+ String namespacePrint = generateCppNameSpaceString(namespaceList);
+ String namespaceDefine = generateCppNameSpaceDefine(namespaceList);
+ String className = getTypeName(root, false);
+ String defineName = namespaceDefine + "_" + getDefineName(className);
+ w.write(""
+ + "/**\n"
+ + " * @class " + namespacePrint + "::" + className + "\n"
+ + " * @ingroup config\n"
+ + " *\n"
+ + " * @brief This is an autogenerated class for handling VESPA config.\n"
+ + " *\n"
+ + " * This class is autogenerated by vespa from a config definition file.\n"
+ + " * To subscribe to config, you need to include the config/config.h header, \n"
+ + " * and create a ConfigSubscriber in order to subscribe for config.\n"
+ );
+ if (root.getComment().length() > 0) {
+ w.write(" *\n");
+ StringTokenizer st = new StringTokenizer(root.getComment(), "\n");
+ while (st.hasMoreTokens()) {
+ w.write(" * " + st.nextToken() + "\n");
+ }
+ }
+ w.write(""
+ + " */\n"
+ + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n"
+ + "#define CLOUD_CONFIG_" + defineName + "_H\n"
+ + "\n"
+ + "#include <vespa/config/common/configvalue.h>\n"
+ + "#include <vespa/config/configgen/configpayload.h>\n"
+ + "#include <vespa/config/configgen/configinstance.h>\n"
+ + "#include <vespa/config/print/configdatabuffer.h>\n"
+ + "#include <vespa/vespalib/stllike/string.h>\n"
+ + "#include <vector>\n"
+ + "#include <map>\n"
+ + "\n");
+ writeNameSpaceBegin(w, namespaceList);
+ w.write("\n");
+ w.write("namespace internal {\n\n");
+ w.write(""
+ + "/**\n"
+ + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n"
+ + " * versions after this class declaration.\n"
+ + " */\n"
+ + "class Internal" + className + "Type : public ::config::ConfigInstance\n"
+ + "{\n"
+ );
+
+ }
+
+
+
+ void writeTypeDeclarations(Writer w, CNode node, String indent) throws IOException {
+ java.util.Set<String> declaredTypes = new java.util.HashSet<String>();
+ for (CNode child : node.getChildren()) {
+ boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf);
+ if (complexType && !declaredTypes.contains(child.getName())) {
+ String typeName = getTypeName(child, false);
+ declaredTypes.add(child.getName());
+ if (child instanceof LeafCNode.EnumLeaf) {
+ w.write(indent + "enum " + typeName + " { ");
+ LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child;
+ for (int i=0; i<leaf.getLegalValues().length; ++i) {
+ if (i != 0) {
+ w.write(", ");
+ }
+ w.write(leaf.getLegalValues()[i]);
+ }
+ w.write(" };\n"
+ + indent + "typedef std::vector<" + typeName + "> "
+ + typeName + "Vector;"
+ + "\n"
+ + indent + "typedef std::map<vespalib::string, " + typeName + "> "
+ + typeName + "Map;"
+ + "\n"
+ + indent + "static " + typeName + " get" + typeName + "(const vespalib::string&);\n"
+ + indent + "static vespalib::string get" + typeName + "Name(" + typeName + " e);\n"
+ + "\n"
+ );
+ w.write(indent + "struct Internal" + typeName + "Converter {\n");
+ w.write(indent + " " + typeName + " operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector);\n");
+ w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector);\n");
+ w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault);\n");
+ w.write(indent + "};\n");
+ } else {
+ w.write(indent + "class " + typeName + " {\n");
+ w.write(indent + "public:\n");
+ writeTypeDeclarations(w, child, indent + " ");
+ writeStructFunctionDeclarations(w, getTypeName(child, false), child, indent + " ");
+ writeMembers(w, child, indent + " ");
+ w.write(indent + "};\n");
+ w.write(indent + "typedef std::vector<" + typeName + "> " + typeName + "Vector;\n\n");
+ w.write(indent + "typedef std::map<vespalib::string, " + typeName + "> " + typeName + "Map;\n\n");
+ }
+ }
+ }
+ }
+
+ void writeHeaderFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException {
+ w.write(""
+ + indent + "const vespalib::string & defName() const { return CONFIG_DEF_NAME; }\n"
+ + indent + "const vespalib::string & defVersion() const { return CONFIG_DEF_VERSION; }\n"
+ + indent + "const vespalib::string & defMd5() const { return CONFIG_DEF_MD5; }\n"
+ + indent + "const vespalib::string & defNamespace() const { return CONFIG_DEF_NAMESPACE; }\n"
+ + indent + "void serialize(::config::ConfigDataBuffer & __buffer) const;\n");
+ writeConfigClassFunctionDeclarations(w, "Internal" + className + "Type", node, indent);
+ }
+
+ void writeConfigClassFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException {
+ w.write(indent + className + "(const ::config::ConfigValue & __value);\n");
+ w.write(indent + className + "(const ::config::ConfigDataBuffer & __value);\n");
+ w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n");
+ writeCommonFunctionDeclarations(w, className, node, indent);
+ }
+
+ void writeStructFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException {
+ w.write(indent + className + "(const std::vector<vespalib::string> & __lines);\n");
+ w.write(indent + className + "(const vespalib::slime::Inspector & __inspector);\n");
+ w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n");
+ writeCommonFunctionDeclarations(w, className, node, indent);
+ w.write(indent + "void serialize(vespalib::slime::Cursor & __cursor) const;\n");
+ }
+
+ void writeClassCopyConstructorDeclaration(Writer w, String className, CNode node, String indent) throws IOException {
+ w.write(indent + className + "(const " + className + " & __rhs);\n");
+ }
+
+
+ void writeClassCopyConstructorDefinitionCommon(Writer w, CNode node) throws IOException {
+ for (int i = 0; i < node.getChildren().length; ++i) {
+ CNode child = node.getChildren()[i];
+ String childName = getIdentifier(child.getName());
+ if (i == 0) {
+ w.write(" " + childName + "(__rhs." + childName + ")");
+ } else {
+ w.write(",\n " + childName + "(__rhs." + childName + ")");
+ }
+ }
+ }
+
+ void writeConfigClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException {
+ w.write(parent + className + "(const " + className + " & __rhs)\n");
+ w.write(" : ConfigInstance(),\n ");
+ writeClassCopyConstructorDefinitionCommon(w, node);
+ w.write("\n"
+ + "{\n"
+ + "}\n"
+ + "\n"
+ );
+ }
+
+ void writeClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException {
+ String typeName = getTypeName(node, false);
+ // Write empty constructor
+ w.write(parent + typeName + "(const " + typeName + " & __rhs)\n");
+ w.write(" :");
+ writeClassCopyConstructorDefinitionCommon(w, node);
+ w.write("\n"
+ + "{\n"
+ + "}\n"
+ + "\n"
+ );
+ }
+
+ void writeCommonFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException {
+ w.write(""
+ + indent + className + "();\n");
+ writeClassCopyConstructorDeclaration(w, className, node, indent);
+ w.write(""
+ + "\n"
+ + indent + "bool operator==(const " + className + "& __rhs) const;\n"
+ + indent + "bool operator!=(const " + className + "& __rhs) const;\n"
+ + "\n"
+ );
+ }
+
+ static String getTypeName(CNode node, boolean includeArray) {
+ String type = null;
+ if (node instanceof InnerCNode) {
+ InnerCNode innerNode = (InnerCNode) node;
+ type = getTypeName(innerNode.getName());
+ } else if (node instanceof LeafCNode) {
+ LeafCNode leaf = (LeafCNode) node;
+ if (leaf.getType().equals("bool")) {
+ type = "bool";
+ } else if (leaf.getType().equals("int")) {
+ type = "int32_t";
+ } else if (leaf.getType().equals("long")) {
+ type = "int64_t";
+ } else if (leaf.getType().equals("double")) {
+ type = "double";
+ } else if (leaf.getType().equals("enum")) {
+ type = getTypeName(node.getName());
+ } else if (leaf.getType().equals("string")) {
+ type = "vespalib::string";
+ } else if (leaf.getType().equals("reference")) {
+ type = "vespalib::string";
+ } else if (leaf.getType().equals("file")) {
+ type = "vespalib::string";
+ } else {
+ throw new IllegalArgumentException("Unknown leaf datatype " + leaf.getType());
+ }
+ }
+ if (type == null) {
+ throw new IllegalArgumentException("Unknown node " + node);
+ }
+ if (node.isArray && includeArray) {
+ if (vectorTypeDefs.containsKey(type)) {
+ type = vectorTypeDefs.get(type);
+ } else {
+ type = type + "Vector";
+ }
+ } else if (node.isMap && includeArray) {
+ if (mapTypeDefs.containsKey(type)) {
+ type = mapTypeDefs.get(type);
+ } else {
+ type = type + "Map";
+ }
+ }
+ return type;
+ }
+
+ void writeStaticMemberDeclarations(Writer w, String indent) throws IOException {
+ w.write(""
+ + indent + "static const vespalib::string CONFIG_DEF_MD5;\n"
+ + indent + "static const vespalib::string CONFIG_DEF_VERSION;\n"
+ + indent + "static const vespalib::string CONFIG_DEF_NAME;\n"
+ + indent + "static const vespalib::string CONFIG_DEF_NAMESPACE;\n"
+ + indent + "static const std::vector<vespalib::string> CONFIG_DEF_SCHEMA;\n"
+ + indent + "static const int64_t CONFIG_DEF_SERIALIZE_VERSION;\n"
+ + "\n"
+ );
+ }
+
+ void writeComment(Writer w, String indent, String comment, boolean javadoc)
+ throws IOException
+ {
+ /** If simple one liner comment, write on one line. */
+ if (javadoc && comment.indexOf('\n') == -1
+ && comment.length() <= 80 - (indent.length() + 7))
+ {
+ w.write(indent + "/** " + comment + " */\n");
+ return;
+ } else if (!javadoc && comment.indexOf('\n') == -1
+ && comment.length() <= 80 - (indent.length() + 3))
+ {
+ w.write(indent + "// " + comment + "\n");
+ return;
+ }
+ /** If not we need to write multi line comment. */
+ int maxLineLen = 80 - (indent.length() + 3);
+ if (javadoc) w.write(indent + "/**\n");
+ do {
+ String current;
+ // Extract first line to write
+ int newLine = comment.indexOf('\n');
+ if (newLine == -1) {
+ current = comment;
+ comment = "";
+ } else {
+ current = comment.substring(0, newLine);
+ comment = comment.substring(newLine + 1);
+ }
+ // If line too long, cut it in two
+ if (current.length() > maxLineLen) {
+ int spaceIndex = current.lastIndexOf(' ', maxLineLen);
+ if (spaceIndex >= maxLineLen - 15) {
+ comment = current.substring(spaceIndex + 1)
+ + "\n" + comment;
+ current = current.substring(0, spaceIndex);
+ } else {
+ comment = current.substring(maxLineLen) + "\n" + comment;
+ current = current.substring(0, maxLineLen) + "-";
+ }
+ }
+ w.write(indent + (javadoc ? " * " : "// ") + current + "\n");
+ } while (comment.length() > 0);
+ if (javadoc) w.write(indent + " */\n");
+ }
+
+ void writeMembers(Writer w, CNode node, String indent) throws IOException {
+ for (CNode child : node.getChildren()) {
+ String typeName = getTypeName(child, true);
+ if (child.getComment().length() > 0) {
+ String comment = child.getComment();
+ int index;
+ do {
+ index = comment.indexOf("\n\n");
+ if (index == -1) break;
+ String next = comment.substring(0, index);
+ comment = comment.substring(index + 2);
+ w.write("\n");
+ writeComment(w, indent, next, false);
+ } while (true);
+ w.write("\n");
+ writeComment(w, indent, comment, true);
+ }
+ w.write(indent + typeName + " " + getIdentifier(child.getName()) + ";");
+ if (child instanceof LeafCNode) {
+ LeafCNode leaf = (LeafCNode) child;
+ DefaultValue value = leaf.getDefaultValue();
+ if (value != null) {
+ w.write(" // Default: " + value.getStringRepresentation());
+ }
+ }
+ w.write("\n");
+ }
+ }
+
+ void writeHeaderTypeDefs(Writer w, CNode root, String indent) throws IOException {
+ w.write(indent + "typedef std::unique_ptr<const " + getInternalClassName(root) + "> UP;\n");
+ for (Map.Entry<String, String> entry : vectorTypeDefs.entrySet()) {
+ String typeName = entry.getKey();
+ String vectorName = entry.getValue();
+ String typeDef = "typedef std::vector<" + typeName + "> " + vectorName;
+ w.write(indent + typeDef + ";\n");
+ }
+ for (Map.Entry<String, String> entry : mapTypeDefs.entrySet()) {
+ String typeName = entry.getKey();
+ String mapName = entry.getValue();
+ String typeDef = "typedef std::map<vespalib::string, " + typeName + "> " + mapName;
+ w.write(indent + typeDef + ";\n");
+ }
+ }
+
+ private static String getInternalClassName(CNode root) {
+ return "Internal" + getTypeName(root, false) + "Type";
+ }
+
+ void writeHeaderFooter(Writer w, CNode root) throws IOException {
+ String [] namespaceList = generateCppNameSpace(root);
+ String namespaceDefine = generateCppNameSpaceDefine(namespaceList);
+
+ String className = getTypeName(root, false);
+ String defineName = namespaceDefine + "_" + getDefineName(className);
+
+ w.write(""
+ + "};\n"
+ + "\n"
+ + "} // namespace internal\n\n");
+
+ w.write("typedef internal::" + getInternalClassName(root) + " " + className + "ConfigBuilder;\n");
+ w.write("typedef const internal::" + getInternalClassName(root) + " " + className + "Config;\n");
+ w.write("\n");
+ writeNameSpaceEnd(w, namespaceList);
+ w.write("#endif // VESPA_config_" + defineName + "_H\n");
+ }
+
+ void writeBodyFile(Writer w, CNode root, String subdir, NormalizedDefinition nd) throws IOException {
+ writeBodyHeader(w, root, subdir);
+ writeStaticMemberDefinitions(w, root, nd);
+ writeDefinition(w, root, null);
+ writeBodyFooter(w, root);
+ }
+
+ void writeBodyHeader(Writer w, CNode root, String subdir) throws IOException {
+ if (subdir == null) {
+ w.write("#include \"" + getFileName(root, "h") + "\"");
+ } else {
+ w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">");
+ }
+ w.write("\n");
+ w.write("#include <set>\n");
+ w.write("#include <vespa/config/common/configparser.h>\n");
+ w.write("#include <vespa/vespalib/data/slime/convenience.h>\n");
+ w.write("#include <vespa/vespalib/stllike/asciistream.h>\n");
+ w.write("#include <vespa/vespalib/stllike/asciistream.h>\n");
+ w.write("#include <vespa/config/configgen/vector_inserter.h>\n");
+ w.write("#include <vespa/config/configgen/map_inserter.h>\n");
+ w.write("\n\n");
+ writeNameSpaceBegin(w, generateCppNameSpace(root));
+ w.write("\n");
+ w.write("namespace internal {\n\n");
+ w.write("using ::config::ConfigParser;\n");
+ w.write("using ::config::InvalidConfigException;\n");
+ w.write("using ::config::ConfigInstance;\n");
+ w.write("using ::config::ConfigValue;\n");
+ w.write("using namespace vespalib::slime::convenience;\n");
+ w.write("\n");
+ }
+
+ void writeStaticMemberDefinitions(Writer w, CNode root, NormalizedDefinition nd) throws IOException {
+ String typeName = getInternalClassName(root);
+ w.write("const vespalib::string " + typeName + "::CONFIG_DEF_MD5(\"" + root.defMd5 + "\");\n"
+ + "const vespalib::string " + typeName + "::CONFIG_DEF_VERSION(\"" + root.defVersion + "\");\n"
+ + "const vespalib::string " + typeName + "::CONFIG_DEF_NAME(\"" + root.defName + "\");\n"
+ + "const vespalib::string " + typeName + "::CONFIG_DEF_NAMESPACE(\"" + root.getNamespace() + "\");\n"
+ + "const int64_t " + typeName + "::CONFIG_DEF_SERIALIZE_VERSION(1);\n");
+ w.write("const static vespalib::string __internalDefSchema[] = {\n");
+ for (String line : nd.getNormalizedContent()) {
+ w.write("\"" + line.replace("\"", "\\\"") + "\",\n");
+ }
+ w.write("};\n");
+ w.write("const std::vector<vespalib::string> " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n");
+ w.write(" __internalDefSchema + (sizeof(__internalDefSchema) / \n");
+ w.write(" sizeof(__internalDefSchema[0])));\n");
+ w.write("\n");
+ }
+
+ void writeDefinition(Writer w, CNode node, String parent) throws IOException {
+ boolean root = false;
+ if (parent == null) {
+ parent = getInternalClassName(node) + "::";
+ root = true;
+ }
+ java.util.Set<String> declaredTypes = new java.util.HashSet<String>();
+ for (CNode child : node.getChildren()) {
+ boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf);
+ if (complexType && !declaredTypes.contains(child.getName())) {
+ String typeName = getTypeName(child, false);
+ declaredTypes.add(child.getName());
+ if (child instanceof LeafCNode.EnumLeaf) {
+ LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child;
+ // Definition of getType(string)
+ w.write(parent + typeName + "\n"
+ + parent + "get" + typeName + "(const vespalib::string& name)\n"
+ + "{\n"
+ );
+ for (int i=0; i<leaf.getLegalValues().length; ++i) {
+ w.write(" " + (i != 0 ? "} else " : ""));
+ w.write("if (name == \"" + leaf.getLegalValues()[i] + "\") {\n"
+ + " return " + leaf.getLegalValues()[i] + ";\n");
+ }
+ w.write(" } else {\n"
+ + " throw InvalidConfigException(\"Illegal enum value '\" + name + \"'\");\n"
+ + " }\n"
+ + "}\n"
+ + "\n"
+ );
+ // Definition of getTypeName(enum)
+ w.write("vespalib::string\n"
+ + parent + "get" + typeName + "Name(" + typeName + " t)\n"
+ + "{\n"
+ + " switch (t) {\n"
+ );
+ for (int i=0; i<leaf.getLegalValues().length; ++i) {
+ w.write(" case " + leaf.getLegalValues()[i] + ": return \"" + leaf.getLegalValues()[i] + "\";\n");
+ }
+ w.write(" default:\n"
+ + " {\n"
+ + " vespalib::asciistream ost;\n"
+ + " ost << \"UNKNOWN(\" << t << \")\";\n"
+ + " return ost.str();\n"
+ + " }\n"
+ + " }\n"
+ + "}\n"
+ + "\n"
+ );
+ w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) {\n");
+ w.write(" if (__inspector.valid()) {\n");
+ w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n");
+ w.write(" }\n");
+ w.write(" throw InvalidConfigException(\"Value for '\" + __fieldName + \"' required but not found\");\n");
+ w.write("}\n");
+ w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector) {\n");
+ w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n");
+ w.write("}\n");
+ w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault) {\n");
+ w.write(" if (__inspector.valid()) {\n");
+ w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n");
+ w.write(" }\n");
+ w.write(" return __eDefault;\n");
+ w.write("}\n\n");
+ } else {
+ writeDefinition(w, child, parent + typeName + "::");
+ }
+ }
+ }
+ String tmpName = getTypeName(node, false);
+ String typeName = root ? getInternalClassName(node) : tmpName;
+ // Write empty constructor
+ w.write(parent + typeName + "()\n");
+ for (int i=0; i<node.getChildren().length; ++i) {
+ CNode child = node.getChildren()[i];
+ String childName = getIdentifier(child.getName());
+ if (i == 0) {
+ w.write(" : " + childName + "(");
+ } else {
+ w.write("),\n " + childName + "(");
+ }
+ if (child.isArray || child.isMap) {
+ // Default array for empty constructor is empty array.
+ } else if (child instanceof LeafCNode) { // If we have a default value, use that..
+ LeafCNode leaf = (LeafCNode) child;
+ if (leaf.getDefaultValue() != null) {
+ w.write(getDefaultValue(leaf));
+ } else {
+ // Defines empty constructor defaults for primitives without default set
+ if (leaf.getType().equals("bool")) {
+ w.write("false");
+ } else if (leaf.getType().equals("int")) {
+ w.write("0");
+ } else if (leaf.getType().equals("double")) {
+ w.write("0");
+ } else if (leaf.getType().equals("string")) {
+ } else if (leaf.getType().equals("enum")) {
+ LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) leaf;
+ w.write(enumNode.getLegalValues()[0]);
+ } else if (leaf.getType().equals("reference")) {
+ } else if (leaf.getType().equals("file")) {
+ }
+ }
+ }
+ // If we hit neither else, we're an inner node, thus special type that has its own empty constructor
+ }
+ if (node.getChildren().length > 0)
+ w.write(")\n");
+ w.write(""
+ + "{\n"
+ + "}\n"
+ + "\n"
+ );
+ // Write copy constructor
+ if (root)
+ writeConfigClassCopyConstructorDefinition(w, parent, typeName, node);
+ else
+ writeClassCopyConstructorDefinition(w, parent, typeName, node);
+
+ // Write parsing constructor
+ String indent = " ";
+ if (root) {
+ w.write(typeName + "::" + typeName + "(const ConfigValue & __value)\n"
+ + "{\n"
+ + indent + "try {\n");
+ indent = " ";
+ w.write(indent + "const std::vector<vespalib::string> & __lines(__value.getLines());\n");
+ } else {
+ w.write(parent + typeName + "(const std::vector<vespalib::string> & __lines)\n"
+ + "{\n");
+ }
+ w.write(""
+ + indent + "std::set<vespalib::string> __remainingValuesToParse("
+ + "__lines.begin(), __lines.end());\n");
+ w.write(indent + "for(std::set<vespalib::string>::iterator __rVTPiter = __remainingValuesToParse.begin();\n"
+ + indent + " __rVTPiter != __remainingValuesToParse.end();)\n"
+ + indent + "{\n"
+ + indent + " if (ConfigParser::stripWhitespace(*__rVTPiter).empty()) {\n"
+ + indent + " std::set<vespalib::string>::iterator __rVTPiter2 = __rVTPiter++;\n"
+ + indent + " __remainingValuesToParse.erase(__rVTPiter2);\n"
+ + indent + " } else {\n"
+ + indent + " ++__rVTPiter;\n"
+ + indent + " }\n"
+ + indent + "}\n");
+ for (CNode child : node.getChildren()) {
+ String childType = getTypeName(child, false);
+ String childName = getIdentifier(child.getName());
+ if (child instanceof LeafCNode.EnumLeaf) {
+ if (child.isArray) {
+ w.write(indent + "std::vector<vespalib::string> " + childName + "__ValueList(\n ");
+ } else if (child.isMap) {
+ w.write(indent + "std::map<vespalib::string, vespalib::string> " + childName + "__ValueMap(\n ");
+ } else {
+ w.write(indent + childName + " = get" + childType + "(");
+ }
+ childType = "vespalib::string";
+ } else {
+ w.write(indent + childName + " = ");
+ }
+ if (child.isArray) {
+ w.write("ConfigParser::parseArray<" + childType + ">(\""
+ + child.getName() + "\", __lines)");
+ } else if (child.isMap) {
+ w.write("ConfigParser::parseMap<" + childType + ">(\""
+ + child.getName() + "\", __lines)");
+ } else {
+ if (child instanceof LeafCNode) {
+ w.write("ConfigParser::parse<" + childType + ">(\""
+ + child.getName() + "\", __lines");
+ } else {
+ w.write("ConfigParser::parseStruct<" + childType + ">(\""
+ + child.getName() + "\", __lines");
+ }
+ if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) {
+ LeafCNode leaf = (LeafCNode) child;
+ if (leaf.getDefaultValue().getValue() != null) {
+ String defaultVal = getDefaultValue(leaf);
+ if (leaf instanceof LeafCNode.EnumLeaf) {
+ defaultVal = '"' + defaultVal + '"';
+ }
+ w.write(", " + defaultVal);
+ }
+ }
+ w.write(")");
+ }
+ if (child instanceof LeafCNode.EnumLeaf) {
+ childType = getTypeName(child, false);
+ w.write(");\n");
+ if (child.isArray) {
+ w.write(indent + childName + ".reserve(" + childName + "__ValueList.size());\n"
+ + indent + "for (std::vector<vespalib::string>::const_iterator __it\n"
+ + indent + " = " + childName + "__ValueList.begin();\n"
+ + indent + " __it != " + childName + "__ValueList.end(); ++__it)\n"
+ + indent + "{\n"
+ + indent + " " + childName + ".push_back(get" + childType + "(*__it));\n"
+ + indent + "}\n"
+ );
+ } else if (child.isMap) {
+ w.write(indent + "typedef std::map<vespalib::string, vespalib::string> __ValueMap;\n");
+ w.write(indent + "for (__ValueMap::iterator __it(" + childName + "__ValueMap.begin()), __mt(" + childName + "__ValueMap.end()); __it != __mt; __it++) {\n"
+ + " " + childName + "[__it->first] = get" + childType + "(__it->second);\n"
+ + "}\n"
+ );
+ }
+ } else {
+ w.write(";\n");
+ }
+ w.write(indent + "ConfigParser::stripLinesForKey(\""
+ + child.getName() + "\", "
+ + "__remainingValuesToParse);\n");
+ }
+ if (root) {
+ indent = " ";
+ w.write(indent + "} catch (InvalidConfigException & __ice) {\n");
+ w.write(indent + " throw InvalidConfigException(\"Error parsing config '\" + CONFIG_DEF_NAME + \"' in namespace '\" + CONFIG_DEF_NAMESPACE + \"'"
+ + ": \" + __ice.getMessage());\n"
+ + indent + "}\n");
+ }
+ w.write("}\n"
+ + "\n"
+ );
+ // Write operator==
+ String lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n");
+ w.write("bool\n"
+ + parent + lineBreak + "operator==(const " + typeName + "& __rhs) const\n"
+ + "{\n"
+ + " return ("
+ );
+ for (int i = 0; i<node.getChildren().length; ++i) {
+ CNode child = node.getChildren()[i];
+ String childName = getIdentifier(child.getName());
+ if (i != 0) {
+ w.write(" &&\n ");
+ }
+ w.write(childName + " == __rhs." + childName);
+ }
+ w.write(");\n"
+ + "}\n"
+ + "\n"
+ );
+ // Write operator!=
+ lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n");
+ w.write("bool\n"
+ + parent + lineBreak + "operator!=(const " + typeName + "& __rhs) const\n"
+ + "{\n"
+ + " return !(operator==(__rhs));\n"
+ + "}\n"
+ + "\n"
+ );
+ writeSlimeEncoder(w, node, parent, root);
+ writeSlimeDecoder(w, node, parent, root);
+ writeSlimeConstructor(w, node, parent, root);
+ }
+
+ public void writeSlimeEncoder(Writer w, CNode node, String parent, boolean root) throws IOException
+ {
+ String indent = " ";
+ if (root) {
+ w.write("void\n"
+ + parent + "serialize(::config::ConfigDataBuffer & __buffer) const\n"
+ + "{\n");
+ w.write(indent + "vespalib::Slime & __slime(__buffer.slimeObject());\n");
+ w.write(indent + "vespalib::slime::Cursor & __croot = __slime.setObject();\n");
+ w.write(indent + "__croot.setDouble(\"version\", CONFIG_DEF_SERIALIZE_VERSION);\n");
+ w.write(indent + "vespalib::slime::Cursor & __key = __croot.setObject(\"configKey\");\n");
+ w.write(indent + "__key.setString(\"defName\", vespalib::slime::Memory(CONFIG_DEF_NAME));\n");
+ w.write(indent + "__key.setString(\"defNamespace\", vespalib::slime::Memory(CONFIG_DEF_NAMESPACE));\n");
+ w.write(indent + "__key.setString(\"defMd5\", vespalib::slime::Memory(CONFIG_DEF_MD5));\n");
+ w.write(indent + "vespalib::slime::Cursor & __keySchema =__key.setArray(\"defSchema\");\n");
+ w.write(indent + "for (size_t i = 0; i < CONFIG_DEF_SCHEMA.size(); i++) {\n");
+ w.write(indent + " __keySchema.addString(vespalib::slime::Memory(CONFIG_DEF_SCHEMA[i]));\n");
+ w.write(indent + "}\n");
+ w.write(indent + "vespalib::slime::Cursor & __cursor = __croot.setObject(\"configPayload\");\n");
+ } else {
+ w.write("void\n"
+ + parent + "serialize(vespalib::slime::Cursor & __cursor) const\n"
+ + "{\n");
+ }
+ for (CNode child : node.getChildren()) {
+ String childName = getIdentifier(child.getName());
+ String childType = getTypeName(child, false);
+ w.write(indent + "{\n");
+ indent = " ";
+ w.write(indent + "vespalib::slime::Cursor & __c = __cursor.setObject(\"" + child.getName() + "\");\n");
+ if (child.isArray) {
+ w.write(indent + "__c.setString(\"type\", \"array\");\n");
+ w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n");
+ w.write(indent + "for (size_t __i = 0; __i < " + childName + ".size(); __i++) {\n");
+ w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write(indent + " __c3.setString(\"type\", \"enum\");\n");
+ w.write(indent + " __c3.set" + repType);
+ w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + "[__i])));\n");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n");
+ w.write(indent + " __c3.set" + repType);
+ if ("String".equals(repType)) {
+ w.write("(\"value\", vespalib::slime::Memory(" + childName + "[__i]));\n");
+ } else {
+ w.write("(\"value\", " + childName + "[__i]);\n");
+ }
+ } else {
+ w.write(indent + " __c3.setString(\"type\", \"struct\");\n");
+ w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n");
+ w.write(indent + " " + childName + "[__i].serialize(__c4);\n");
+ }
+ w.write(indent + "}\n");
+ } else if (child.isMap) {
+ w.write(indent + "__c.setString(\"type\", \"map\");\n");
+ w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n");
+ String childMapType = getTypeName(child, true);
+ w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n");
+ w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n");
+ w.write(indent + " __c3.setString(\"key\", vespalib::slime::Memory(it->first));\n");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write(indent + " __c3.setString(\"type\", \"enum\");\n");
+ w.write(indent + " __c3.set" + repType);
+ w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(it->second)));\n");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n");
+ w.write(indent + " __c3.set" + repType);
+ if ("String".equals(repType)) {
+ w.write("(\"value\", vespalib::slime::Memory(it->second));\n");
+ } else {
+ w.write("(\"value\", it->second);\n");
+ }
+ } else {
+ w.write(indent + " __c3.setString(\"type\", \"struct\");\n");
+ w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n");
+ w.write(indent + " it->second.serialize(__c4);\n");
+ }
+ w.write(indent + "}\n");
+ } else {
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write(indent + "__c.setString(\"type\", \"enum\");\n");
+ w.write(indent + "__c.set" + repType);
+ w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + ")));\n");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ w.write(indent + "__c.setString(\"type\", \"" + type + "\");\n");
+ w.write(indent + "__c.set" + repType);
+ if ("String".equals(repType)) {
+ w.write("(\"value\", vespalib::slime::Memory(" + childName + "));\n");
+ } else {
+ w.write("(\"value\", " + childName + ");\n");
+ }
+ } else {
+ w.write(indent + "__c.setString(\"type\", \"struct\");\n");
+ w.write(indent + "Cursor & __c2 = __c.setObject(\"value\");\n");
+ w.write(indent + childName + ".serialize(__c2);\n");
+ }
+ }
+ indent = " ";
+ w.write(indent + "}\n");
+
+ }
+ w.write("}\n\n");
+ }
+
+ public void writeSlimeDecoder(Writer w, CNode node, String parent, boolean root) throws IOException {
+ String tmpName = getTypeName(node, false);
+ String typeName = root ? getInternalClassName(node) : tmpName;
+ String indent = " ";
+ if (root) {
+ w.write(""
+ + typeName + "::" + typeName + "(const ::config::ConfigDataBuffer & __buffer)\n"
+ + "{\n");
+ w.write(indent + "const vespalib::Slime & __slime(__buffer.slimeObject());\n");
+ w.write(indent + "vespalib::slime::Inspector & __croot = __slime.get();\n");
+ w.write(indent + "vespalib::slime::Inspector & __inspector = __croot[\"configPayload\"];\n");
+ } else {
+ w.write(""
+ + parent + typeName + "(const vespalib::slime::Inspector & __inspector)\n"
+ + "{\n");
+ }
+
+ for (CNode child : node.getChildren()) {
+ String childName = getIdentifier(child.getName());
+ String childType = getTypeName(child, false);
+ String inspectorLine = "__inspector[\"" + child.getName() + "\"][\"value\"]";
+ if (child.isArray) {
+ w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n");
+ w.write(indent + " " + childName + ".push_back(");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ if ("String".equals(repType)) {
+ w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()");
+ } else {
+ w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()");
+ }
+ } else {
+ w.write(childType + "(" + inspectorLine + "[__i][\"value\"])");
+ }
+ w.write(");\n");
+ w.write(indent + "}\n");
+ } else if (child.isMap) {
+ w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n");
+ w.write(indent + " " + childName + "[" + inspectorLine + "[__i][\"key\"].asString().make_string()] = ");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ if ("String".equals(repType)) {
+ w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()");
+ } else {
+ w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()");
+ }
+ } else {
+ w.write(childType + "(" + inspectorLine + "[__i][\"value\"])");
+ }
+ w.write(";\n");
+ w.write(indent + "}\n");
+ } else {
+ w.write(indent + childName + " = ");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ String repType = slimeTypeMap.get("enum");
+ w.write("get" + childType + "(" + inspectorLine + ".as" + repType + "().make_string())");
+ } else if (child instanceof LeafCNode) {
+ String type = ((LeafCNode) child).getType();
+ String repType = slimeTypeMap.get(type);
+ if ("String".equals(repType)) {
+ w.write("" + inspectorLine + ".as" + repType + "().make_string()");
+ } else {
+ w.write("" + inspectorLine + ".as" + repType + "()");
+ }
+ } else {
+ w.write(childType + "(" + inspectorLine + ")");
+ }
+ w.write(";\n");
+ }
+ }
+ w.write("}\n\n");
+ }
+
+ public void writeSlimeConstructor(Writer w, CNode node, String parent, boolean root) throws IOException {
+ String tmpName = getTypeName(node, false);
+ String typeName = root ? getInternalClassName(node) : tmpName;
+ String indent = " ";
+ if (root) {
+ w.write(""
+ + typeName + "::" + typeName + "(const ::config::ConfigPayload & __payload)\n"
+ + "{\n");
+ } else {
+ w.write(""
+ + parent + typeName + "(const ::config::ConfigPayload & __payload)\n"
+ + "{\n");
+ }
+ w.write(indent + "const vespalib::slime::Inspector & __inspector(__payload.get());\n");
+ for (CNode child : node.getChildren()) {
+ String childName = getIdentifier(child.getName());
+ String childType = getTypeName(child, false);
+ String childInspector = "__inspector[\"" + child.getName() + "\"]";
+ if (child.isArray) {
+ String inserterName = "__" + childName + "Inserter";
+ w.write(indent + "::config::internal::VectorInserter<" + childType);
+ if (child instanceof LeafCNode.EnumLeaf) {
+ w.write(", Internal" + childType + "Converter");
+ }
+ w.write("> " + inserterName + "(" + childName + ");\n");
+ w.write(indent + childInspector + ".traverse(" + inserterName + ");\n");
+ } else if (child.isMap) {
+ String inserterName = "__" + childName + "Inserter";
+ w.write(indent + "::config::internal::MapInserter<" + childType);
+ if (child instanceof LeafCNode.EnumLeaf) {
+ w.write(", Internal" + childType + "Converter");
+ }
+ w.write("> " + inserterName + "(" + childName + ");\n");
+ w.write(indent + childInspector + ".traverse(" + inserterName + ");\n");
+ } else {
+ w.write(indent + childName + " = ");
+ if (child instanceof LeafCNode.EnumLeaf) {
+ w.write("Internal" + childType + "Converter");
+ } else {
+ w.write("::config::internal::ValueConverter<" + childType + ">");
+ }
+ if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) {
+ LeafCNode leaf = (LeafCNode) child;
+ String defaultValue = getDefaultValue(leaf);
+ w.write("()(" + childInspector + ", " + defaultValue + ");\n");
+ } else if (child instanceof InnerCNode) {
+ w.write("()(" + childInspector + ");\n");
+ } else {
+ w.write("()(\"" + child.getName() + "\", " + childInspector + ");\n");
+ }
+ }
+ }
+ w.write("}\n\n");
+ }
+
+ void writeBodyFooter(Writer w, CNode root) throws IOException {
+ w.write("} // namespace internal\n\n");
+ writeNameSpaceEnd(w, generateCppNameSpace(root));
+ }
+
+ String getDefaultValue(LeafCNode leaf) {
+ String defaultVal = leaf.getDefaultValue().getStringRepresentation();
+ if (leaf.getType().equals("string") && defaultVal.equals("null"))
+ throw new CodegenRuntimeException("Default value null not allowed for C++ config");
+ if (leaf.getType().equals("long") && "-9223372036854775808".equals(defaultVal)) {
+ return "LONG_MIN";
+ } else if (leaf.getType().equals("int") && "-2147483648".equals(defaultVal)) {
+ return "INT_MIN";
+ } else {
+ return defaultVal;
+ }
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java
new file mode 100644
index 00000000000..f58c202f70f
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java
@@ -0,0 +1,274 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ */
+public class DefLine {
+ private final static Pattern defaultPattern = Pattern.compile("^\\s*default\\s*=\\s*(\\S+)");
+ private final static Pattern rangePattern = Pattern.compile("^\\s*range\\s*=\\s*([\\(\\[].*?[\\)\\]])");
+ private final static Pattern restartPattern = Pattern.compile("^\\s*restart\\s*");
+ private final static Pattern wordPattern = Pattern.compile("\\S+");
+ private final static Pattern enumPattern = Pattern.compile("\\s*\\{(\\s*\\w+\\s*)+(\\s*,\\s*\\w+\\s*)*\\s*\\}");
+ private final static Pattern enumPattern2 = Pattern.compile("\\s*,\\s*");
+ private final static Pattern wordPattern2 = Pattern.compile("\\w+");
+ private final static Pattern digitPattern = Pattern.compile("\\d");
+ private final static Pattern namePattern = Pattern.compile("\\s*[a-zA-Z0-9_]+\\s*");
+ private final static Pattern whitespacePattern = Pattern.compile("\\s+");
+
+ private String name = null;
+ private final Type type = new Type();
+
+ private DefaultValue defaultValue = null;
+
+ private String range = null;
+ private boolean restart = false;
+
+ String enumString = null;
+ final String[] enumArray = null;
+
+ private final static Pattern defaultNullPattern = Pattern.compile("^\\s*default\\s*=\\s*null");
+
+ public DefLine(String line) {
+ StringBuilder sb = new StringBuilder(line);
+ int parsed = parseNameType(sb);
+ sb.delete(0, parsed);
+ if (type.name.equals("enum")) {
+ parsed = parseEnum(sb);
+ sb.delete(0, parsed);
+ }
+
+ while (sb.length() > 0) {
+ parsed = parseOptions(sb);
+ sb.delete(0, parsed);
+ }
+ validateName();
+ validateReservedWords();
+ }
+
+ /**
+ * Currently (2012-03-05) not used. Ranges are not checked by the
+ */
+ public String getRange() {
+ return range;
+ }
+
+ public DefaultValue getDefault() {
+ return defaultValue;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public boolean getRestart() {
+ return restart;
+ }
+
+ public String getEnumString() {
+ return enumString;
+ }
+
+ public String[] getEnumArray() {
+ return enumArray;
+ }
+
+ /**
+ * Special function that searches through s and returns the index
+ * of the first occurrence of " that is not escaped.
+ */
+ private String findStringEnd(CharSequence s, int from) {
+ boolean escaped = false;
+ for (int i = from; i < s.length(); i++) {
+ switch (s.charAt(i)) {
+ case'\\':
+ escaped = !escaped;
+ break;
+ case'"':
+ if (!escaped) {
+ return s.subSequence(from, i).toString();
+ }
+ break;
+ }
+ }
+ return null;
+ }
+
+
+ private int parseOptions(CharSequence string) {
+ Matcher defaultNullMatcher = defaultNullPattern.matcher(string);
+ Matcher defaultMatcher = defaultPattern.matcher(string);
+ Matcher rangeMatcher = rangePattern.matcher(string);
+ Matcher restartMatcher = restartPattern.matcher(string);
+
+ if (defaultNullMatcher.find()) {
+ throw new IllegalArgumentException("Null default value is not allowed: " + string.toString());
+ } else if (defaultMatcher.find()) {
+ String deflt = defaultMatcher.group(1);
+ if (deflt.charAt(0) == '"') {
+ int begin = defaultMatcher.start(1) + 1;
+ deflt = findStringEnd(string, begin);
+ if (deflt == null) {
+ throw new IllegalArgumentException(string.toString());
+ }
+ defaultValue = new DefaultValue(deflt, type);
+ return begin + deflt.length() + 1;
+ } else {
+ defaultValue = new DefaultValue(deflt, type);
+ }
+ return defaultMatcher.end();
+ } else if (rangeMatcher.find()) {
+ range = rangeMatcher.group(1);
+ return rangeMatcher.end();
+ } else if (restartMatcher.find()) {
+ restart = true;
+ return restartMatcher.end();
+ } else {
+ throw new IllegalArgumentException(string.toString());
+ }
+ }
+
+ private int parseNameType(CharSequence string) {
+ Matcher wordMatcher = wordPattern.matcher(string);
+ if (wordMatcher.find()) {
+ name = wordMatcher.group();
+ }
+ if (wordMatcher.find()) {
+ type.name = wordMatcher.group();
+ }
+ if (type.name == null || name == null) {
+ throw new IllegalArgumentException(string.toString());
+ }
+ return wordMatcher.end();
+ }
+
+ private int parseEnum(CharSequence string) {
+ Matcher enumMatcher = enumPattern.matcher(string);
+ if (enumMatcher.find()) {
+ enumString = enumMatcher.group(0).trim();
+ }
+ if (enumString == null) {
+ throw new IllegalArgumentException(string + " is not valid syntax");
+ }
+ enumString = enumString.replaceFirst("\\{\\s*", "");
+ enumString = enumString.replaceFirst("\\s*\\}", "");
+ String result[] = enumPattern2.split(enumString);
+ type.enumArray = new String[result.length];
+ for (int i = 0; i < result.length; i++) {
+ String s = result[i].trim();
+ type.enumArray[i] = s;
+ Matcher wordMatcher2 = wordPattern2.matcher(s);
+ if (!wordMatcher2.matches()) {
+ throw new IllegalArgumentException(s + " is not valid syntax");
+ }
+ }
+ return enumMatcher.end();
+ }
+
+ public static class Type {
+ String name;
+ String[] enumArray;
+
+ public Type(String name) {
+ this.name=name;
+ }
+
+ public Type() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String[] getEnumArray() {
+ return enumArray;
+ }
+
+ public Type setEnumArray(String[] enumArray) {
+ this.enumArray = enumArray;
+ return this;
+ }
+
+ public String toString() {
+ return "type " + name;
+ }
+
+ }
+
+ // A naive approach to imitate the checking previously done in make-config-preproc.pl
+ // TODO: method too long
+ void validateName() {
+ Matcher digitMatcher;
+ Matcher nameMatcher;
+ Matcher whitespaceMatcher;
+
+ boolean atStart = true;
+ boolean arrayOk = true;
+ boolean mapOk = true;
+ for (int i = 0; i < name.length(); i++) {
+ String s = name.substring(i, i + 1);
+ digitMatcher = digitPattern.matcher(s);
+ nameMatcher = namePattern.matcher(s);
+ whitespaceMatcher = whitespacePattern.matcher(s);
+ if (atStart) {
+ if (digitMatcher.matches()) {
+ throw new IllegalArgumentException(name + " must start with a non-digit character");
+ }
+ if (!nameMatcher.matches()) {
+ throw new IllegalArgumentException(name + " contains unexpected character");
+ }
+ atStart = false;
+ } else {
+ if (nameMatcher.matches()) {
+ // do nothing
+ } else if (s.equals(".")) {
+ arrayOk = true;
+ mapOk = true;
+ atStart = true;
+ } else if (s.equals("[")) {
+ if (!arrayOk) {
+ throw new IllegalArgumentException(name + " Arrays cannot be multidimensional");
+ }
+ arrayOk = false;
+ if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("]"))) {
+ throw new IllegalArgumentException(name + " Expected ] to terminate array definition");
+ }
+ i++;
+ } else if (s.equals("{")) {
+ if (!mapOk) {
+ throw new IllegalArgumentException(name + " Maps cannot be multidimensional");
+ }
+ mapOk = false;
+ if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("}"))) {
+ throw new IllegalArgumentException(name + " Expected } to terminate map definition");
+ }
+ i++;
+ } else if (whitespaceMatcher.matches()) {
+ break;
+ } else {
+ throw new IllegalArgumentException(name + " contains unexpected character");
+ }
+ }
+ }
+ }
+
+ void validateReservedWords() {
+ if (ReservedWords.isReservedWord(name)) {
+ throw new IllegalArgumentException(name + " is a reserved word in " +
+ ReservedWords.getLanguageForReservedWord(name));
+ }
+ if (ReservedWords.capitalizedPattern.matcher(name).matches()) {
+ throw new IllegalArgumentException("'" + name + "' cannot start with an uppercase letter");
+ }
+ if (ReservedWords.internalPrefixPattern.matcher(name).matches()) {
+ throw new IllegalArgumentException("'" + name + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'");
+ }
+ }
+}
+
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java
new file mode 100644
index 00000000000..0a6c225fa19
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java
@@ -0,0 +1,216 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class generates a tree of CNodes from a .def file.
+ *
+ * @author gjoranv
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class DefParser {
+ static final Pattern commentPattern = Pattern.compile("^\\s*#+\\s*(.*?)\\s*$");
+ public static final Pattern versionPattern = Pattern.compile("^(version\\s*=\\s*)([0-9][0-9-]*)$");
+ // Namespace must start with a letter, since Java (Java language Spec, section 3.8) and C++ identifiers cannot start with a digit
+ public static final Pattern namespacePattern = Pattern.compile("^(namespace\\s*=\\s*)(([a-z][a-z0-9_]*)+([.][a-z][a-z0-9_]*)*)$");
+
+ private final BufferedReader reader;
+ private final String name;
+ private InnerCNode root = null;
+ private NormalizedDefinition normalizedDefinition = null;
+
+
+ private String comment = "";
+
+ /**
+ * Creates a new parser for a .def file with the given name and that can be accessed by the given reader.
+ *
+ * @param name The name of the .def file (not including version number and the '.def' suffix).
+ * @param defReader A reader to the .def file.
+ */
+ public DefParser(String name, Reader defReader) {
+ this.name = createName(name);
+ if (defReader == null) {
+ throw new CodegenRuntimeException("Must have a non-null reader for a .def file.");
+ }
+ if (defReader instanceof BufferedReader) {
+ reader = (BufferedReader)defReader;
+ } else {
+ reader = new BufferedReader(defReader);
+ }
+ }
+
+ // If name contains namespace, return just name
+ private String createName(String name) {
+ if (name.contains(".")) {
+ return name.substring(name.lastIndexOf(".") + 1);
+ } else {
+ return name;
+ }
+ }
+
+ /**
+ * Parses the .def file upon the initial call. Subsequent calls returns the result from the initial call.
+ *
+ * @return A tree of CNodes representing this instance's .def file.
+ * @throws CodegenRuntimeException upon errors.
+ */
+ public InnerCNode getTree() throws CodegenRuntimeException {
+ try {
+ if (root == null) parse();
+ } catch (DefParserException | IOException e) {
+ throw new CodegenRuntimeException("Error parsing or reading config definition." + e.getMessage(), e);
+ }
+ return root;
+ }
+
+ /**
+ * Parses the input from the reader and builds a tree of CNodes representing the .def file.
+ *
+ * @throws IOException upon reader errors.
+ * @throws DefParserException upon parsing errors.
+ */
+ void parse() throws IOException, DefParserException {
+ root = new InnerCNode(name);
+ normalizedDefinition = new NormalizedDefinition();
+
+ String s;
+ List<String> originalInput = new ArrayList<>();
+ while ((s = reader.readLine()) != null) {
+ originalInput.add(s);
+ }
+ reader.close();
+
+
+ // Parse and build tree of the original input
+ parseLines(root, originalInput, normalizedDefinition);
+ root.setMd5(normalizedDefinition.generateMd5Sum());
+ }
+
+ /**
+ * Parses one line from def-file and adds it to the tree.
+ * TODO: Method too long!
+ *
+ * @param root The root CNode in the tree.
+ * @param line A line from the def-file.
+ * @param nd A NormalizedDefinition object
+ * @throws IllegalArgumentException upon error in line.
+ */
+ private void parseLine(CNode root, String line, NormalizedDefinition nd) throws IllegalArgumentException {
+ line = NormalizedDefinition.normalize(line);
+ line = line.trim();
+ if (line.length() == 0) {
+ // If having empty lines in between comments, that is logically a break in the comment too
+ if (!comment.isEmpty()) {
+ comment += "\n";
+ }
+ return;
+ }
+ Matcher commentMatch = commentPattern.matcher(line);
+ if (commentMatch.matches()) {
+ parseCommentLine(commentMatch);
+ return;
+ }
+ Matcher versionMatch = versionPattern.matcher(line);
+ if (versionMatch.matches()) {
+ parseVersionLine(versionMatch);
+ return;
+ }
+ Matcher namespaceMatcher = namespacePattern.matcher(line);
+ if (namespaceMatcher.matches()) {
+ parseNamespaceLine(namespaceMatcher.group(2));
+ nd.addNormalizedLine(line);
+ return;
+ }
+ // Only add lines that are not version, namespace or comment lines
+ nd.addNormalizedLine(line);
+ DefLine defLine = new DefLine(line);
+ root.setLeaf(root.getName() + "." + defLine.getName(), defLine, comment);
+ comment = "";
+ }
+
+ private void parseCommentLine(Matcher commentMatch) {
+ if (!comment.isEmpty()) comment += "\n";
+ String addition = commentMatch.group(1);
+ if (addition.isEmpty()) addition = " ";
+ comment += addition;
+ }
+
+ private void parseVersionLine(Matcher matcher) {
+ root.setVersion(matcher.group(2));
+ root.setComment(comment);
+ comment = "";
+ }
+
+ private void parseNamespaceLine(String namespace) {
+ if (namespace.startsWith("com.yahoo."))
+ throw new IllegalArgumentException("Remove 'com.yahoo.' from the namespace '" + namespace +
+ "' - it will be automatically added to the java package name.");
+ root.setNamespace(namespace);
+ root.setComment(comment);
+ comment = "";
+ }
+
+ void parseLines(CNode root, List<String> defLines, NormalizedDefinition nd) throws DefParserException {
+ DefParserException failure = null;
+ int lineNumber = 1;
+ for (String line : defLines) {
+ try {
+ parseLine(root, line, nd);
+ lineNumber++;
+ } catch (IllegalArgumentException e) {
+ String msg = "Error when parsing line " + lineNumber + ": " + line + "\n" + e.getMessage();
+ failure = new DefParserException(msg, e);
+ break;
+ }
+ }
+
+ if (failure != null) {
+ throw (failure);
+ }
+ }
+
+ public NormalizedDefinition getNormalizedDefinition() {
+ return normalizedDefinition;
+ }
+
+ /**
+ * For debugging - dump the tree from the given root to System.out.
+ */
+ public static void dumpTree(CNode root, String indent) {
+ StringBuilder sb = new StringBuilder(indent + root.getName());
+ if (root instanceof LeafCNode) {
+ LeafCNode leaf = ((LeafCNode)root);
+ if (leaf.getDefaultValue() != null) {
+ sb.append(" = ").append(((LeafCNode)root).getDefaultValue().getValue());
+ }
+ }
+ System.out.println(sb.toString());
+ if (!root.getComment().isEmpty()) {
+ String comment = root.getComment();
+ if (comment.contains("\n")) {
+ comment = comment.substring(0, comment.indexOf("\n")) + "...";
+ }
+ if (comment.length() > 60) {
+ comment = comment.substring(0, 57) + "...";
+ }
+ System.out.println(indent + " comment: " + comment);
+ }
+ CNode[] children = root.getChildren();
+ for (CNode c : children) {
+ dumpTree(c, indent + " ");
+ }
+
+ }
+
+ class DefParserException extends Exception {
+ DefParserException(String s, Throwable cause) {
+ super(s, cause);
+ }
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java
new file mode 100644
index 00000000000..8518ff04573
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+/**
+ * An immutable class representing a default value of a config variable
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class DefaultValue {
+
+ private String value = null;
+
+ // The variable type. Always set UNLESS the value is null.
+ private DefLine.Type type = null;
+
+
+ /**
+ * Null value
+ */
+ public DefaultValue() {
+ }
+
+ /**
+ * A default value with the given value and type.
+ */
+ public DefaultValue(String value, DefLine.Type type) {
+ this.value = value;
+ this.type = type;
+ }
+
+ /**
+ * Returns the toString of the default value.
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * Returns the string representation of this value
+ */
+ public String getStringRepresentation() {
+ if (value == null)
+ return "null";
+ else if ("bool".equals(type.getName()))
+ return value;
+ else if ("int".equals(type.getName()))
+ return value;
+ else if ("long".equals(type.getName()))
+ return value;
+ else if ("double".equals(type.getName()))
+ return value;
+ else if ("enum".equals(type.getName()))
+ return value;
+ else {
+ // building a string, do unicode-escaping
+ StringBuilder sb = new StringBuilder();
+ for (char c : value.toCharArray()) {
+ if (c > '\u007f') {
+ sb.append(String.format("\\u%04X", (int) c));
+ } else {
+ sb.append(c);
+ }
+ }
+ return "\"" + sb.toString() + "\"";
+ }
+ }
+
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java
new file mode 100644
index 00000000000..b7803fc8c22
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+/**
+ * Represents an inner node in the configuration tree.
+ *
+ * @author gjoranv
+ */
+public class InnerCNode extends CNode {
+
+ /**
+ * The children of this Node. Mapped using their short name as
+ * string. This variable is null only if Node is a leaf Node.
+ */
+ private final Map<String, CNode> children = new LinkedHashMap<String, CNode>();
+ private boolean restart = false;
+
+ /**
+ * Constructor for the root node.
+ */
+ public InnerCNode(String name) {
+ super(null, name.split("\\.def")[0]);
+ defName = this.name;
+ }
+
+ /**
+ * Constructor for inner nodes.
+ */
+ private InnerCNode(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public CNode[] getChildren() {
+ CNode[] ret = new CNode[children.size()];
+ children.values().toArray(ret);
+ return ret;
+ }
+
+ /**
+ * Access to children for testing
+ * @return modifiable children map
+ */
+ public Map<String, CNode> children() {
+ return children;
+ }
+
+ @Override
+ public CNode getChild(String name) {
+ return children.get(name);
+ }
+
+ /**
+ * Returns and eventually creates the given subnode.
+ */
+ private CNode createOrGetChild(DefLine.Type type, String name) throws IllegalArgumentException {
+ String key = name;
+ int split = name.indexOf('.');
+ CNode newChild;
+ if (split != -1) {
+ key = name.substring(0, split).trim();
+ newChild = new InnerCNode(this, key);
+ } else {
+ newChild = LeafCNode.newInstance(type, this, key);
+ if (newChild == null)
+ throw new IllegalArgumentException
+ ("Could not create " + type.name + " " + name);
+ }
+ return children.containsKey(newChild.getName())
+ ? children.get(newChild.getName())
+ : newChild;
+ }
+
+ /**
+ * Adds a child to this node with the given type, name and value. Necessary children on the path
+ * to the given leaf node will be added as well.
+ * @param name the full name/path of the node to add.
+ * @param defLine the parsed .def-file line to add.
+ * @param comment comment extracted from the .def-file.
+ */
+ @Override
+ protected void setLeaf(String name, DefLine defLine, String comment)
+ throws IllegalArgumentException {
+ if (name.indexOf('.') < 0) {
+ throw new IllegalArgumentException("Parameter with name '" + name +
+ "' cannot be a leaf node as it has already been declared as an inner node.");
+ }
+ checkMyName(name.substring(0, name.indexOf('.')));
+ String childName = name.substring(name.indexOf('.') + 1);
+
+ CNode child = createOrGetChild(defLine.getType(), childName);
+/*
+ System.out.println("\nAdding child name: " + name);
+ System.out.println(" getName: " + child.getName());
+ System.out.println(" full name: " + child.getFullName());
+ System.out.println(" classname: " + child.getClassName());
+ System.out.println(" full classname: " + child.getFullClassName());
+*/
+ restart |= defLine.getRestart();
+ child.setLeaf(childName, defLine, comment);
+ children.put(child.getName(), child);
+ }
+
+ @Override
+ public boolean needRestart() {
+ return restart;
+ }
+
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java
new file mode 100644
index 00000000000..3cd7b61ba55
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java
@@ -0,0 +1,267 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+/**
+ * @author gjoranv
+ */
+public abstract class LeafCNode extends CNode {
+
+ private boolean isInitialized = false;
+ private DefaultValue defaultValue = null;
+ private boolean restart = false;
+
+ /**
+ * Constructor for the leaf nodes.
+ */
+ protected LeafCNode(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name) {
+ try {
+ switch (type.name) {
+ case "int":
+ return new IntegerLeaf(parent, name);
+ case "long":
+ return new LongLeaf(parent, name);
+ case "double":
+ return new DoubleLeaf(parent, name);
+ case "bool":
+ return new BooleanLeaf(parent, name);
+ case "string":
+ return new StringLeaf(parent, name);
+ case "reference":
+ return new ReferenceLeaf(parent, name);
+ case "file":
+ return new FileLeaf(parent, name);
+ case "path":
+ return new PathLeaf(parent, name);
+ case "enum":
+ return new EnumLeaf(parent, name, type.enumArray);
+ default:
+ return null;
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name, String defVal) {
+ LeafCNode ret = newInstance(type, parent, name);
+ if (defVal!=null) {
+ DefaultValue def = new DefaultValue(defVal, type);
+ ret.setDefaultValue(def);
+ }
+ return ret;
+ }
+
+ public abstract String getType();
+
+ @Override
+ public CNode[] getChildren() {
+ return new CNode[0];
+ }
+
+ @Override
+ public CNode getChild(String name) {
+ return null;
+ }
+
+ public DefaultValue getDefaultValue() {
+ return defaultValue;
+ }
+
+ public LeafCNode setDefaultValue(DefaultValue defaultValue) {
+ this.defaultValue = defaultValue;
+ return this;
+ }
+
+ /**
+ * @param defaultValue The value to check.
+ * @throws IllegalArgumentException if the value is illegal according to the node type.
+ */
+ public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException {
+ }
+
+ @Override
+ protected void setLeaf(String name, DefLine defLine, String comment)
+ throws IllegalArgumentException {
+ DefLine.Type type = defLine.getType();
+ // TODO: why the !is... conditions?
+ if (!isMap && !isArray && isInitialized) {
+ throw new IllegalArgumentException(name + " is already defined");
+ }
+ isInitialized = true;
+ checkMyName(name);
+ if (!type.name.equalsIgnoreCase(getType())) {
+ throw new IllegalArgumentException("Type " + type.name + " does not match " + getType());
+ }
+ setValue(defLine.getDefault());
+ setComment(comment);
+ restart |= defLine.getRestart();
+ }
+
+ @Override
+ public boolean needRestart() {
+ return restart;
+ }
+
+ public final void setValue(DefaultValue defaultValue) throws IllegalArgumentException {
+ try {
+ checkDefaultValue(defaultValue);
+ setDefaultValue(defaultValue);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException
+ ("Invalid default value", e);
+ }
+ }
+
+ /**
+ * Superclass for leaf nodes that should not generate class.
+ */
+ public static abstract class NoClassLeafCNode extends LeafCNode {
+ protected NoClassLeafCNode(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+ }
+
+ /**
+ * Superclass for no-class leaf nodes that cannot have a default.
+ */
+ public static abstract class NoClassNoDefaultLeafCNode extends LeafCNode {
+ protected NoClassNoDefaultLeafCNode(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public LeafCNode setDefaultValue(DefaultValue defaultValue) {
+ if (defaultValue != null)
+ throw new IllegalArgumentException("Parameters of type '" + getType() + "' cannot have a default value.");
+ return this;
+ }
+ }
+
+ public static class IntegerLeaf extends NoClassLeafCNode {
+ protected IntegerLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "int";
+ }
+ }
+
+ public static class LongLeaf extends NoClassLeafCNode {
+ protected LongLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "long";
+ }
+ }
+
+ public static class DoubleLeaf extends NoClassLeafCNode {
+ protected DoubleLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "double";
+ }
+ }
+
+ public static class BooleanLeaf extends NoClassLeafCNode {
+ protected BooleanLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "bool";
+ }
+ }
+
+ public static class StringLeaf extends NoClassLeafCNode {
+ protected StringLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "string";
+ }
+ }
+
+ public static class ReferenceLeaf extends StringLeaf {
+ ReferenceLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "reference";
+ }
+ }
+
+ public static class FileLeaf extends NoClassNoDefaultLeafCNode {
+ FileLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "file";
+ }
+ }
+
+ public static class PathLeaf extends NoClassNoDefaultLeafCNode {
+ PathLeaf(InnerCNode parent, String name) {
+ super(parent, name);
+ }
+
+ @Override
+ public String getType() {
+ return "path";
+ }
+ }
+
+ public static class EnumLeaf extends LeafCNode {
+
+ private final String[] legalValues;
+
+ protected EnumLeaf(InnerCNode parent, String name, String[] valArray) {
+ super(parent, name);
+ this.legalValues = valArray;
+ }
+
+ @Override
+ public String getType() {
+ return "enum";
+ }
+
+ /** @return This enum's legal values. */
+ public String[] getLegalValues() {
+ return legalValues;
+ }
+
+ @Override
+ public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException {
+ if ((defaultValue != null) && (defaultValue.getValue() != null)) {
+ String defaultString = null;
+ String value = defaultValue.getValue();
+ for (String val : legalValues) {
+ if (value.equals(val)) {
+ defaultString = val;
+ }
+ }
+ if (defaultString == null)
+ throw new IllegalArgumentException("Could not initialize enum with: " + value);
+ }
+ }
+ }
+
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java
new file mode 100644
index 00000000000..f28549275ef
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.*;
+import java.util.logging.Logger;
+
+/**
+ * This class generates code for a config class from a given def-file.
+ */
+public class MakeConfig {
+
+ private final static Logger log = Logger.getLogger(MakeConfig.class.getName());
+
+ private final ClassBuilder classBuilder;
+
+ public MakeConfig(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties properties) {
+ classBuilder = createClassBuilder(root, nd, path, properties);
+ }
+
+ public static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties prop) {
+ if (prop.language.equals("cppng") || prop.language.equals("cpp"))
+ return new CppClassBuilder(root, nd, prop.destDir, prop.dirInRoot);
+ else
+ return new JavaClassBuilder(root, nd, prop.destDir);
+ }
+
+ /**
+ * Generates the code and print it to this.out.
+ */
+ void buildClasses() {
+ classBuilder.createConfigClasses();
+ }
+
+ private static void printUsage(PrintStream out) {
+ out.println("Usage: java -Dconfig.dest=<dir> -Dconfig.spec=<path> [-Dconfig.lang=cpp -Dconfig.subdir=<dir>] [-Dconfig.dumpTree=true] MakeConfig");
+ out.println(" (default language for generated code is Java)");
+ }
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ try {
+ MakeConfigProperties props = new MakeConfigProperties();
+ for (File specFile : props.specFiles) {
+ String path = specFile.toURI().toString();
+ String name = specFile.getName();
+ if (name.endsWith(".def")) name = name.substring(0, name.length() - 4);
+ DefParser parser = new DefParser(name, new FileReader(specFile));
+ InnerCNode configRoot = parser.getTree();
+ checkNamespace(name, configRoot);
+ if (configRoot != null) {
+ MakeConfig mc = new MakeConfig(configRoot, parser.getNormalizedDefinition(), path, props);
+ mc.buildClasses();
+ if (props.dumpTree) {
+ System.out.println("\nTree dump:");
+ DefParser.dumpTree(configRoot, "");
+ }
+ } else {
+ System.exit(1);
+ }
+ }
+ } catch (PropertyException e) {
+ System.out.println(Exceptions.toMessageString(e));
+ printUsage(System.err);
+ System.exit(1);
+ } catch (CodegenRuntimeException e) {
+ System.out.println(Exceptions.toMessageString(e));
+ System.exit(1);
+ }
+ }
+
+ private static void checkNamespace(String name, InnerCNode configRoot) {
+ if (configRoot.defNamespace == null)
+ throw new IllegalArgumentException("In config definition '" + name + "': A namespace is required");
+ }
+
+ // The Exceptions class below is copied from vespajlib/com.yahoo.protect.Exceptions
+
+ /**
+ * Helper methods for handling exceptions
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+ static class Exceptions {
+
+ /**
+ * <p>Returns a use friendly error message string which includes information from all nested exceptions.
+ *
+ * <p>The form of this string is
+ * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code>
+ * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages.
+ */
+ public static String toMessageString(Throwable t) {
+ StringBuilder b = new StringBuilder();
+ String lastMessage = null;
+ String message;
+ for (; t != null; t = t.getCause(), lastMessage = message) {
+ message = getMessage(t);
+ if (message == null) continue;
+ if (lastMessage != null && lastMessage.equals(message)) continue;
+ if (b.length() > 0)
+ b.append(": ");
+ b.append(message);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Returns a useful message from *this* exception, or null if none
+ */
+ private static String getMessage(Throwable t) {
+ String message = t.getMessage();
+ if (t.getCause() == null) {
+ if (message == null) return toShortClassName(t);
+ return message;
+ } else {
+ if (message == null) return null;
+ if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null;
+ return message;
+ }
+ }
+
+ private static String toShortClassName(Object o) {
+ String longName = o.getClass().getName();
+ int lastDot = longName.lastIndexOf(".");
+ if (lastDot < 0) return longName;
+ return longName.substring(lastDot + 1);
+ }
+ }
+}
+
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java
new file mode 100644
index 00000000000..d055a1fe1fb
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.StringTokenizer;
+
+/**
+ * Encapsulates data extracted from system properties.
+ *
+ * @author <a href="gv@yahoo-inc.com">Gjoran Voldengen</a>
+ */
+public class MakeConfigProperties {
+
+ private final List<String> legalLanguages = Arrays.asList("java", "cpp", "cppng" );
+
+ final File destDir;
+ final File[] specFiles;
+ final String language;
+ final String dirInRoot; // Where within fileroot to store generated class files
+ final boolean dumpTree;
+ final boolean generateFrameworkCode;
+
+ MakeConfigProperties() throws PropertyException {
+ destDir = checkDestinationDir();
+ specFiles = checkSpecificationFiles();
+ language = checkLanguage();
+ dirInRoot = checkDirInRoot();
+ dumpTree = System.getProperty("config.dumpTree") != null &&
+ System.getProperty("config.dumpTree").equalsIgnoreCase("true");
+ generateFrameworkCode = System.getProperty("config.useFramework") == null ||
+ System.getProperty("config.useFramework").equalsIgnoreCase("true");
+ }
+
+ private File checkDestinationDir() throws PropertyException {
+ String destination = System.getProperty("config.dest");
+ if (destination == null)
+ throw new PropertyException("Missing property: config.dest.");
+
+ File dir = new File(destination);
+ if (!dir.isDirectory()) {
+ throw new PropertyException("Could not find directory: " + dir.getPath());
+ }
+ return dir;
+ }
+
+ private String checkDirInRoot() throws PropertyException {
+ String dirInRoot = System.getProperty("config.subdir");
+ // Optional parameter
+ if (dirInRoot == null) { return null; }
+ File f = new File(destDir, dirInRoot);
+ if (!f.isDirectory()) {
+ throw new PropertyException("Could not find directory: " + f.getPath());
+ }
+ return dirInRoot;
+ }
+
+ /**
+ * @return Desired programming language of generated code, default is "java".
+ * @throws PropertyException if supplied language is not a legal language.
+ */
+ private String checkLanguage() throws PropertyException {
+ String inputLang = System.getProperty("config.lang", "java").toLowerCase();
+ if (! legalLanguages.contains(inputLang)) {
+ throw new PropertyException
+ ("Unsupported code language: '" + inputLang + "'. Supported languages are: " + legalLanguages);
+ }
+ return inputLang;
+ }
+
+ private static File[] checkSpecificationFiles() throws PropertyException {
+ String string = System.getProperty("config.spec");
+ if (string == null)
+ throw new PropertyException("Missing property: config.spec.");
+
+ StringTokenizer st = new StringTokenizer(string, ",");
+ if (st.countTokens() == 0)
+ throw new PropertyException("Missing property: config.spec.");
+
+ File[] files = new File[st.countTokens()];
+ for (int i = 0; st.hasMoreElements(); i++) {
+ files[i] = new File((String) st.nextElement());
+ if (!files[i].isFile())
+ throw new PropertyException("Could not read file " + files[i].getPath());
+ }
+ return files;
+ }
+
+}
+
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java
new file mode 100644
index 00000000000..1847150d86d
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java
@@ -0,0 +1,200 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.text.DecimalFormat;
+import java.security.MessageDigest;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+/**
+ *
+ * Does normalizing (removing comments, trimming whitespace etc.) and calculation of md5sum
+ * of config definitions
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class NormalizedDefinition {
+ /* Patterns used for finding ranges in config definitions */
+ private static final Pattern intPattern = Pattern.compile(".*int.*range.*");
+ private static final Pattern doublePattern = Pattern.compile(".*double.*range.*");
+ private MessageDigest md5;
+
+ String defMd5 = null;
+ List<String> normalizedContent = null;
+
+ public NormalizedDefinition() {
+ try {
+ md5 = MessageDigest.getInstance("MD5");
+ } catch (java.security.NoSuchAlgorithmException e) {
+ throw new RuntimeException("Unable to create MD5 digest", e);
+ }
+ normalizedContent = new ArrayList<>();
+ }
+
+ public NormalizedDefinition normalize(BufferedReader reader) throws IOException {
+ String s;
+ List<String> input = new ArrayList<>();
+ while ((s = reader.readLine()) != null) {
+ String normalized = normalize(s);
+ if (normalized.length() > 0) {
+ input.add(normalized);
+ }
+ }
+ normalizedContent = input;
+ return this;
+ }
+
+ /**
+ * Normalizes a config definition line. Each string is normalized according to the
+ * rules of config and definition files before they are used:
+ * <ul>
+ * <li>Remove trailing space.<li>
+ * <li>Remove trailing comments, and spaces before trailing comments.</li>
+ * <li>Remove empty lines</li>
+ * <li>Keep comment lines</li>
+ * </ul>
+ * The supplied list is changed in-place
+ *
+ * @param line A config definition line
+ * @return a normalized config definition line
+ */
+ public static String normalize(String line) {
+ //System.out.println("before line=" + line + ";");
+ // Normalize line
+ line = line.trim();
+ Matcher m = intPattern.matcher(line);
+ if (m.matches()) {
+ String formattedMax = new DecimalFormat("#.#").format(0x7fffffff);
+ String formattedMin = new DecimalFormat("#.#").format(-0x80000000);
+ line = line.replaceFirst("\\[,", "["+formattedMin+",");
+ line = line.replaceFirst(",\\]", ","+formattedMax+"]");
+ }
+ m = doublePattern.matcher(line);
+ if (m.matches()) {
+ String formattedMax = new DecimalFormat("#.#").format(1e308);
+ String formattedMin = new DecimalFormat("#.#").format(-1e308);
+ line = line.replaceFirst("\\[,", "["+formattedMin+",");
+ line = line.replaceFirst(",\\]", ","+formattedMax+"]");
+ }
+ line = removeComment(line);
+ if (!line.isEmpty()) {
+ line = stripSpaces(line);
+ line = line.replaceAll("\\s,", ","); // Remove space before comma (for enums)
+ line += "\n";
+ }
+ //System.out.println("after line=" + line + ";");
+ return line;
+ }
+
+ // Removes comment char and text after it, unless comment char is inside a string
+ // Keeps comment lines (lines that start with #)
+ private static String removeComment(String line) {
+ int index = line.indexOf("#");
+ if (!line.contains("#") || index == 0) return line;
+
+ int firstQuote = line.indexOf("\"");
+ if (firstQuote > 0) {
+ int secondQuote = line.indexOf("\"", firstQuote + 1);
+ if (index > secondQuote) {
+ line = line.substring(0, index);
+ line = line.trim();
+ }
+ } else {
+ line = line.substring(0, index);
+ line = line.trim();
+ }
+
+ return line;
+ }
+
+ public void addNormalizedLine(String line) {
+ normalizedContent.add(line);
+ }
+
+ public String generateMd5Sum() {
+ for (String line : normalizedContent) {
+ String s = normalize(line);
+ if (!s.isEmpty()) {
+ md5.update(toBytes(s));
+ }
+ }
+ defMd5 = toHexString(md5.digest()).toLowerCase();
+ //System.out.println("md5=" + defMd5) ;
+ return defMd5;
+ }
+
+
+ // The two methods below are copied from vespajlib (com.yahoo.text.Utf8 and com.yahoo.io.HexDump)
+ // since configgen cannot depend on any other modules (at least not as it is done now)
+ public static byte[] toBytes(String str) {
+ Charset charset = Charset.forName("utf-8");
+
+ ByteBuffer b = charset.encode(str);
+ byte[] result = new byte[b.remaining()];
+ b.get(result);
+ return result;
+ }
+
+ private String toHexString(byte[] bytes) {
+ StringBuilder sb = new StringBuilder(bytes.length * 2);
+ for (byte aByte : bytes) {
+ sb.append(String.format("%02x", aByte));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing;
+ * @param str String to strip spaces from
+ * @return String with spaces stripped
+ */
+ public static String stripSpaces(String str) {
+ StringBuilder ret = new StringBuilder("");
+ boolean inQuotes = false;
+ boolean inSpaceSequence = false;
+ for (char c : str.toCharArray()) {
+ if (Character.isWhitespace(c)) {
+ if (inQuotes) {
+ ret.append(c);
+ continue;
+ }
+ if (!inSpaceSequence) {
+ // start of space sequence
+ inSpaceSequence=true;
+ ret.append(" ");
+ }
+ } else {
+ if (inSpaceSequence) {
+ inSpaceSequence=false;
+ }
+ if (c=='\"') {
+ inQuotes=!inQuotes;
+ }
+ ret.append(c);
+ }
+ }
+ return ret.toString();
+ }
+
+ public List<String> getNormalizedContent() {
+ return normalizedContent;
+ }
+
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ for (String line : normalizedContent) {
+ builder.append(line.replace("\"", "\\\""));
+ builder.append("\\n\\\n");
+ }
+ return builder.toString();
+ }
+
+ public String getDefMd5() {
+ return defMd5;
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java
new file mode 100644
index 00000000000..a42f76340de
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+public class PropertyException extends Exception {
+ PropertyException(String s) {
+ super(s);
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java
new file mode 100644
index 00000000000..145f2da3245
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.util.HashMap;
+import java.util.regex.Pattern;
+
+/**
+ * Reserved words that cannot be used as variable names in a config definition file.
+ *
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ * @since 2009-06-24
+ */
+
+public class ReservedWords {
+
+ public static final String INTERNAL_PREFIX = "__";
+ final static Pattern internalPrefixPattern = Pattern.compile("^" + INTERNAL_PREFIX + ".*");
+ final static Pattern capitalizedPattern = Pattern.compile("^[A-Z].*");
+
+ private static final String[] cKeywords =
+ {"asm", "auto", "bool", "break", "case", "catch",
+ "char", "class", "const", "const_cast", "continue", "default",
+ "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit",
+ "export", "extern", "false", "float", "for", "friend", "goto", "if",
+ "inline", "int", "item", "long", "mutable", "namespace", "new", "operator",
+ "private", "protected", "public", "register", "reinterpret_cast",
+ "return", "short", "signed", "sizeof", "static", "static_cast",
+ "struct", "switch", "template", "this", "throw", "true", "try",
+ "typedef", "typeid", "typename", "union", "unsigned",
+ "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor",
+ "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq",
+ "bitand"};
+
+ private static final String[] javaKeywords =
+ {"abstract", "boolean", "break", "byte", "case",
+ "catch", "char", "class","continue", "default", "do", "double",
+ "else", "extends","false", "final", "finally", "float", "for",
+ "if","implements", "import", "instanceof", "int", "interface",
+ "item", "long","native", "new", "null", "package", "private",
+ "protected","public", "return", "short", "static",
+ "strictfp","super","switch", "synchronized", "this",
+ "throw","throws","transient", "true", "try", "void",
+ "volatile","while", "byvalue", "cast", "const", "future",
+ "generic","goto", "inner", "operator", "outer", "rest", "var"};
+
+ private static final HashMap<String, String> allKeyWords;
+
+ static {
+ allKeyWords = new HashMap<String, String>();
+ for (String s : cKeywords) {
+ allKeyWords.put(s, "C");
+ }
+ for (String s : javaKeywords) {
+ if (allKeyWords.containsKey(s)) {
+ allKeyWords.put(s, "C and Java");
+ } else {
+ allKeyWords.put(s, "Java");
+ }
+ }
+ }
+
+
+ public static boolean isReservedWord(String word) {
+ return allKeyWords.containsKey(word);
+ }
+
+ public static String getLanguageForReservedWord(String word) {
+ return allKeyWords.get(word);
+ }
+
+
+}