diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /configgen/src/main/java/com/yahoo/config/codegen |
Publish
Diffstat (limited to 'configgen/src/main/java/com/yahoo/config/codegen')
14 files changed, 2749 insertions, 0 deletions
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CNode.java b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java new file mode 100644 index 00000000000..02fec548b40 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.StringTokenizer; + +/** + * Abstract superclass for all nodes representing a definition file. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public abstract class CNode { + + public static final String DEFAULT_NAMESPACE = "config"; + + // TODO: replace by "type" enum + public final boolean isArray; + public final boolean isMap; + final String name; + final InnerCNode parent; + + // TODO: remove! Only set for the root node, and root.getName() returns the same thing! + String defName = null; + String defVersion = ""; + String defNamespace = null; + String defMd5 = "MISSING MD5"; + String comment = ""; + + + protected CNode(InnerCNode parent, String name) { + this.parent = parent; + int bracketIdx = name.indexOf('['); + int curlyIdx = name.indexOf('{'); + if (bracketIdx != -1) { + this.name = name.substring(0, bracketIdx); + isArray = true; + isMap = false; + } else if (curlyIdx != -1) { + this.name = name.substring(0, curlyIdx); + isMap = true; + isArray = false; + } else { + this.name = name; + isMap = false; + isArray = false; + } + } + + /** + * Returns the simple name of this node. + * @return the simple name of this node + */ + public String getName() { + return name; + } + + public InnerCNode getParent() { + return parent; + } + + public abstract CNode[] getChildren(); + + public abstract CNode getChild(String name); + + public String getMd5() { + return defMd5; + } + + void setMd5(String md5) { + defMd5 = md5; + } + + public String getVersion() { + return defVersion; + } + + void setVersion(String version) { + defVersion = version; + } + + public String getNamespace() { + return defNamespace; + } + + void setNamespace(String namespace) { + defNamespace = namespace; + } + + public String getComment() { + return comment; + } + + void setComment(String comment) { + this.comment = comment; + } + + protected abstract void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException; + + public abstract boolean needRestart(); + + protected void checkMyName(String myName) throws IllegalArgumentException { + if (isArray) { + int n1 = myName.indexOf('['); + int n2 = myName.indexOf(']'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid array syntax: " + myName); + myName = myName.substring(0, n1); + } else if (isMap) { + int n1 = myName.indexOf('{'); + int n2 = myName.indexOf('}'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid map syntax: " + myName); + myName = myName.substring(0, n1); + } else if (myName.contains("[]")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-array type."); + } else if (myName.contains("{}")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-map type."); + } + if (!myName.equals(getName())) + throw new IllegalArgumentException(myName + " does not match " + getName() + "."); + } + + /** + * @return the full name as a config path of this node. + */ + public String getFullName() { + StringBuilder buf = new StringBuilder(); + if (parent != null) + buf.append(parent.getFullName()); + if (buf.length() > 0) + buf.append('.'); + StringBuilder theName = new StringBuilder(this.getName()); + + if (isArray) theName.append("[]"); + else if (isMap) theName.append("{}"); + + return buf.append(theName).toString(); + } + + /** + * @param prefix The prefix to use, usually an indent (spaces) followed by either '*' or "//" + * @return a comment block where each line is prefixed, but the caller must close it if using '*'. + */ + public String getCommentBlock(String prefix) { + prefix = prefix + " "; + StringBuilder ret = new StringBuilder(); + if (getComment().length() > 0) { + StringTokenizer st = new StringTokenizer(getComment(), "\n"); + while (st.hasMoreTokens()) { + ret.append(prefix).append(st.nextToken()).append("\n"); + } + } + return ret.toString(); + } + + @Override + public String toString() { + return getNamespace()+"."+getName()+","+getVersion(); + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java new file mode 100644 index 00000000000..dab2ce98510 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public interface ClassBuilder { + + /** + * Generate config class file(s). + */ + public void createConfigClasses(); +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java new file mode 100644 index 00000000000..56a87c0506e --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * This exception is thrown on internal errors. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class CodegenRuntimeException extends RuntimeException { + public CodegenRuntimeException(String s, Throwable cause) { + super(s, cause); + } + + public CodegenRuntimeException(Throwable cause) { + super(cause); + } + + public CodegenRuntimeException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java new file mode 100644 index 00000000000..e6d7f29ad36 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -0,0 +1,1117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; + + +/** + * This class autogenerates C++ code for the C++ config, based on a CNode tree given. + */ +public class CppClassBuilder implements ClassBuilder { + private final CNode root; + private final NormalizedDefinition nd; + private final File rootDir; + private final String relativePathUnderRoot; + private static final Map<String, String> vectorTypeDefs; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "BoolVector"); + map.put("int32_t", "IntVector"); + map.put("int64_t", "LongVector"); + map.put("double", "DoubleVector"); + map.put("vespalib::string", "StringVector"); + vectorTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> mapTypeDefs; + static { + Map<String, String> map = new HashMap<>(); + map.put("bool", "BoolMap"); + map.put("int32_t", "IntMap"); + map.put("int64_t", "LongMap"); + map.put("double", "DoubleMap"); + map.put("vespalib::string", "StringMap"); + mapTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> slimeTypeMap; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "Bool"); + map.put("int", "Long"); + map.put("long", "Long"); + map.put("double", "Double"); + map.put("string", "String"); + map.put("enum", "String"); + map.put("file", "String"); + map.put("reference", "String"); + slimeTypeMap = Collections.unmodifiableMap(map); + } + + public CppClassBuilder(CNode root, NormalizedDefinition nd, File rootDir, String relativePathUnderRoot) { + this.root = root; + this.nd = nd; + this.rootDir = rootDir; + this.relativePathUnderRoot = relativePathUnderRoot; + } + + public void createConfigClasses() { + generateConfig(root, nd); + } + + String readFile(File f) throws IOException { + if (!f.isFile()) return null; + StringBuilder sb = new StringBuilder(); + BufferedReader sr = new BufferedReader(new FileReader(f)); + while (true) { + String line = sr.readLine(); + if (line == null) break; + sb.append(line).append("\n"); + } + return sb.toString(); + } + + void writeFile(File f, String content) throws IOException { + FileWriter fw = new FileWriter(f); + fw.write(content); + fw.close(); + } + + void generateConfig(CNode root, NormalizedDefinition nd) { + try{ + StringWriter headerWriter = new StringWriter(); + StringWriter bodyWriter = new StringWriter(); + writeHeaderFile(headerWriter, root); + writeBodyFile(bodyWriter, root, relativePathUnderRoot, nd); + + String newHeader = headerWriter.toString(); + String newBody = bodyWriter.toString(); + + File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h")); + File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp")); + + String oldHeader = readFile(headerFile); + String oldBody = readFile(bodyFile); + + if (oldHeader == null || !oldHeader.equals(newHeader)) { + writeFile(headerFile, newHeader); + } + if (oldBody == null || !oldBody.equals(newBody)) { + writeFile(bodyFile, newBody); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + String getFileName(CNode node, String extension) { + return "config-" + node.getName() + "." + extension; + } + + static String removeDashesAndUpperCaseAllFirstChars(String source, boolean capitalizeFirst) { + // Create upper case chars after each dash + String parts[] = source.split("[-_]"); + StringBuilder sb = new StringBuilder(); + for (String s : parts) { + sb.append(s.substring(0, 1).toUpperCase()).append(s.substring(1)); + } + String result = sb.toString(); + if (!capitalizeFirst) { + result = result.substring(0,1).toLowerCase() + result.substring(1); + } + return result; + } + + /** Convert name of type to the name we want to use in macro ifdefs in file. */ + String getDefineName(String name) { + return name.toUpperCase().replace("-", ""); + } + + /** Convert name of type to the name we want to use as type name in the generated code. */ + static String getTypeName(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, true); + } + + /** Convert name of an identifier from value in def file to name to use in C++ file. */ + String getIdentifier(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, false); + } + + void writeHeaderFile(Writer w, CNode root) throws IOException { + writeHeaderHeader(w, root); + writeHeaderPublic(w, root); + writeHeaderFooter(w, root); + } + + void writeHeaderPublic(Writer w, CNode root) throws IOException { + w.write("public:\n"); + writeHeaderTypeDefs(w, root, " "); + writeTypeDeclarations(w, root, " "); + writeHeaderFunctionDeclarations(w, getTypeName(root, false), root, " "); + writeStaticMemberDeclarations(w, " "); + writeMembers(w, root, " "); + } + + String [] generateCppNameSpace(CNode root) { + String namespace = root.getNamespace(); + if (namespace.contains(".")) { + return namespace.split("\\."); + } + return new String[]{namespace}; + } + + String generateCppNameSpaceString(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i]); + str.append("::"); + } + str.append(namespaceList[namespaceList.length - 1]); + return str.toString(); + } + + String generateCppNameSpaceDefine(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i].toUpperCase()); + str.append("_"); + } + str.append(namespaceList[namespaceList.length - 1].toUpperCase()); + return str.toString(); + } + + void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("namespace " + namespaceList[i] + " {\n\n"); + } + } + + void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("} // namespace " + namespaceList[i] + "\n\n"); + } + } + + void writeHeaderHeader(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespacePrint = generateCppNameSpaceString(namespaceList); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + w.write("" + + "/**\n" + + " * @class " + namespacePrint + "::" + className + "\n" + + " * @ingroup config\n" + + " *\n" + + " * @brief This is an autogenerated class for handling VESPA config.\n" + + " *\n" + + " * This class is autogenerated by vespa from a config definition file.\n" + + " * To subscribe to config, you need to include the config/config.h header, \n" + + " * and create a ConfigSubscriber in order to subscribe for config.\n" + ); + if (root.getComment().length() > 0) { + w.write(" *\n"); + StringTokenizer st = new StringTokenizer(root.getComment(), "\n"); + while (st.hasMoreTokens()) { + w.write(" * " + st.nextToken() + "\n"); + } + } + w.write("" + + " */\n" + + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n" + + "#define CLOUD_CONFIG_" + defineName + "_H\n" + + "\n" + + "#include <vespa/config/common/configvalue.h>\n" + + "#include <vespa/config/configgen/configpayload.h>\n" + + "#include <vespa/config/configgen/configinstance.h>\n" + + "#include <vespa/config/print/configdatabuffer.h>\n" + + "#include <vespa/vespalib/stllike/string.h>\n" + + "#include <vector>\n" + + "#include <map>\n" + + "\n"); + writeNameSpaceBegin(w, namespaceList); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("" + + "/**\n" + + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n" + + " * versions after this class declaration.\n" + + " */\n" + + "class Internal" + className + "Type : public ::config::ConfigInstance\n" + + "{\n" + ); + + } + + + + void writeTypeDeclarations(Writer w, CNode node, String indent) throws IOException { + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(indent + "enum " + typeName + " { "); + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + for (int i=0; i<leaf.getLegalValues().length; ++i) { + if (i != 0) { + w.write(", "); + } + w.write(leaf.getLegalValues()[i]); + } + w.write(" };\n" + + indent + "typedef std::vector<" + typeName + "> " + + typeName + "Vector;" + + "\n" + + indent + "typedef std::map<vespalib::string, " + typeName + "> " + + typeName + "Map;" + + "\n" + + indent + "static " + typeName + " get" + typeName + "(const vespalib::string&);\n" + + indent + "static vespalib::string get" + typeName + "Name(" + typeName + " e);\n" + + "\n" + ); + w.write(indent + "struct Internal" + typeName + "Converter {\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault);\n"); + w.write(indent + "};\n"); + } else { + w.write(indent + "class " + typeName + " {\n"); + w.write(indent + "public:\n"); + writeTypeDeclarations(w, child, indent + " "); + writeStructFunctionDeclarations(w, getTypeName(child, false), child, indent + " "); + writeMembers(w, child, indent + " "); + w.write(indent + "};\n"); + w.write(indent + "typedef std::vector<" + typeName + "> " + typeName + "Vector;\n\n"); + w.write(indent + "typedef std::map<vespalib::string, " + typeName + "> " + typeName + "Map;\n\n"); + } + } + } + } + + void writeHeaderFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + "const vespalib::string & defName() const { return CONFIG_DEF_NAME; }\n" + + indent + "const vespalib::string & defVersion() const { return CONFIG_DEF_VERSION; }\n" + + indent + "const vespalib::string & defMd5() const { return CONFIG_DEF_MD5; }\n" + + indent + "const vespalib::string & defNamespace() const { return CONFIG_DEF_NAMESPACE; }\n" + + indent + "void serialize(::config::ConfigDataBuffer & __buffer) const;\n"); + writeConfigClassFunctionDeclarations(w, "Internal" + className + "Type", node, indent); + } + + void writeConfigClassFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const ::config::ConfigValue & __value);\n"); + w.write(indent + className + "(const ::config::ConfigDataBuffer & __value);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + } + + void writeStructFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const std::vector<vespalib::string> & __lines);\n"); + w.write(indent + className + "(const vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + w.write(indent + "void serialize(vespalib::slime::Cursor & __cursor) const;\n"); + } + + void writeClassCopyConstructorDeclaration(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const " + className + " & __rhs);\n"); + } + + + void writeClassCopyConstructorDefinitionCommon(Writer w, CNode node) throws IOException { + for (int i = 0; i < node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" " + childName + "(__rhs." + childName + ")"); + } else { + w.write(",\n " + childName + "(__rhs." + childName + ")"); + } + } + } + + void writeConfigClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + w.write(parent + className + "(const " + className + " & __rhs)\n"); + w.write(" : ConfigInstance(),\n "); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + String typeName = getTypeName(node, false); + // Write empty constructor + w.write(parent + typeName + "(const " + typeName + " & __rhs)\n"); + w.write(" :"); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeCommonFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + className + "();\n"); + writeClassCopyConstructorDeclaration(w, className, node, indent); + w.write("" + + "\n" + + indent + "bool operator==(const " + className + "& __rhs) const;\n" + + indent + "bool operator!=(const " + className + "& __rhs) const;\n" + + "\n" + ); + } + + static String getTypeName(CNode node, boolean includeArray) { + String type = null; + if (node instanceof InnerCNode) { + InnerCNode innerNode = (InnerCNode) node; + type = getTypeName(innerNode.getName()); + } else if (node instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) node; + if (leaf.getType().equals("bool")) { + type = "bool"; + } else if (leaf.getType().equals("int")) { + type = "int32_t"; + } else if (leaf.getType().equals("long")) { + type = "int64_t"; + } else if (leaf.getType().equals("double")) { + type = "double"; + } else if (leaf.getType().equals("enum")) { + type = getTypeName(node.getName()); + } else if (leaf.getType().equals("string")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("reference")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("file")) { + type = "vespalib::string"; + } else { + throw new IllegalArgumentException("Unknown leaf datatype " + leaf.getType()); + } + } + if (type == null) { + throw new IllegalArgumentException("Unknown node " + node); + } + if (node.isArray && includeArray) { + if (vectorTypeDefs.containsKey(type)) { + type = vectorTypeDefs.get(type); + } else { + type = type + "Vector"; + } + } else if (node.isMap && includeArray) { + if (mapTypeDefs.containsKey(type)) { + type = mapTypeDefs.get(type); + } else { + type = type + "Map"; + } + } + return type; + } + + void writeStaticMemberDeclarations(Writer w, String indent) throws IOException { + w.write("" + + indent + "static const vespalib::string CONFIG_DEF_MD5;\n" + + indent + "static const vespalib::string CONFIG_DEF_VERSION;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAME;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAMESPACE;\n" + + indent + "static const std::vector<vespalib::string> CONFIG_DEF_SCHEMA;\n" + + indent + "static const int64_t CONFIG_DEF_SERIALIZE_VERSION;\n" + + "\n" + ); + } + + void writeComment(Writer w, String indent, String comment, boolean javadoc) + throws IOException + { + /** If simple one liner comment, write on one line. */ + if (javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 7)) + { + w.write(indent + "/** " + comment + " */\n"); + return; + } else if (!javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 3)) + { + w.write(indent + "// " + comment + "\n"); + return; + } + /** If not we need to write multi line comment. */ + int maxLineLen = 80 - (indent.length() + 3); + if (javadoc) w.write(indent + "/**\n"); + do { + String current; + // Extract first line to write + int newLine = comment.indexOf('\n'); + if (newLine == -1) { + current = comment; + comment = ""; + } else { + current = comment.substring(0, newLine); + comment = comment.substring(newLine + 1); + } + // If line too long, cut it in two + if (current.length() > maxLineLen) { + int spaceIndex = current.lastIndexOf(' ', maxLineLen); + if (spaceIndex >= maxLineLen - 15) { + comment = current.substring(spaceIndex + 1) + + "\n" + comment; + current = current.substring(0, spaceIndex); + } else { + comment = current.substring(maxLineLen) + "\n" + comment; + current = current.substring(0, maxLineLen) + "-"; + } + } + w.write(indent + (javadoc ? " * " : "// ") + current + "\n"); + } while (comment.length() > 0); + if (javadoc) w.write(indent + " */\n"); + } + + void writeMembers(Writer w, CNode node, String indent) throws IOException { + for (CNode child : node.getChildren()) { + String typeName = getTypeName(child, true); + if (child.getComment().length() > 0) { + String comment = child.getComment(); + int index; + do { + index = comment.indexOf("\n\n"); + if (index == -1) break; + String next = comment.substring(0, index); + comment = comment.substring(index + 2); + w.write("\n"); + writeComment(w, indent, next, false); + } while (true); + w.write("\n"); + writeComment(w, indent, comment, true); + } + w.write(indent + typeName + " " + getIdentifier(child.getName()) + ";"); + if (child instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) child; + DefaultValue value = leaf.getDefaultValue(); + if (value != null) { + w.write(" // Default: " + value.getStringRepresentation()); + } + } + w.write("\n"); + } + } + + void writeHeaderTypeDefs(Writer w, CNode root, String indent) throws IOException { + w.write(indent + "typedef std::unique_ptr<const " + getInternalClassName(root) + "> UP;\n"); + for (Map.Entry<String, String> entry : vectorTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String vectorName = entry.getValue(); + String typeDef = "typedef std::vector<" + typeName + "> " + vectorName; + w.write(indent + typeDef + ";\n"); + } + for (Map.Entry<String, String> entry : mapTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String mapName = entry.getValue(); + String typeDef = "typedef std::map<vespalib::string, " + typeName + "> " + mapName; + w.write(indent + typeDef + ";\n"); + } + } + + private static String getInternalClassName(CNode root) { + return "Internal" + getTypeName(root, false) + "Type"; + } + + void writeHeaderFooter(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + + w.write("" + + "};\n" + + "\n" + + "} // namespace internal\n\n"); + + w.write("typedef internal::" + getInternalClassName(root) + " " + className + "ConfigBuilder;\n"); + w.write("typedef const internal::" + getInternalClassName(root) + " " + className + "Config;\n"); + w.write("\n"); + writeNameSpaceEnd(w, namespaceList); + w.write("#endif // VESPA_config_" + defineName + "_H\n"); + } + + void writeBodyFile(Writer w, CNode root, String subdir, NormalizedDefinition nd) throws IOException { + writeBodyHeader(w, root, subdir); + writeStaticMemberDefinitions(w, root, nd); + writeDefinition(w, root, null); + writeBodyFooter(w, root); + } + + void writeBodyHeader(Writer w, CNode root, String subdir) throws IOException { + if (subdir == null) { + w.write("#include \"" + getFileName(root, "h") + "\""); + } else { + w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">"); + } + w.write("\n"); + w.write("#include <set>\n"); + w.write("#include <vespa/config/common/configparser.h>\n"); + w.write("#include <vespa/vespalib/data/slime/convenience.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/config/configgen/vector_inserter.h>\n"); + w.write("#include <vespa/config/configgen/map_inserter.h>\n"); + w.write("\n\n"); + writeNameSpaceBegin(w, generateCppNameSpace(root)); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("using ::config::ConfigParser;\n"); + w.write("using ::config::InvalidConfigException;\n"); + w.write("using ::config::ConfigInstance;\n"); + w.write("using ::config::ConfigValue;\n"); + w.write("using namespace vespalib::slime::convenience;\n"); + w.write("\n"); + } + + void writeStaticMemberDefinitions(Writer w, CNode root, NormalizedDefinition nd) throws IOException { + String typeName = getInternalClassName(root); + w.write("const vespalib::string " + typeName + "::CONFIG_DEF_MD5(\"" + root.defMd5 + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_VERSION(\"" + root.defVersion + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAME(\"" + root.defName + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAMESPACE(\"" + root.getNamespace() + "\");\n" + + "const int64_t " + typeName + "::CONFIG_DEF_SERIALIZE_VERSION(1);\n"); + w.write("const static vespalib::string __internalDefSchema[] = {\n"); + for (String line : nd.getNormalizedContent()) { + w.write("\"" + line.replace("\"", "\\\"") + "\",\n"); + } + w.write("};\n"); + w.write("const std::vector<vespalib::string> " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n"); + w.write(" __internalDefSchema + (sizeof(__internalDefSchema) / \n"); + w.write(" sizeof(__internalDefSchema[0])));\n"); + w.write("\n"); + } + + void writeDefinition(Writer w, CNode node, String parent) throws IOException { + boolean root = false; + if (parent == null) { + parent = getInternalClassName(node) + "::"; + root = true; + } + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + // Definition of getType(string) + w.write(parent + typeName + "\n" + + parent + "get" + typeName + "(const vespalib::string& name)\n" + + "{\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" " + (i != 0 ? "} else " : "")); + w.write("if (name == \"" + leaf.getLegalValues()[i] + "\") {\n" + + " return " + leaf.getLegalValues()[i] + ";\n"); + } + w.write(" } else {\n" + + " throw InvalidConfigException(\"Illegal enum value '\" + name + \"'\");\n" + + " }\n" + + "}\n" + + "\n" + ); + // Definition of getTypeName(enum) + w.write("vespalib::string\n" + + parent + "get" + typeName + "Name(" + typeName + " t)\n" + + "{\n" + + " switch (t) {\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" case " + leaf.getLegalValues()[i] + ": return \"" + leaf.getLegalValues()[i] + "\";\n"); + } + w.write(" default:\n" + + " {\n" + + " vespalib::asciistream ost;\n" + + " ost << \"UNKNOWN(\" << t << \")\";\n" + + " return ost.str();\n" + + " }\n" + + " }\n" + + "}\n" + + "\n" + ); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" throw InvalidConfigException(\"Value for '\" + __fieldName + \"' required but not found\");\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" return __eDefault;\n"); + w.write("}\n\n"); + } else { + writeDefinition(w, child, parent + typeName + "::"); + } + } + } + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + // Write empty constructor + w.write(parent + typeName + "()\n"); + for (int i=0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" : " + childName + "("); + } else { + w.write("),\n " + childName + "("); + } + if (child.isArray || child.isMap) { + // Default array for empty constructor is empty array. + } else if (child instanceof LeafCNode) { // If we have a default value, use that.. + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue() != null) { + w.write(getDefaultValue(leaf)); + } else { + // Defines empty constructor defaults for primitives without default set + if (leaf.getType().equals("bool")) { + w.write("false"); + } else if (leaf.getType().equals("int")) { + w.write("0"); + } else if (leaf.getType().equals("double")) { + w.write("0"); + } else if (leaf.getType().equals("string")) { + } else if (leaf.getType().equals("enum")) { + LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) leaf; + w.write(enumNode.getLegalValues()[0]); + } else if (leaf.getType().equals("reference")) { + } else if (leaf.getType().equals("file")) { + } + } + } + // If we hit neither else, we're an inner node, thus special type that has its own empty constructor + } + if (node.getChildren().length > 0) + w.write(")\n"); + w.write("" + + "{\n" + + "}\n" + + "\n" + ); + // Write copy constructor + if (root) + writeConfigClassCopyConstructorDefinition(w, parent, typeName, node); + else + writeClassCopyConstructorDefinition(w, parent, typeName, node); + + // Write parsing constructor + String indent = " "; + if (root) { + w.write(typeName + "::" + typeName + "(const ConfigValue & __value)\n" + + "{\n" + + indent + "try {\n"); + indent = " "; + w.write(indent + "const std::vector<vespalib::string> & __lines(__value.getLines());\n"); + } else { + w.write(parent + typeName + "(const std::vector<vespalib::string> & __lines)\n" + + "{\n"); + } + w.write("" + + indent + "std::set<vespalib::string> __remainingValuesToParse(" + + "__lines.begin(), __lines.end());\n"); + w.write(indent + "for(std::set<vespalib::string>::iterator __rVTPiter = __remainingValuesToParse.begin();\n" + + indent + " __rVTPiter != __remainingValuesToParse.end();)\n" + + indent + "{\n" + + indent + " if (ConfigParser::stripWhitespace(*__rVTPiter).empty()) {\n" + + indent + " std::set<vespalib::string>::iterator __rVTPiter2 = __rVTPiter++;\n" + + indent + " __remainingValuesToParse.erase(__rVTPiter2);\n" + + indent + " } else {\n" + + indent + " ++__rVTPiter;\n" + + indent + " }\n" + + indent + "}\n"); + for (CNode child : node.getChildren()) { + String childType = getTypeName(child, false); + String childName = getIdentifier(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + if (child.isArray) { + w.write(indent + "std::vector<vespalib::string> " + childName + "__ValueList(\n "); + } else if (child.isMap) { + w.write(indent + "std::map<vespalib::string, vespalib::string> " + childName + "__ValueMap(\n "); + } else { + w.write(indent + childName + " = get" + childType + "("); + } + childType = "vespalib::string"; + } else { + w.write(indent + childName + " = "); + } + if (child.isArray) { + w.write("ConfigParser::parseArray<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else if (child.isMap) { + w.write("ConfigParser::parseMap<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else { + if (child instanceof LeafCNode) { + w.write("ConfigParser::parse<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } else { + w.write("ConfigParser::parseStruct<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue().getValue() != null) { + String defaultVal = getDefaultValue(leaf); + if (leaf instanceof LeafCNode.EnumLeaf) { + defaultVal = '"' + defaultVal + '"'; + } + w.write(", " + defaultVal); + } + } + w.write(")"); + } + if (child instanceof LeafCNode.EnumLeaf) { + childType = getTypeName(child, false); + w.write(");\n"); + if (child.isArray) { + w.write(indent + childName + ".reserve(" + childName + "__ValueList.size());\n" + + indent + "for (std::vector<vespalib::string>::const_iterator __it\n" + + indent + " = " + childName + "__ValueList.begin();\n" + + indent + " __it != " + childName + "__ValueList.end(); ++__it)\n" + + indent + "{\n" + + indent + " " + childName + ".push_back(get" + childType + "(*__it));\n" + + indent + "}\n" + ); + } else if (child.isMap) { + w.write(indent + "typedef std::map<vespalib::string, vespalib::string> __ValueMap;\n"); + w.write(indent + "for (__ValueMap::iterator __it(" + childName + "__ValueMap.begin()), __mt(" + childName + "__ValueMap.end()); __it != __mt; __it++) {\n" + + " " + childName + "[__it->first] = get" + childType + "(__it->second);\n" + + "}\n" + ); + } + } else { + w.write(";\n"); + } + w.write(indent + "ConfigParser::stripLinesForKey(\"" + + child.getName() + "\", " + + "__remainingValuesToParse);\n"); + } + if (root) { + indent = " "; + w.write(indent + "} catch (InvalidConfigException & __ice) {\n"); + w.write(indent + " throw InvalidConfigException(\"Error parsing config '\" + CONFIG_DEF_NAME + \"' in namespace '\" + CONFIG_DEF_NAMESPACE + \"'" + + ": \" + __ice.getMessage());\n" + + indent + "}\n"); + } + w.write("}\n" + + "\n" + ); + // Write operator== + String lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator==(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return (" + ); + for (int i = 0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i != 0) { + w.write(" &&\n "); + } + w.write(childName + " == __rhs." + childName); + } + w.write(");\n" + + "}\n" + + "\n" + ); + // Write operator!= + lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator!=(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return !(operator==(__rhs));\n" + + "}\n" + + "\n" + ); + writeSlimeEncoder(w, node, parent, root); + writeSlimeDecoder(w, node, parent, root); + writeSlimeConstructor(w, node, parent, root); + } + + public void writeSlimeEncoder(Writer w, CNode node, String parent, boolean root) throws IOException + { + String indent = " "; + if (root) { + w.write("void\n" + + parent + "serialize(::config::ConfigDataBuffer & __buffer) const\n" + + "{\n"); + w.write(indent + "vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Cursor & __croot = __slime.setObject();\n"); + w.write(indent + "__croot.setDouble(\"version\", CONFIG_DEF_SERIALIZE_VERSION);\n"); + w.write(indent + "vespalib::slime::Cursor & __key = __croot.setObject(\"configKey\");\n"); + w.write(indent + "__key.setString(\"defName\", vespalib::slime::Memory(CONFIG_DEF_NAME));\n"); + w.write(indent + "__key.setString(\"defNamespace\", vespalib::slime::Memory(CONFIG_DEF_NAMESPACE));\n"); + w.write(indent + "__key.setString(\"defMd5\", vespalib::slime::Memory(CONFIG_DEF_MD5));\n"); + w.write(indent + "vespalib::slime::Cursor & __keySchema =__key.setArray(\"defSchema\");\n"); + w.write(indent + "for (size_t i = 0; i < CONFIG_DEF_SCHEMA.size(); i++) {\n"); + w.write(indent + " __keySchema.addString(vespalib::slime::Memory(CONFIG_DEF_SCHEMA[i]));\n"); + w.write(indent + "}\n"); + w.write(indent + "vespalib::slime::Cursor & __cursor = __croot.setObject(\"configPayload\");\n"); + } else { + w.write("void\n" + + parent + "serialize(vespalib::slime::Cursor & __cursor) const\n" + + "{\n"); + } + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + w.write(indent + "{\n"); + indent = " "; + w.write(indent + "vespalib::slime::Cursor & __c = __cursor.setObject(\"" + child.getName() + "\");\n"); + if (child.isArray) { + w.write(indent + "__c.setString(\"type\", \"array\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + w.write(indent + "for (size_t __i = 0; __i < " + childName + ".size(); __i++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "[__i]));\n"); + } else { + w.write("(\"value\", " + childName + "[__i]);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " " + childName + "[__i].serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "__c.setString(\"type\", \"map\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + String childMapType = getTypeName(child, true); + w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + w.write(indent + " __c3.setString(\"key\", vespalib::slime::Memory(it->first));\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(it->second)));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(it->second));\n"); + } else { + w.write("(\"value\", it->second);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " it->second.serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else { + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + "__c.setString(\"type\", \"enum\");\n"); + w.write(indent + "__c.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + ")));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + "__c.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + "__c.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "));\n"); + } else { + w.write("(\"value\", " + childName + ");\n"); + } + } else { + w.write(indent + "__c.setString(\"type\", \"struct\");\n"); + w.write(indent + "Cursor & __c2 = __c.setObject(\"value\");\n"); + w.write(indent + childName + ".serialize(__c2);\n"); + } + } + indent = " "; + w.write(indent + "}\n"); + + } + w.write("}\n\n"); + } + + public void writeSlimeDecoder(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigDataBuffer & __buffer)\n" + + "{\n"); + w.write(indent + "const vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Inspector & __croot = __slime.get();\n"); + w.write(indent + "vespalib::slime::Inspector & __inspector = __croot[\"configPayload\"];\n"); + } else { + w.write("" + + parent + typeName + "(const vespalib::slime::Inspector & __inspector)\n" + + "{\n"); + } + + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String inspectorLine = "__inspector[\"" + child.getName() + "\"][\"value\"]"; + if (child.isArray) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + ".push_back("); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(");\n"); + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + "[" + inspectorLine + "[__i][\"key\"].asString().make_string()] = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(";\n"); + w.write(indent + "}\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + ".as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + ".as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + ".as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + ")"); + } + w.write(";\n"); + } + } + w.write("}\n\n"); + } + + public void writeSlimeConstructor(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } else { + w.write("" + + parent + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } + w.write(indent + "const vespalib::slime::Inspector & __inspector(__payload.get());\n"); + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String childInspector = "__inspector[\"" + child.getName() + "\"]"; + if (child.isArray) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::VectorInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else if (child.isMap) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::MapInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + w.write("Internal" + childType + "Converter"); + } else { + w.write("::config::internal::ValueConverter<" + childType + ">"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + String defaultValue = getDefaultValue(leaf); + w.write("()(" + childInspector + ", " + defaultValue + ");\n"); + } else if (child instanceof InnerCNode) { + w.write("()(" + childInspector + ");\n"); + } else { + w.write("()(\"" + child.getName() + "\", " + childInspector + ");\n"); + } + } + } + w.write("}\n\n"); + } + + void writeBodyFooter(Writer w, CNode root) throws IOException { + w.write("} // namespace internal\n\n"); + writeNameSpaceEnd(w, generateCppNameSpace(root)); + } + + String getDefaultValue(LeafCNode leaf) { + String defaultVal = leaf.getDefaultValue().getStringRepresentation(); + if (leaf.getType().equals("string") && defaultVal.equals("null")) + throw new CodegenRuntimeException("Default value null not allowed for C++ config"); + if (leaf.getType().equals("long") && "-9223372036854775808".equals(defaultVal)) { + return "LONG_MIN"; + } else if (leaf.getType().equals("int") && "-2147483648".equals(defaultVal)) { + return "INT_MIN"; + } else { + return defaultVal; + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java new file mode 100644 index 00000000000..f58c202f70f --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java @@ -0,0 +1,274 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + */ +public class DefLine { + private final static Pattern defaultPattern = Pattern.compile("^\\s*default\\s*=\\s*(\\S+)"); + private final static Pattern rangePattern = Pattern.compile("^\\s*range\\s*=\\s*([\\(\\[].*?[\\)\\]])"); + private final static Pattern restartPattern = Pattern.compile("^\\s*restart\\s*"); + private final static Pattern wordPattern = Pattern.compile("\\S+"); + private final static Pattern enumPattern = Pattern.compile("\\s*\\{(\\s*\\w+\\s*)+(\\s*,\\s*\\w+\\s*)*\\s*\\}"); + private final static Pattern enumPattern2 = Pattern.compile("\\s*,\\s*"); + private final static Pattern wordPattern2 = Pattern.compile("\\w+"); + private final static Pattern digitPattern = Pattern.compile("\\d"); + private final static Pattern namePattern = Pattern.compile("\\s*[a-zA-Z0-9_]+\\s*"); + private final static Pattern whitespacePattern = Pattern.compile("\\s+"); + + private String name = null; + private final Type type = new Type(); + + private DefaultValue defaultValue = null; + + private String range = null; + private boolean restart = false; + + String enumString = null; + final String[] enumArray = null; + + private final static Pattern defaultNullPattern = Pattern.compile("^\\s*default\\s*=\\s*null"); + + public DefLine(String line) { + StringBuilder sb = new StringBuilder(line); + int parsed = parseNameType(sb); + sb.delete(0, parsed); + if (type.name.equals("enum")) { + parsed = parseEnum(sb); + sb.delete(0, parsed); + } + + while (sb.length() > 0) { + parsed = parseOptions(sb); + sb.delete(0, parsed); + } + validateName(); + validateReservedWords(); + } + + /** + * Currently (2012-03-05) not used. Ranges are not checked by the + */ + public String getRange() { + return range; + } + + public DefaultValue getDefault() { + return defaultValue; + } + + public String getName() { + return name; + } + + public Type getType() { + return type; + } + + public boolean getRestart() { + return restart; + } + + public String getEnumString() { + return enumString; + } + + public String[] getEnumArray() { + return enumArray; + } + + /** + * Special function that searches through s and returns the index + * of the first occurrence of " that is not escaped. + */ + private String findStringEnd(CharSequence s, int from) { + boolean escaped = false; + for (int i = from; i < s.length(); i++) { + switch (s.charAt(i)) { + case'\\': + escaped = !escaped; + break; + case'"': + if (!escaped) { + return s.subSequence(from, i).toString(); + } + break; + } + } + return null; + } + + + private int parseOptions(CharSequence string) { + Matcher defaultNullMatcher = defaultNullPattern.matcher(string); + Matcher defaultMatcher = defaultPattern.matcher(string); + Matcher rangeMatcher = rangePattern.matcher(string); + Matcher restartMatcher = restartPattern.matcher(string); + + if (defaultNullMatcher.find()) { + throw new IllegalArgumentException("Null default value is not allowed: " + string.toString()); + } else if (defaultMatcher.find()) { + String deflt = defaultMatcher.group(1); + if (deflt.charAt(0) == '"') { + int begin = defaultMatcher.start(1) + 1; + deflt = findStringEnd(string, begin); + if (deflt == null) { + throw new IllegalArgumentException(string.toString()); + } + defaultValue = new DefaultValue(deflt, type); + return begin + deflt.length() + 1; + } else { + defaultValue = new DefaultValue(deflt, type); + } + return defaultMatcher.end(); + } else if (rangeMatcher.find()) { + range = rangeMatcher.group(1); + return rangeMatcher.end(); + } else if (restartMatcher.find()) { + restart = true; + return restartMatcher.end(); + } else { + throw new IllegalArgumentException(string.toString()); + } + } + + private int parseNameType(CharSequence string) { + Matcher wordMatcher = wordPattern.matcher(string); + if (wordMatcher.find()) { + name = wordMatcher.group(); + } + if (wordMatcher.find()) { + type.name = wordMatcher.group(); + } + if (type.name == null || name == null) { + throw new IllegalArgumentException(string.toString()); + } + return wordMatcher.end(); + } + + private int parseEnum(CharSequence string) { + Matcher enumMatcher = enumPattern.matcher(string); + if (enumMatcher.find()) { + enumString = enumMatcher.group(0).trim(); + } + if (enumString == null) { + throw new IllegalArgumentException(string + " is not valid syntax"); + } + enumString = enumString.replaceFirst("\\{\\s*", ""); + enumString = enumString.replaceFirst("\\s*\\}", ""); + String result[] = enumPattern2.split(enumString); + type.enumArray = new String[result.length]; + for (int i = 0; i < result.length; i++) { + String s = result[i].trim(); + type.enumArray[i] = s; + Matcher wordMatcher2 = wordPattern2.matcher(s); + if (!wordMatcher2.matches()) { + throw new IllegalArgumentException(s + " is not valid syntax"); + } + } + return enumMatcher.end(); + } + + public static class Type { + String name; + String[] enumArray; + + public Type(String name) { + this.name=name; + } + + public Type() { + } + + public String getName() { + return name; + } + + public String[] getEnumArray() { + return enumArray; + } + + public Type setEnumArray(String[] enumArray) { + this.enumArray = enumArray; + return this; + } + + public String toString() { + return "type " + name; + } + + } + + // A naive approach to imitate the checking previously done in make-config-preproc.pl + // TODO: method too long + void validateName() { + Matcher digitMatcher; + Matcher nameMatcher; + Matcher whitespaceMatcher; + + boolean atStart = true; + boolean arrayOk = true; + boolean mapOk = true; + for (int i = 0; i < name.length(); i++) { + String s = name.substring(i, i + 1); + digitMatcher = digitPattern.matcher(s); + nameMatcher = namePattern.matcher(s); + whitespaceMatcher = whitespacePattern.matcher(s); + if (atStart) { + if (digitMatcher.matches()) { + throw new IllegalArgumentException(name + " must start with a non-digit character"); + } + if (!nameMatcher.matches()) { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + atStart = false; + } else { + if (nameMatcher.matches()) { + // do nothing + } else if (s.equals(".")) { + arrayOk = true; + mapOk = true; + atStart = true; + } else if (s.equals("[")) { + if (!arrayOk) { + throw new IllegalArgumentException(name + " Arrays cannot be multidimensional"); + } + arrayOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("]"))) { + throw new IllegalArgumentException(name + " Expected ] to terminate array definition"); + } + i++; + } else if (s.equals("{")) { + if (!mapOk) { + throw new IllegalArgumentException(name + " Maps cannot be multidimensional"); + } + mapOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("}"))) { + throw new IllegalArgumentException(name + " Expected } to terminate map definition"); + } + i++; + } else if (whitespaceMatcher.matches()) { + break; + } else { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + } + } + } + + void validateReservedWords() { + if (ReservedWords.isReservedWord(name)) { + throw new IllegalArgumentException(name + " is a reserved word in " + + ReservedWords.getLanguageForReservedWord(name)); + } + if (ReservedWords.capitalizedPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with an uppercase letter"); + } + if (ReservedWords.internalPrefixPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'"); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java new file mode 100644 index 00000000000..0a6c225fa19 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class generates a tree of CNodes from a .def file. + * + * @author gjoranv + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class DefParser { + static final Pattern commentPattern = Pattern.compile("^\\s*#+\\s*(.*?)\\s*$"); + public static final Pattern versionPattern = Pattern.compile("^(version\\s*=\\s*)([0-9][0-9-]*)$"); + // Namespace must start with a letter, since Java (Java language Spec, section 3.8) and C++ identifiers cannot start with a digit + public static final Pattern namespacePattern = Pattern.compile("^(namespace\\s*=\\s*)(([a-z][a-z0-9_]*)+([.][a-z][a-z0-9_]*)*)$"); + + private final BufferedReader reader; + private final String name; + private InnerCNode root = null; + private NormalizedDefinition normalizedDefinition = null; + + + private String comment = ""; + + /** + * Creates a new parser for a .def file with the given name and that can be accessed by the given reader. + * + * @param name The name of the .def file (not including version number and the '.def' suffix). + * @param defReader A reader to the .def file. + */ + public DefParser(String name, Reader defReader) { + this.name = createName(name); + if (defReader == null) { + throw new CodegenRuntimeException("Must have a non-null reader for a .def file."); + } + if (defReader instanceof BufferedReader) { + reader = (BufferedReader)defReader; + } else { + reader = new BufferedReader(defReader); + } + } + + // If name contains namespace, return just name + private String createName(String name) { + if (name.contains(".")) { + return name.substring(name.lastIndexOf(".") + 1); + } else { + return name; + } + } + + /** + * Parses the .def file upon the initial call. Subsequent calls returns the result from the initial call. + * + * @return A tree of CNodes representing this instance's .def file. + * @throws CodegenRuntimeException upon errors. + */ + public InnerCNode getTree() throws CodegenRuntimeException { + try { + if (root == null) parse(); + } catch (DefParserException | IOException e) { + throw new CodegenRuntimeException("Error parsing or reading config definition." + e.getMessage(), e); + } + return root; + } + + /** + * Parses the input from the reader and builds a tree of CNodes representing the .def file. + * + * @throws IOException upon reader errors. + * @throws DefParserException upon parsing errors. + */ + void parse() throws IOException, DefParserException { + root = new InnerCNode(name); + normalizedDefinition = new NormalizedDefinition(); + + String s; + List<String> originalInput = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + originalInput.add(s); + } + reader.close(); + + + // Parse and build tree of the original input + parseLines(root, originalInput, normalizedDefinition); + root.setMd5(normalizedDefinition.generateMd5Sum()); + } + + /** + * Parses one line from def-file and adds it to the tree. + * TODO: Method too long! + * + * @param root The root CNode in the tree. + * @param line A line from the def-file. + * @param nd A NormalizedDefinition object + * @throws IllegalArgumentException upon error in line. + */ + private void parseLine(CNode root, String line, NormalizedDefinition nd) throws IllegalArgumentException { + line = NormalizedDefinition.normalize(line); + line = line.trim(); + if (line.length() == 0) { + // If having empty lines in between comments, that is logically a break in the comment too + if (!comment.isEmpty()) { + comment += "\n"; + } + return; + } + Matcher commentMatch = commentPattern.matcher(line); + if (commentMatch.matches()) { + parseCommentLine(commentMatch); + return; + } + Matcher versionMatch = versionPattern.matcher(line); + if (versionMatch.matches()) { + parseVersionLine(versionMatch); + return; + } + Matcher namespaceMatcher = namespacePattern.matcher(line); + if (namespaceMatcher.matches()) { + parseNamespaceLine(namespaceMatcher.group(2)); + nd.addNormalizedLine(line); + return; + } + // Only add lines that are not version, namespace or comment lines + nd.addNormalizedLine(line); + DefLine defLine = new DefLine(line); + root.setLeaf(root.getName() + "." + defLine.getName(), defLine, comment); + comment = ""; + } + + private void parseCommentLine(Matcher commentMatch) { + if (!comment.isEmpty()) comment += "\n"; + String addition = commentMatch.group(1); + if (addition.isEmpty()) addition = " "; + comment += addition; + } + + private void parseVersionLine(Matcher matcher) { + root.setVersion(matcher.group(2)); + root.setComment(comment); + comment = ""; + } + + private void parseNamespaceLine(String namespace) { + if (namespace.startsWith("com.yahoo.")) + throw new IllegalArgumentException("Remove 'com.yahoo.' from the namespace '" + namespace + + "' - it will be automatically added to the java package name."); + root.setNamespace(namespace); + root.setComment(comment); + comment = ""; + } + + void parseLines(CNode root, List<String> defLines, NormalizedDefinition nd) throws DefParserException { + DefParserException failure = null; + int lineNumber = 1; + for (String line : defLines) { + try { + parseLine(root, line, nd); + lineNumber++; + } catch (IllegalArgumentException e) { + String msg = "Error when parsing line " + lineNumber + ": " + line + "\n" + e.getMessage(); + failure = new DefParserException(msg, e); + break; + } + } + + if (failure != null) { + throw (failure); + } + } + + public NormalizedDefinition getNormalizedDefinition() { + return normalizedDefinition; + } + + /** + * For debugging - dump the tree from the given root to System.out. + */ + public static void dumpTree(CNode root, String indent) { + StringBuilder sb = new StringBuilder(indent + root.getName()); + if (root instanceof LeafCNode) { + LeafCNode leaf = ((LeafCNode)root); + if (leaf.getDefaultValue() != null) { + sb.append(" = ").append(((LeafCNode)root).getDefaultValue().getValue()); + } + } + System.out.println(sb.toString()); + if (!root.getComment().isEmpty()) { + String comment = root.getComment(); + if (comment.contains("\n")) { + comment = comment.substring(0, comment.indexOf("\n")) + "..."; + } + if (comment.length() > 60) { + comment = comment.substring(0, 57) + "..."; + } + System.out.println(indent + " comment: " + comment); + } + CNode[] children = root.getChildren(); + for (CNode c : children) { + dumpTree(c, indent + " "); + } + + } + + class DefParserException extends Exception { + DefParserException(String s, Throwable cause) { + super(s, cause); + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java new file mode 100644 index 00000000000..8518ff04573 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * An immutable class representing a default value of a config variable + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DefaultValue { + + private String value = null; + + // The variable type. Always set UNLESS the value is null. + private DefLine.Type type = null; + + + /** + * Null value + */ + public DefaultValue() { + } + + /** + * A default value with the given value and type. + */ + public DefaultValue(String value, DefLine.Type type) { + this.value = value; + this.type = type; + } + + /** + * Returns the toString of the default value. + */ + public String getValue() { + return value; + } + + /** + * Returns the string representation of this value + */ + public String getStringRepresentation() { + if (value == null) + return "null"; + else if ("bool".equals(type.getName())) + return value; + else if ("int".equals(type.getName())) + return value; + else if ("long".equals(type.getName())) + return value; + else if ("double".equals(type.getName())) + return value; + else if ("enum".equals(type.getName())) + return value; + else { + // building a string, do unicode-escaping + StringBuilder sb = new StringBuilder(); + for (char c : value.toCharArray()) { + if (c > '\u007f') { + sb.append(String.format("\\u%04X", (int) c)); + } else { + sb.append(c); + } + } + return "\"" + sb.toString() + "\""; + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java new file mode 100644 index 00000000000..b7803fc8c22 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.Map; +import java.util.LinkedHashMap; + +/** + * Represents an inner node in the configuration tree. + * + * @author gjoranv + */ +public class InnerCNode extends CNode { + + /** + * The children of this Node. Mapped using their short name as + * string. This variable is null only if Node is a leaf Node. + */ + private final Map<String, CNode> children = new LinkedHashMap<String, CNode>(); + private boolean restart = false; + + /** + * Constructor for the root node. + */ + public InnerCNode(String name) { + super(null, name.split("\\.def")[0]); + defName = this.name; + } + + /** + * Constructor for inner nodes. + */ + private InnerCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public CNode[] getChildren() { + CNode[] ret = new CNode[children.size()]; + children.values().toArray(ret); + return ret; + } + + /** + * Access to children for testing + * @return modifiable children map + */ + public Map<String, CNode> children() { + return children; + } + + @Override + public CNode getChild(String name) { + return children.get(name); + } + + /** + * Returns and eventually creates the given subnode. + */ + private CNode createOrGetChild(DefLine.Type type, String name) throws IllegalArgumentException { + String key = name; + int split = name.indexOf('.'); + CNode newChild; + if (split != -1) { + key = name.substring(0, split).trim(); + newChild = new InnerCNode(this, key); + } else { + newChild = LeafCNode.newInstance(type, this, key); + if (newChild == null) + throw new IllegalArgumentException + ("Could not create " + type.name + " " + name); + } + return children.containsKey(newChild.getName()) + ? children.get(newChild.getName()) + : newChild; + } + + /** + * Adds a child to this node with the given type, name and value. Necessary children on the path + * to the given leaf node will be added as well. + * @param name the full name/path of the node to add. + * @param defLine the parsed .def-file line to add. + * @param comment comment extracted from the .def-file. + */ + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + if (name.indexOf('.') < 0) { + throw new IllegalArgumentException("Parameter with name '" + name + + "' cannot be a leaf node as it has already been declared as an inner node."); + } + checkMyName(name.substring(0, name.indexOf('.'))); + String childName = name.substring(name.indexOf('.') + 1); + + CNode child = createOrGetChild(defLine.getType(), childName); +/* + System.out.println("\nAdding child name: " + name); + System.out.println(" getName: " + child.getName()); + System.out.println(" full name: " + child.getFullName()); + System.out.println(" classname: " + child.getClassName()); + System.out.println(" full classname: " + child.getFullClassName()); +*/ + restart |= defLine.getRestart(); + child.setLeaf(childName, defLine, comment); + children.put(child.getName(), child); + } + + @Override + public boolean needRestart() { + return restart; + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java new file mode 100644 index 00000000000..3cd7b61ba55 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java @@ -0,0 +1,267 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author gjoranv + */ +public abstract class LeafCNode extends CNode { + + private boolean isInitialized = false; + private DefaultValue defaultValue = null; + private boolean restart = false; + + /** + * Constructor for the leaf nodes. + */ + protected LeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name) { + try { + switch (type.name) { + case "int": + return new IntegerLeaf(parent, name); + case "long": + return new LongLeaf(parent, name); + case "double": + return new DoubleLeaf(parent, name); + case "bool": + return new BooleanLeaf(parent, name); + case "string": + return new StringLeaf(parent, name); + case "reference": + return new ReferenceLeaf(parent, name); + case "file": + return new FileLeaf(parent, name); + case "path": + return new PathLeaf(parent, name); + case "enum": + return new EnumLeaf(parent, name, type.enumArray); + default: + return null; + } + } catch (NumberFormatException e) { + return null; + } + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name, String defVal) { + LeafCNode ret = newInstance(type, parent, name); + if (defVal!=null) { + DefaultValue def = new DefaultValue(defVal, type); + ret.setDefaultValue(def); + } + return ret; + } + + public abstract String getType(); + + @Override + public CNode[] getChildren() { + return new CNode[0]; + } + + @Override + public CNode getChild(String name) { + return null; + } + + public DefaultValue getDefaultValue() { + return defaultValue; + } + + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + /** + * @param defaultValue The value to check. + * @throws IllegalArgumentException if the value is illegal according to the node type. + */ + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + } + + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + DefLine.Type type = defLine.getType(); + // TODO: why the !is... conditions? + if (!isMap && !isArray && isInitialized) { + throw new IllegalArgumentException(name + " is already defined"); + } + isInitialized = true; + checkMyName(name); + if (!type.name.equalsIgnoreCase(getType())) { + throw new IllegalArgumentException("Type " + type.name + " does not match " + getType()); + } + setValue(defLine.getDefault()); + setComment(comment); + restart |= defLine.getRestart(); + } + + @Override + public boolean needRestart() { + return restart; + } + + public final void setValue(DefaultValue defaultValue) throws IllegalArgumentException { + try { + checkDefaultValue(defaultValue); + setDefaultValue(defaultValue); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException + ("Invalid default value", e); + } + } + + /** + * Superclass for leaf nodes that should not generate class. + */ + public static abstract class NoClassLeafCNode extends LeafCNode { + protected NoClassLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + } + + /** + * Superclass for no-class leaf nodes that cannot have a default. + */ + public static abstract class NoClassNoDefaultLeafCNode extends LeafCNode { + protected NoClassNoDefaultLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + if (defaultValue != null) + throw new IllegalArgumentException("Parameters of type '" + getType() + "' cannot have a default value."); + return this; + } + } + + public static class IntegerLeaf extends NoClassLeafCNode { + protected IntegerLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "int"; + } + } + + public static class LongLeaf extends NoClassLeafCNode { + protected LongLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "long"; + } + } + + public static class DoubleLeaf extends NoClassLeafCNode { + protected DoubleLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "double"; + } + } + + public static class BooleanLeaf extends NoClassLeafCNode { + protected BooleanLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "bool"; + } + } + + public static class StringLeaf extends NoClassLeafCNode { + protected StringLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "string"; + } + } + + public static class ReferenceLeaf extends StringLeaf { + ReferenceLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "reference"; + } + } + + public static class FileLeaf extends NoClassNoDefaultLeafCNode { + FileLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "file"; + } + } + + public static class PathLeaf extends NoClassNoDefaultLeafCNode { + PathLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "path"; + } + } + + public static class EnumLeaf extends LeafCNode { + + private final String[] legalValues; + + protected EnumLeaf(InnerCNode parent, String name, String[] valArray) { + super(parent, name); + this.legalValues = valArray; + } + + @Override + public String getType() { + return "enum"; + } + + /** @return This enum's legal values. */ + public String[] getLegalValues() { + return legalValues; + } + + @Override + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + if ((defaultValue != null) && (defaultValue.getValue() != null)) { + String defaultString = null; + String value = defaultValue.getValue(); + for (String val : legalValues) { + if (value.equals(val)) { + defaultString = val; + } + } + if (defaultString == null) + throw new IllegalArgumentException("Could not initialize enum with: " + value); + } + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java new file mode 100644 index 00000000000..f28549275ef --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.logging.Logger; + +/** + * This class generates code for a config class from a given def-file. + */ +public class MakeConfig { + + private final static Logger log = Logger.getLogger(MakeConfig.class.getName()); + + private final ClassBuilder classBuilder; + + public MakeConfig(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties properties) { + classBuilder = createClassBuilder(root, nd, path, properties); + } + + public static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties prop) { + if (prop.language.equals("cppng") || prop.language.equals("cpp")) + return new CppClassBuilder(root, nd, prop.destDir, prop.dirInRoot); + else + return new JavaClassBuilder(root, nd, prop.destDir); + } + + /** + * Generates the code and print it to this.out. + */ + void buildClasses() { + classBuilder.createConfigClasses(); + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java -Dconfig.dest=<dir> -Dconfig.spec=<path> [-Dconfig.lang=cpp -Dconfig.subdir=<dir>] [-Dconfig.dumpTree=true] MakeConfig"); + out.println(" (default language for generated code is Java)"); + } + + public static void main(String[] args) throws IOException, InterruptedException { + try { + MakeConfigProperties props = new MakeConfigProperties(); + for (File specFile : props.specFiles) { + String path = specFile.toURI().toString(); + String name = specFile.getName(); + if (name.endsWith(".def")) name = name.substring(0, name.length() - 4); + DefParser parser = new DefParser(name, new FileReader(specFile)); + InnerCNode configRoot = parser.getTree(); + checkNamespace(name, configRoot); + if (configRoot != null) { + MakeConfig mc = new MakeConfig(configRoot, parser.getNormalizedDefinition(), path, props); + mc.buildClasses(); + if (props.dumpTree) { + System.out.println("\nTree dump:"); + DefParser.dumpTree(configRoot, ""); + } + } else { + System.exit(1); + } + } + } catch (PropertyException e) { + System.out.println(Exceptions.toMessageString(e)); + printUsage(System.err); + System.exit(1); + } catch (CodegenRuntimeException e) { + System.out.println(Exceptions.toMessageString(e)); + System.exit(1); + } + } + + private static void checkNamespace(String name, InnerCNode configRoot) { + if (configRoot.defNamespace == null) + throw new IllegalArgumentException("In config definition '" + name + "': A namespace is required"); + } + + // The Exceptions class below is copied from vespajlib/com.yahoo.protect.Exceptions + + /** + * Helper methods for handling exceptions + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ + static class Exceptions { + + /** + * <p>Returns a use friendly error message string which includes information from all nested exceptions. + * + * <p>The form of this string is + * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> + * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. + */ + public static String toMessageString(Throwable t) { + StringBuilder b = new StringBuilder(); + String lastMessage = null; + String message; + for (; t != null; t = t.getCause(), lastMessage = message) { + message = getMessage(t); + if (message == null) continue; + if (lastMessage != null && lastMessage.equals(message)) continue; + if (b.length() > 0) + b.append(": "); + b.append(message); + } + return b.toString(); + } + + /** + * Returns a useful message from *this* exception, or null if none + */ + private static String getMessage(Throwable t) { + String message = t.getMessage(); + if (t.getCause() == null) { + if (message == null) return toShortClassName(t); + return message; + } else { + if (message == null) return null; + if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; + return message; + } + } + + private static String toShortClassName(Object o) { + String longName = o.getClass().getName(); + int lastDot = longName.lastIndexOf("."); + if (lastDot < 0) return longName; + return longName.substring(lastDot + 1); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java new file mode 100644 index 00000000000..d055a1fe1fb --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.StringTokenizer; + +/** + * Encapsulates data extracted from system properties. + * + * @author <a href="gv@yahoo-inc.com">Gjoran Voldengen</a> + */ +public class MakeConfigProperties { + + private final List<String> legalLanguages = Arrays.asList("java", "cpp", "cppng" ); + + final File destDir; + final File[] specFiles; + final String language; + final String dirInRoot; // Where within fileroot to store generated class files + final boolean dumpTree; + final boolean generateFrameworkCode; + + MakeConfigProperties() throws PropertyException { + destDir = checkDestinationDir(); + specFiles = checkSpecificationFiles(); + language = checkLanguage(); + dirInRoot = checkDirInRoot(); + dumpTree = System.getProperty("config.dumpTree") != null && + System.getProperty("config.dumpTree").equalsIgnoreCase("true"); + generateFrameworkCode = System.getProperty("config.useFramework") == null || + System.getProperty("config.useFramework").equalsIgnoreCase("true"); + } + + private File checkDestinationDir() throws PropertyException { + String destination = System.getProperty("config.dest"); + if (destination == null) + throw new PropertyException("Missing property: config.dest."); + + File dir = new File(destination); + if (!dir.isDirectory()) { + throw new PropertyException("Could not find directory: " + dir.getPath()); + } + return dir; + } + + private String checkDirInRoot() throws PropertyException { + String dirInRoot = System.getProperty("config.subdir"); + // Optional parameter + if (dirInRoot == null) { return null; } + File f = new File(destDir, dirInRoot); + if (!f.isDirectory()) { + throw new PropertyException("Could not find directory: " + f.getPath()); + } + return dirInRoot; + } + + /** + * @return Desired programming language of generated code, default is "java". + * @throws PropertyException if supplied language is not a legal language. + */ + private String checkLanguage() throws PropertyException { + String inputLang = System.getProperty("config.lang", "java").toLowerCase(); + if (! legalLanguages.contains(inputLang)) { + throw new PropertyException + ("Unsupported code language: '" + inputLang + "'. Supported languages are: " + legalLanguages); + } + return inputLang; + } + + private static File[] checkSpecificationFiles() throws PropertyException { + String string = System.getProperty("config.spec"); + if (string == null) + throw new PropertyException("Missing property: config.spec."); + + StringTokenizer st = new StringTokenizer(string, ","); + if (st.countTokens() == 0) + throw new PropertyException("Missing property: config.spec."); + + File[] files = new File[st.countTokens()]; + for (int i = 0; st.hasMoreElements(); i++) { + files[i] = new File((String) st.nextElement()); + if (!files[i].isFile()) + throw new PropertyException("Could not read file " + files[i].getPath()); + } + return files; + } + +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java new file mode 100644 index 00000000000..1847150d86d --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.text.DecimalFormat; +import java.security.MessageDigest; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * + * Does normalizing (removing comments, trimming whitespace etc.) and calculation of md5sum + * of config definitions + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class NormalizedDefinition { + /* Patterns used for finding ranges in config definitions */ + private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); + private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); + private MessageDigest md5; + + String defMd5 = null; + List<String> normalizedContent = null; + + public NormalizedDefinition() { + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (java.security.NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to create MD5 digest", e); + } + normalizedContent = new ArrayList<>(); + } + + public NormalizedDefinition normalize(BufferedReader reader) throws IOException { + String s; + List<String> input = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + String normalized = normalize(s); + if (normalized.length() > 0) { + input.add(normalized); + } + } + normalizedContent = input; + return this; + } + + /** + * Normalizes a config definition line. Each string is normalized according to the + * rules of config and definition files before they are used: + * <ul> + * <li>Remove trailing space.<li> + * <li>Remove trailing comments, and spaces before trailing comments.</li> + * <li>Remove empty lines</li> + * <li>Keep comment lines</li> + * </ul> + * The supplied list is changed in-place + * + * @param line A config definition line + * @return a normalized config definition line + */ + public static String normalize(String line) { + //System.out.println("before line=" + line + ";"); + // Normalize line + line = line.trim(); + Matcher m = intPattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(0x7fffffff); + String formattedMin = new DecimalFormat("#.#").format(-0x80000000); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + m = doublePattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(1e308); + String formattedMin = new DecimalFormat("#.#").format(-1e308); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + line = removeComment(line); + if (!line.isEmpty()) { + line = stripSpaces(line); + line = line.replaceAll("\\s,", ","); // Remove space before comma (for enums) + line += "\n"; + } + //System.out.println("after line=" + line + ";"); + return line; + } + + // Removes comment char and text after it, unless comment char is inside a string + // Keeps comment lines (lines that start with #) + private static String removeComment(String line) { + int index = line.indexOf("#"); + if (!line.contains("#") || index == 0) return line; + + int firstQuote = line.indexOf("\""); + if (firstQuote > 0) { + int secondQuote = line.indexOf("\"", firstQuote + 1); + if (index > secondQuote) { + line = line.substring(0, index); + line = line.trim(); + } + } else { + line = line.substring(0, index); + line = line.trim(); + } + + return line; + } + + public void addNormalizedLine(String line) { + normalizedContent.add(line); + } + + public String generateMd5Sum() { + for (String line : normalizedContent) { + String s = normalize(line); + if (!s.isEmpty()) { + md5.update(toBytes(s)); + } + } + defMd5 = toHexString(md5.digest()).toLowerCase(); + //System.out.println("md5=" + defMd5) ; + return defMd5; + } + + + // The two methods below are copied from vespajlib (com.yahoo.text.Utf8 and com.yahoo.io.HexDump) + // since configgen cannot depend on any other modules (at least not as it is done now) + public static byte[] toBytes(String str) { + Charset charset = Charset.forName("utf-8"); + + ByteBuffer b = charset.encode(str); + byte[] result = new byte[b.remaining()]; + b.get(result); + return result; + } + + private String toHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte aByte : bytes) { + sb.append(String.format("%02x", aByte)); + } + return sb.toString(); + } + + /** + * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * @param str String to strip spaces from + * @return String with spaces stripped + */ + public static String stripSpaces(String str) { + StringBuilder ret = new StringBuilder(""); + boolean inQuotes = false; + boolean inSpaceSequence = false; + for (char c : str.toCharArray()) { + if (Character.isWhitespace(c)) { + if (inQuotes) { + ret.append(c); + continue; + } + if (!inSpaceSequence) { + // start of space sequence + inSpaceSequence=true; + ret.append(" "); + } + } else { + if (inSpaceSequence) { + inSpaceSequence=false; + } + if (c=='\"') { + inQuotes=!inQuotes; + } + ret.append(c); + } + } + return ret.toString(); + } + + public List<String> getNormalizedContent() { + return normalizedContent; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String line : normalizedContent) { + builder.append(line.replace("\"", "\\\"")); + builder.append("\\n\\\n"); + } + return builder.toString(); + } + + public String getDefMd5() { + return defMd5; + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java new file mode 100644 index 00000000000..a42f76340de --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +public class PropertyException extends Exception { + PropertyException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java new file mode 100644 index 00000000000..145f2da3245 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.HashMap; +import java.util.regex.Pattern; + +/** + * Reserved words that cannot be used as variable names in a config definition file. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 2009-06-24 + */ + +public class ReservedWords { + + public static final String INTERNAL_PREFIX = "__"; + final static Pattern internalPrefixPattern = Pattern.compile("^" + INTERNAL_PREFIX + ".*"); + final static Pattern capitalizedPattern = Pattern.compile("^[A-Z].*"); + + private static final String[] cKeywords = + {"asm", "auto", "bool", "break", "case", "catch", + "char", "class", "const", "const_cast", "continue", "default", + "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", + "export", "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "item", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", "static_cast", + "struct", "switch", "template", "this", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", + "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", + "bitand"}; + + private static final String[] javaKeywords = + {"abstract", "boolean", "break", "byte", "case", + "catch", "char", "class","continue", "default", "do", "double", + "else", "extends","false", "final", "finally", "float", "for", + "if","implements", "import", "instanceof", "int", "interface", + "item", "long","native", "new", "null", "package", "private", + "protected","public", "return", "short", "static", + "strictfp","super","switch", "synchronized", "this", + "throw","throws","transient", "true", "try", "void", + "volatile","while", "byvalue", "cast", "const", "future", + "generic","goto", "inner", "operator", "outer", "rest", "var"}; + + private static final HashMap<String, String> allKeyWords; + + static { + allKeyWords = new HashMap<String, String>(); + for (String s : cKeywords) { + allKeyWords.put(s, "C"); + } + for (String s : javaKeywords) { + if (allKeyWords.containsKey(s)) { + allKeyWords.put(s, "C and Java"); + } else { + allKeyWords.put(s, "Java"); + } + } + } + + + public static boolean isReservedWord(String word) { + return allKeyWords.containsKey(word); + } + + public static String getLanguageForReservedWord(String word) { + return allKeyWords.get(word); + } + + +} |