diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /configgen/src/main |
Publish
Diffstat (limited to 'configgen/src/main')
19 files changed, 4692 insertions, 0 deletions
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CNode.java b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java new file mode 100644 index 00000000000..02fec548b40 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.StringTokenizer; + +/** + * Abstract superclass for all nodes representing a definition file. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public abstract class CNode { + + public static final String DEFAULT_NAMESPACE = "config"; + + // TODO: replace by "type" enum + public final boolean isArray; + public final boolean isMap; + final String name; + final InnerCNode parent; + + // TODO: remove! Only set for the root node, and root.getName() returns the same thing! + String defName = null; + String defVersion = ""; + String defNamespace = null; + String defMd5 = "MISSING MD5"; + String comment = ""; + + + protected CNode(InnerCNode parent, String name) { + this.parent = parent; + int bracketIdx = name.indexOf('['); + int curlyIdx = name.indexOf('{'); + if (bracketIdx != -1) { + this.name = name.substring(0, bracketIdx); + isArray = true; + isMap = false; + } else if (curlyIdx != -1) { + this.name = name.substring(0, curlyIdx); + isMap = true; + isArray = false; + } else { + this.name = name; + isMap = false; + isArray = false; + } + } + + /** + * Returns the simple name of this node. + * @return the simple name of this node + */ + public String getName() { + return name; + } + + public InnerCNode getParent() { + return parent; + } + + public abstract CNode[] getChildren(); + + public abstract CNode getChild(String name); + + public String getMd5() { + return defMd5; + } + + void setMd5(String md5) { + defMd5 = md5; + } + + public String getVersion() { + return defVersion; + } + + void setVersion(String version) { + defVersion = version; + } + + public String getNamespace() { + return defNamespace; + } + + void setNamespace(String namespace) { + defNamespace = namespace; + } + + public String getComment() { + return comment; + } + + void setComment(String comment) { + this.comment = comment; + } + + protected abstract void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException; + + public abstract boolean needRestart(); + + protected void checkMyName(String myName) throws IllegalArgumentException { + if (isArray) { + int n1 = myName.indexOf('['); + int n2 = myName.indexOf(']'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid array syntax: " + myName); + myName = myName.substring(0, n1); + } else if (isMap) { + int n1 = myName.indexOf('{'); + int n2 = myName.indexOf('}'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid map syntax: " + myName); + myName = myName.substring(0, n1); + } else if (myName.contains("[]")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-array type."); + } else if (myName.contains("{}")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-map type."); + } + if (!myName.equals(getName())) + throw new IllegalArgumentException(myName + " does not match " + getName() + "."); + } + + /** + * @return the full name as a config path of this node. + */ + public String getFullName() { + StringBuilder buf = new StringBuilder(); + if (parent != null) + buf.append(parent.getFullName()); + if (buf.length() > 0) + buf.append('.'); + StringBuilder theName = new StringBuilder(this.getName()); + + if (isArray) theName.append("[]"); + else if (isMap) theName.append("{}"); + + return buf.append(theName).toString(); + } + + /** + * @param prefix The prefix to use, usually an indent (spaces) followed by either '*' or "//" + * @return a comment block where each line is prefixed, but the caller must close it if using '*'. + */ + public String getCommentBlock(String prefix) { + prefix = prefix + " "; + StringBuilder ret = new StringBuilder(); + if (getComment().length() > 0) { + StringTokenizer st = new StringTokenizer(getComment(), "\n"); + while (st.hasMoreTokens()) { + ret.append(prefix).append(st.nextToken()).append("\n"); + } + } + return ret.toString(); + } + + @Override + public String toString() { + return getNamespace()+"."+getName()+","+getVersion(); + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java new file mode 100644 index 00000000000..dab2ce98510 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public interface ClassBuilder { + + /** + * Generate config class file(s). + */ + public void createConfigClasses(); +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java new file mode 100644 index 00000000000..56a87c0506e --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * This exception is thrown on internal errors. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class CodegenRuntimeException extends RuntimeException { + public CodegenRuntimeException(String s, Throwable cause) { + super(s, cause); + } + + public CodegenRuntimeException(Throwable cause) { + super(cause); + } + + public CodegenRuntimeException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java new file mode 100644 index 00000000000..e6d7f29ad36 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -0,0 +1,1117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; + + +/** + * This class autogenerates C++ code for the C++ config, based on a CNode tree given. + */ +public class CppClassBuilder implements ClassBuilder { + private final CNode root; + private final NormalizedDefinition nd; + private final File rootDir; + private final String relativePathUnderRoot; + private static final Map<String, String> vectorTypeDefs; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "BoolVector"); + map.put("int32_t", "IntVector"); + map.put("int64_t", "LongVector"); + map.put("double", "DoubleVector"); + map.put("vespalib::string", "StringVector"); + vectorTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> mapTypeDefs; + static { + Map<String, String> map = new HashMap<>(); + map.put("bool", "BoolMap"); + map.put("int32_t", "IntMap"); + map.put("int64_t", "LongMap"); + map.put("double", "DoubleMap"); + map.put("vespalib::string", "StringMap"); + mapTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> slimeTypeMap; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "Bool"); + map.put("int", "Long"); + map.put("long", "Long"); + map.put("double", "Double"); + map.put("string", "String"); + map.put("enum", "String"); + map.put("file", "String"); + map.put("reference", "String"); + slimeTypeMap = Collections.unmodifiableMap(map); + } + + public CppClassBuilder(CNode root, NormalizedDefinition nd, File rootDir, String relativePathUnderRoot) { + this.root = root; + this.nd = nd; + this.rootDir = rootDir; + this.relativePathUnderRoot = relativePathUnderRoot; + } + + public void createConfigClasses() { + generateConfig(root, nd); + } + + String readFile(File f) throws IOException { + if (!f.isFile()) return null; + StringBuilder sb = new StringBuilder(); + BufferedReader sr = new BufferedReader(new FileReader(f)); + while (true) { + String line = sr.readLine(); + if (line == null) break; + sb.append(line).append("\n"); + } + return sb.toString(); + } + + void writeFile(File f, String content) throws IOException { + FileWriter fw = new FileWriter(f); + fw.write(content); + fw.close(); + } + + void generateConfig(CNode root, NormalizedDefinition nd) { + try{ + StringWriter headerWriter = new StringWriter(); + StringWriter bodyWriter = new StringWriter(); + writeHeaderFile(headerWriter, root); + writeBodyFile(bodyWriter, root, relativePathUnderRoot, nd); + + String newHeader = headerWriter.toString(); + String newBody = bodyWriter.toString(); + + File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h")); + File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp")); + + String oldHeader = readFile(headerFile); + String oldBody = readFile(bodyFile); + + if (oldHeader == null || !oldHeader.equals(newHeader)) { + writeFile(headerFile, newHeader); + } + if (oldBody == null || !oldBody.equals(newBody)) { + writeFile(bodyFile, newBody); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + String getFileName(CNode node, String extension) { + return "config-" + node.getName() + "." + extension; + } + + static String removeDashesAndUpperCaseAllFirstChars(String source, boolean capitalizeFirst) { + // Create upper case chars after each dash + String parts[] = source.split("[-_]"); + StringBuilder sb = new StringBuilder(); + for (String s : parts) { + sb.append(s.substring(0, 1).toUpperCase()).append(s.substring(1)); + } + String result = sb.toString(); + if (!capitalizeFirst) { + result = result.substring(0,1).toLowerCase() + result.substring(1); + } + return result; + } + + /** Convert name of type to the name we want to use in macro ifdefs in file. */ + String getDefineName(String name) { + return name.toUpperCase().replace("-", ""); + } + + /** Convert name of type to the name we want to use as type name in the generated code. */ + static String getTypeName(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, true); + } + + /** Convert name of an identifier from value in def file to name to use in C++ file. */ + String getIdentifier(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, false); + } + + void writeHeaderFile(Writer w, CNode root) throws IOException { + writeHeaderHeader(w, root); + writeHeaderPublic(w, root); + writeHeaderFooter(w, root); + } + + void writeHeaderPublic(Writer w, CNode root) throws IOException { + w.write("public:\n"); + writeHeaderTypeDefs(w, root, " "); + writeTypeDeclarations(w, root, " "); + writeHeaderFunctionDeclarations(w, getTypeName(root, false), root, " "); + writeStaticMemberDeclarations(w, " "); + writeMembers(w, root, " "); + } + + String [] generateCppNameSpace(CNode root) { + String namespace = root.getNamespace(); + if (namespace.contains(".")) { + return namespace.split("\\."); + } + return new String[]{namespace}; + } + + String generateCppNameSpaceString(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i]); + str.append("::"); + } + str.append(namespaceList[namespaceList.length - 1]); + return str.toString(); + } + + String generateCppNameSpaceDefine(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i].toUpperCase()); + str.append("_"); + } + str.append(namespaceList[namespaceList.length - 1].toUpperCase()); + return str.toString(); + } + + void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("namespace " + namespaceList[i] + " {\n\n"); + } + } + + void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("} // namespace " + namespaceList[i] + "\n\n"); + } + } + + void writeHeaderHeader(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespacePrint = generateCppNameSpaceString(namespaceList); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + w.write("" + + "/**\n" + + " * @class " + namespacePrint + "::" + className + "\n" + + " * @ingroup config\n" + + " *\n" + + " * @brief This is an autogenerated class for handling VESPA config.\n" + + " *\n" + + " * This class is autogenerated by vespa from a config definition file.\n" + + " * To subscribe to config, you need to include the config/config.h header, \n" + + " * and create a ConfigSubscriber in order to subscribe for config.\n" + ); + if (root.getComment().length() > 0) { + w.write(" *\n"); + StringTokenizer st = new StringTokenizer(root.getComment(), "\n"); + while (st.hasMoreTokens()) { + w.write(" * " + st.nextToken() + "\n"); + } + } + w.write("" + + " */\n" + + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n" + + "#define CLOUD_CONFIG_" + defineName + "_H\n" + + "\n" + + "#include <vespa/config/common/configvalue.h>\n" + + "#include <vespa/config/configgen/configpayload.h>\n" + + "#include <vespa/config/configgen/configinstance.h>\n" + + "#include <vespa/config/print/configdatabuffer.h>\n" + + "#include <vespa/vespalib/stllike/string.h>\n" + + "#include <vector>\n" + + "#include <map>\n" + + "\n"); + writeNameSpaceBegin(w, namespaceList); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("" + + "/**\n" + + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n" + + " * versions after this class declaration.\n" + + " */\n" + + "class Internal" + className + "Type : public ::config::ConfigInstance\n" + + "{\n" + ); + + } + + + + void writeTypeDeclarations(Writer w, CNode node, String indent) throws IOException { + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(indent + "enum " + typeName + " { "); + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + for (int i=0; i<leaf.getLegalValues().length; ++i) { + if (i != 0) { + w.write(", "); + } + w.write(leaf.getLegalValues()[i]); + } + w.write(" };\n" + + indent + "typedef std::vector<" + typeName + "> " + + typeName + "Vector;" + + "\n" + + indent + "typedef std::map<vespalib::string, " + typeName + "> " + + typeName + "Map;" + + "\n" + + indent + "static " + typeName + " get" + typeName + "(const vespalib::string&);\n" + + indent + "static vespalib::string get" + typeName + "Name(" + typeName + " e);\n" + + "\n" + ); + w.write(indent + "struct Internal" + typeName + "Converter {\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault);\n"); + w.write(indent + "};\n"); + } else { + w.write(indent + "class " + typeName + " {\n"); + w.write(indent + "public:\n"); + writeTypeDeclarations(w, child, indent + " "); + writeStructFunctionDeclarations(w, getTypeName(child, false), child, indent + " "); + writeMembers(w, child, indent + " "); + w.write(indent + "};\n"); + w.write(indent + "typedef std::vector<" + typeName + "> " + typeName + "Vector;\n\n"); + w.write(indent + "typedef std::map<vespalib::string, " + typeName + "> " + typeName + "Map;\n\n"); + } + } + } + } + + void writeHeaderFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + "const vespalib::string & defName() const { return CONFIG_DEF_NAME; }\n" + + indent + "const vespalib::string & defVersion() const { return CONFIG_DEF_VERSION; }\n" + + indent + "const vespalib::string & defMd5() const { return CONFIG_DEF_MD5; }\n" + + indent + "const vespalib::string & defNamespace() const { return CONFIG_DEF_NAMESPACE; }\n" + + indent + "void serialize(::config::ConfigDataBuffer & __buffer) const;\n"); + writeConfigClassFunctionDeclarations(w, "Internal" + className + "Type", node, indent); + } + + void writeConfigClassFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const ::config::ConfigValue & __value);\n"); + w.write(indent + className + "(const ::config::ConfigDataBuffer & __value);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + } + + void writeStructFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const std::vector<vespalib::string> & __lines);\n"); + w.write(indent + className + "(const vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + w.write(indent + "void serialize(vespalib::slime::Cursor & __cursor) const;\n"); + } + + void writeClassCopyConstructorDeclaration(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const " + className + " & __rhs);\n"); + } + + + void writeClassCopyConstructorDefinitionCommon(Writer w, CNode node) throws IOException { + for (int i = 0; i < node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" " + childName + "(__rhs." + childName + ")"); + } else { + w.write(",\n " + childName + "(__rhs." + childName + ")"); + } + } + } + + void writeConfigClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + w.write(parent + className + "(const " + className + " & __rhs)\n"); + w.write(" : ConfigInstance(),\n "); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + String typeName = getTypeName(node, false); + // Write empty constructor + w.write(parent + typeName + "(const " + typeName + " & __rhs)\n"); + w.write(" :"); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeCommonFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + className + "();\n"); + writeClassCopyConstructorDeclaration(w, className, node, indent); + w.write("" + + "\n" + + indent + "bool operator==(const " + className + "& __rhs) const;\n" + + indent + "bool operator!=(const " + className + "& __rhs) const;\n" + + "\n" + ); + } + + static String getTypeName(CNode node, boolean includeArray) { + String type = null; + if (node instanceof InnerCNode) { + InnerCNode innerNode = (InnerCNode) node; + type = getTypeName(innerNode.getName()); + } else if (node instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) node; + if (leaf.getType().equals("bool")) { + type = "bool"; + } else if (leaf.getType().equals("int")) { + type = "int32_t"; + } else if (leaf.getType().equals("long")) { + type = "int64_t"; + } else if (leaf.getType().equals("double")) { + type = "double"; + } else if (leaf.getType().equals("enum")) { + type = getTypeName(node.getName()); + } else if (leaf.getType().equals("string")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("reference")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("file")) { + type = "vespalib::string"; + } else { + throw new IllegalArgumentException("Unknown leaf datatype " + leaf.getType()); + } + } + if (type == null) { + throw new IllegalArgumentException("Unknown node " + node); + } + if (node.isArray && includeArray) { + if (vectorTypeDefs.containsKey(type)) { + type = vectorTypeDefs.get(type); + } else { + type = type + "Vector"; + } + } else if (node.isMap && includeArray) { + if (mapTypeDefs.containsKey(type)) { + type = mapTypeDefs.get(type); + } else { + type = type + "Map"; + } + } + return type; + } + + void writeStaticMemberDeclarations(Writer w, String indent) throws IOException { + w.write("" + + indent + "static const vespalib::string CONFIG_DEF_MD5;\n" + + indent + "static const vespalib::string CONFIG_DEF_VERSION;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAME;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAMESPACE;\n" + + indent + "static const std::vector<vespalib::string> CONFIG_DEF_SCHEMA;\n" + + indent + "static const int64_t CONFIG_DEF_SERIALIZE_VERSION;\n" + + "\n" + ); + } + + void writeComment(Writer w, String indent, String comment, boolean javadoc) + throws IOException + { + /** If simple one liner comment, write on one line. */ + if (javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 7)) + { + w.write(indent + "/** " + comment + " */\n"); + return; + } else if (!javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 3)) + { + w.write(indent + "// " + comment + "\n"); + return; + } + /** If not we need to write multi line comment. */ + int maxLineLen = 80 - (indent.length() + 3); + if (javadoc) w.write(indent + "/**\n"); + do { + String current; + // Extract first line to write + int newLine = comment.indexOf('\n'); + if (newLine == -1) { + current = comment; + comment = ""; + } else { + current = comment.substring(0, newLine); + comment = comment.substring(newLine + 1); + } + // If line too long, cut it in two + if (current.length() > maxLineLen) { + int spaceIndex = current.lastIndexOf(' ', maxLineLen); + if (spaceIndex >= maxLineLen - 15) { + comment = current.substring(spaceIndex + 1) + + "\n" + comment; + current = current.substring(0, spaceIndex); + } else { + comment = current.substring(maxLineLen) + "\n" + comment; + current = current.substring(0, maxLineLen) + "-"; + } + } + w.write(indent + (javadoc ? " * " : "// ") + current + "\n"); + } while (comment.length() > 0); + if (javadoc) w.write(indent + " */\n"); + } + + void writeMembers(Writer w, CNode node, String indent) throws IOException { + for (CNode child : node.getChildren()) { + String typeName = getTypeName(child, true); + if (child.getComment().length() > 0) { + String comment = child.getComment(); + int index; + do { + index = comment.indexOf("\n\n"); + if (index == -1) break; + String next = comment.substring(0, index); + comment = comment.substring(index + 2); + w.write("\n"); + writeComment(w, indent, next, false); + } while (true); + w.write("\n"); + writeComment(w, indent, comment, true); + } + w.write(indent + typeName + " " + getIdentifier(child.getName()) + ";"); + if (child instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) child; + DefaultValue value = leaf.getDefaultValue(); + if (value != null) { + w.write(" // Default: " + value.getStringRepresentation()); + } + } + w.write("\n"); + } + } + + void writeHeaderTypeDefs(Writer w, CNode root, String indent) throws IOException { + w.write(indent + "typedef std::unique_ptr<const " + getInternalClassName(root) + "> UP;\n"); + for (Map.Entry<String, String> entry : vectorTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String vectorName = entry.getValue(); + String typeDef = "typedef std::vector<" + typeName + "> " + vectorName; + w.write(indent + typeDef + ";\n"); + } + for (Map.Entry<String, String> entry : mapTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String mapName = entry.getValue(); + String typeDef = "typedef std::map<vespalib::string, " + typeName + "> " + mapName; + w.write(indent + typeDef + ";\n"); + } + } + + private static String getInternalClassName(CNode root) { + return "Internal" + getTypeName(root, false) + "Type"; + } + + void writeHeaderFooter(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + + w.write("" + + "};\n" + + "\n" + + "} // namespace internal\n\n"); + + w.write("typedef internal::" + getInternalClassName(root) + " " + className + "ConfigBuilder;\n"); + w.write("typedef const internal::" + getInternalClassName(root) + " " + className + "Config;\n"); + w.write("\n"); + writeNameSpaceEnd(w, namespaceList); + w.write("#endif // VESPA_config_" + defineName + "_H\n"); + } + + void writeBodyFile(Writer w, CNode root, String subdir, NormalizedDefinition nd) throws IOException { + writeBodyHeader(w, root, subdir); + writeStaticMemberDefinitions(w, root, nd); + writeDefinition(w, root, null); + writeBodyFooter(w, root); + } + + void writeBodyHeader(Writer w, CNode root, String subdir) throws IOException { + if (subdir == null) { + w.write("#include \"" + getFileName(root, "h") + "\""); + } else { + w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">"); + } + w.write("\n"); + w.write("#include <set>\n"); + w.write("#include <vespa/config/common/configparser.h>\n"); + w.write("#include <vespa/vespalib/data/slime/convenience.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/config/configgen/vector_inserter.h>\n"); + w.write("#include <vespa/config/configgen/map_inserter.h>\n"); + w.write("\n\n"); + writeNameSpaceBegin(w, generateCppNameSpace(root)); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("using ::config::ConfigParser;\n"); + w.write("using ::config::InvalidConfigException;\n"); + w.write("using ::config::ConfigInstance;\n"); + w.write("using ::config::ConfigValue;\n"); + w.write("using namespace vespalib::slime::convenience;\n"); + w.write("\n"); + } + + void writeStaticMemberDefinitions(Writer w, CNode root, NormalizedDefinition nd) throws IOException { + String typeName = getInternalClassName(root); + w.write("const vespalib::string " + typeName + "::CONFIG_DEF_MD5(\"" + root.defMd5 + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_VERSION(\"" + root.defVersion + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAME(\"" + root.defName + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAMESPACE(\"" + root.getNamespace() + "\");\n" + + "const int64_t " + typeName + "::CONFIG_DEF_SERIALIZE_VERSION(1);\n"); + w.write("const static vespalib::string __internalDefSchema[] = {\n"); + for (String line : nd.getNormalizedContent()) { + w.write("\"" + line.replace("\"", "\\\"") + "\",\n"); + } + w.write("};\n"); + w.write("const std::vector<vespalib::string> " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n"); + w.write(" __internalDefSchema + (sizeof(__internalDefSchema) / \n"); + w.write(" sizeof(__internalDefSchema[0])));\n"); + w.write("\n"); + } + + void writeDefinition(Writer w, CNode node, String parent) throws IOException { + boolean root = false; + if (parent == null) { + parent = getInternalClassName(node) + "::"; + root = true; + } + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + // Definition of getType(string) + w.write(parent + typeName + "\n" + + parent + "get" + typeName + "(const vespalib::string& name)\n" + + "{\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" " + (i != 0 ? "} else " : "")); + w.write("if (name == \"" + leaf.getLegalValues()[i] + "\") {\n" + + " return " + leaf.getLegalValues()[i] + ";\n"); + } + w.write(" } else {\n" + + " throw InvalidConfigException(\"Illegal enum value '\" + name + \"'\");\n" + + " }\n" + + "}\n" + + "\n" + ); + // Definition of getTypeName(enum) + w.write("vespalib::string\n" + + parent + "get" + typeName + "Name(" + typeName + " t)\n" + + "{\n" + + " switch (t) {\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" case " + leaf.getLegalValues()[i] + ": return \"" + leaf.getLegalValues()[i] + "\";\n"); + } + w.write(" default:\n" + + " {\n" + + " vespalib::asciistream ost;\n" + + " ost << \"UNKNOWN(\" << t << \")\";\n" + + " return ost.str();\n" + + " }\n" + + " }\n" + + "}\n" + + "\n" + ); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" throw InvalidConfigException(\"Value for '\" + __fieldName + \"' required but not found\");\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" return __eDefault;\n"); + w.write("}\n\n"); + } else { + writeDefinition(w, child, parent + typeName + "::"); + } + } + } + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + // Write empty constructor + w.write(parent + typeName + "()\n"); + for (int i=0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" : " + childName + "("); + } else { + w.write("),\n " + childName + "("); + } + if (child.isArray || child.isMap) { + // Default array for empty constructor is empty array. + } else if (child instanceof LeafCNode) { // If we have a default value, use that.. + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue() != null) { + w.write(getDefaultValue(leaf)); + } else { + // Defines empty constructor defaults for primitives without default set + if (leaf.getType().equals("bool")) { + w.write("false"); + } else if (leaf.getType().equals("int")) { + w.write("0"); + } else if (leaf.getType().equals("double")) { + w.write("0"); + } else if (leaf.getType().equals("string")) { + } else if (leaf.getType().equals("enum")) { + LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) leaf; + w.write(enumNode.getLegalValues()[0]); + } else if (leaf.getType().equals("reference")) { + } else if (leaf.getType().equals("file")) { + } + } + } + // If we hit neither else, we're an inner node, thus special type that has its own empty constructor + } + if (node.getChildren().length > 0) + w.write(")\n"); + w.write("" + + "{\n" + + "}\n" + + "\n" + ); + // Write copy constructor + if (root) + writeConfigClassCopyConstructorDefinition(w, parent, typeName, node); + else + writeClassCopyConstructorDefinition(w, parent, typeName, node); + + // Write parsing constructor + String indent = " "; + if (root) { + w.write(typeName + "::" + typeName + "(const ConfigValue & __value)\n" + + "{\n" + + indent + "try {\n"); + indent = " "; + w.write(indent + "const std::vector<vespalib::string> & __lines(__value.getLines());\n"); + } else { + w.write(parent + typeName + "(const std::vector<vespalib::string> & __lines)\n" + + "{\n"); + } + w.write("" + + indent + "std::set<vespalib::string> __remainingValuesToParse(" + + "__lines.begin(), __lines.end());\n"); + w.write(indent + "for(std::set<vespalib::string>::iterator __rVTPiter = __remainingValuesToParse.begin();\n" + + indent + " __rVTPiter != __remainingValuesToParse.end();)\n" + + indent + "{\n" + + indent + " if (ConfigParser::stripWhitespace(*__rVTPiter).empty()) {\n" + + indent + " std::set<vespalib::string>::iterator __rVTPiter2 = __rVTPiter++;\n" + + indent + " __remainingValuesToParse.erase(__rVTPiter2);\n" + + indent + " } else {\n" + + indent + " ++__rVTPiter;\n" + + indent + " }\n" + + indent + "}\n"); + for (CNode child : node.getChildren()) { + String childType = getTypeName(child, false); + String childName = getIdentifier(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + if (child.isArray) { + w.write(indent + "std::vector<vespalib::string> " + childName + "__ValueList(\n "); + } else if (child.isMap) { + w.write(indent + "std::map<vespalib::string, vespalib::string> " + childName + "__ValueMap(\n "); + } else { + w.write(indent + childName + " = get" + childType + "("); + } + childType = "vespalib::string"; + } else { + w.write(indent + childName + " = "); + } + if (child.isArray) { + w.write("ConfigParser::parseArray<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else if (child.isMap) { + w.write("ConfigParser::parseMap<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else { + if (child instanceof LeafCNode) { + w.write("ConfigParser::parse<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } else { + w.write("ConfigParser::parseStruct<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue().getValue() != null) { + String defaultVal = getDefaultValue(leaf); + if (leaf instanceof LeafCNode.EnumLeaf) { + defaultVal = '"' + defaultVal + '"'; + } + w.write(", " + defaultVal); + } + } + w.write(")"); + } + if (child instanceof LeafCNode.EnumLeaf) { + childType = getTypeName(child, false); + w.write(");\n"); + if (child.isArray) { + w.write(indent + childName + ".reserve(" + childName + "__ValueList.size());\n" + + indent + "for (std::vector<vespalib::string>::const_iterator __it\n" + + indent + " = " + childName + "__ValueList.begin();\n" + + indent + " __it != " + childName + "__ValueList.end(); ++__it)\n" + + indent + "{\n" + + indent + " " + childName + ".push_back(get" + childType + "(*__it));\n" + + indent + "}\n" + ); + } else if (child.isMap) { + w.write(indent + "typedef std::map<vespalib::string, vespalib::string> __ValueMap;\n"); + w.write(indent + "for (__ValueMap::iterator __it(" + childName + "__ValueMap.begin()), __mt(" + childName + "__ValueMap.end()); __it != __mt; __it++) {\n" + + " " + childName + "[__it->first] = get" + childType + "(__it->second);\n" + + "}\n" + ); + } + } else { + w.write(";\n"); + } + w.write(indent + "ConfigParser::stripLinesForKey(\"" + + child.getName() + "\", " + + "__remainingValuesToParse);\n"); + } + if (root) { + indent = " "; + w.write(indent + "} catch (InvalidConfigException & __ice) {\n"); + w.write(indent + " throw InvalidConfigException(\"Error parsing config '\" + CONFIG_DEF_NAME + \"' in namespace '\" + CONFIG_DEF_NAMESPACE + \"'" + + ": \" + __ice.getMessage());\n" + + indent + "}\n"); + } + w.write("}\n" + + "\n" + ); + // Write operator== + String lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator==(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return (" + ); + for (int i = 0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i != 0) { + w.write(" &&\n "); + } + w.write(childName + " == __rhs." + childName); + } + w.write(");\n" + + "}\n" + + "\n" + ); + // Write operator!= + lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator!=(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return !(operator==(__rhs));\n" + + "}\n" + + "\n" + ); + writeSlimeEncoder(w, node, parent, root); + writeSlimeDecoder(w, node, parent, root); + writeSlimeConstructor(w, node, parent, root); + } + + public void writeSlimeEncoder(Writer w, CNode node, String parent, boolean root) throws IOException + { + String indent = " "; + if (root) { + w.write("void\n" + + parent + "serialize(::config::ConfigDataBuffer & __buffer) const\n" + + "{\n"); + w.write(indent + "vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Cursor & __croot = __slime.setObject();\n"); + w.write(indent + "__croot.setDouble(\"version\", CONFIG_DEF_SERIALIZE_VERSION);\n"); + w.write(indent + "vespalib::slime::Cursor & __key = __croot.setObject(\"configKey\");\n"); + w.write(indent + "__key.setString(\"defName\", vespalib::slime::Memory(CONFIG_DEF_NAME));\n"); + w.write(indent + "__key.setString(\"defNamespace\", vespalib::slime::Memory(CONFIG_DEF_NAMESPACE));\n"); + w.write(indent + "__key.setString(\"defMd5\", vespalib::slime::Memory(CONFIG_DEF_MD5));\n"); + w.write(indent + "vespalib::slime::Cursor & __keySchema =__key.setArray(\"defSchema\");\n"); + w.write(indent + "for (size_t i = 0; i < CONFIG_DEF_SCHEMA.size(); i++) {\n"); + w.write(indent + " __keySchema.addString(vespalib::slime::Memory(CONFIG_DEF_SCHEMA[i]));\n"); + w.write(indent + "}\n"); + w.write(indent + "vespalib::slime::Cursor & __cursor = __croot.setObject(\"configPayload\");\n"); + } else { + w.write("void\n" + + parent + "serialize(vespalib::slime::Cursor & __cursor) const\n" + + "{\n"); + } + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + w.write(indent + "{\n"); + indent = " "; + w.write(indent + "vespalib::slime::Cursor & __c = __cursor.setObject(\"" + child.getName() + "\");\n"); + if (child.isArray) { + w.write(indent + "__c.setString(\"type\", \"array\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + w.write(indent + "for (size_t __i = 0; __i < " + childName + ".size(); __i++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "[__i]));\n"); + } else { + w.write("(\"value\", " + childName + "[__i]);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " " + childName + "[__i].serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "__c.setString(\"type\", \"map\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + String childMapType = getTypeName(child, true); + w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + w.write(indent + " __c3.setString(\"key\", vespalib::slime::Memory(it->first));\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(it->second)));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(it->second));\n"); + } else { + w.write("(\"value\", it->second);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " it->second.serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else { + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + "__c.setString(\"type\", \"enum\");\n"); + w.write(indent + "__c.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + ")));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + "__c.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + "__c.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "));\n"); + } else { + w.write("(\"value\", " + childName + ");\n"); + } + } else { + w.write(indent + "__c.setString(\"type\", \"struct\");\n"); + w.write(indent + "Cursor & __c2 = __c.setObject(\"value\");\n"); + w.write(indent + childName + ".serialize(__c2);\n"); + } + } + indent = " "; + w.write(indent + "}\n"); + + } + w.write("}\n\n"); + } + + public void writeSlimeDecoder(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigDataBuffer & __buffer)\n" + + "{\n"); + w.write(indent + "const vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Inspector & __croot = __slime.get();\n"); + w.write(indent + "vespalib::slime::Inspector & __inspector = __croot[\"configPayload\"];\n"); + } else { + w.write("" + + parent + typeName + "(const vespalib::slime::Inspector & __inspector)\n" + + "{\n"); + } + + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String inspectorLine = "__inspector[\"" + child.getName() + "\"][\"value\"]"; + if (child.isArray) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + ".push_back("); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(");\n"); + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + "[" + inspectorLine + "[__i][\"key\"].asString().make_string()] = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(";\n"); + w.write(indent + "}\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + ".as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + ".as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + ".as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + ")"); + } + w.write(";\n"); + } + } + w.write("}\n\n"); + } + + public void writeSlimeConstructor(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } else { + w.write("" + + parent + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } + w.write(indent + "const vespalib::slime::Inspector & __inspector(__payload.get());\n"); + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String childInspector = "__inspector[\"" + child.getName() + "\"]"; + if (child.isArray) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::VectorInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else if (child.isMap) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::MapInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + w.write("Internal" + childType + "Converter"); + } else { + w.write("::config::internal::ValueConverter<" + childType + ">"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + String defaultValue = getDefaultValue(leaf); + w.write("()(" + childInspector + ", " + defaultValue + ");\n"); + } else if (child instanceof InnerCNode) { + w.write("()(" + childInspector + ");\n"); + } else { + w.write("()(\"" + child.getName() + "\", " + childInspector + ");\n"); + } + } + } + w.write("}\n\n"); + } + + void writeBodyFooter(Writer w, CNode root) throws IOException { + w.write("} // namespace internal\n\n"); + writeNameSpaceEnd(w, generateCppNameSpace(root)); + } + + String getDefaultValue(LeafCNode leaf) { + String defaultVal = leaf.getDefaultValue().getStringRepresentation(); + if (leaf.getType().equals("string") && defaultVal.equals("null")) + throw new CodegenRuntimeException("Default value null not allowed for C++ config"); + if (leaf.getType().equals("long") && "-9223372036854775808".equals(defaultVal)) { + return "LONG_MIN"; + } else if (leaf.getType().equals("int") && "-2147483648".equals(defaultVal)) { + return "INT_MIN"; + } else { + return defaultVal; + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java new file mode 100644 index 00000000000..f58c202f70f --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java @@ -0,0 +1,274 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + */ +public class DefLine { + private final static Pattern defaultPattern = Pattern.compile("^\\s*default\\s*=\\s*(\\S+)"); + private final static Pattern rangePattern = Pattern.compile("^\\s*range\\s*=\\s*([\\(\\[].*?[\\)\\]])"); + private final static Pattern restartPattern = Pattern.compile("^\\s*restart\\s*"); + private final static Pattern wordPattern = Pattern.compile("\\S+"); + private final static Pattern enumPattern = Pattern.compile("\\s*\\{(\\s*\\w+\\s*)+(\\s*,\\s*\\w+\\s*)*\\s*\\}"); + private final static Pattern enumPattern2 = Pattern.compile("\\s*,\\s*"); + private final static Pattern wordPattern2 = Pattern.compile("\\w+"); + private final static Pattern digitPattern = Pattern.compile("\\d"); + private final static Pattern namePattern = Pattern.compile("\\s*[a-zA-Z0-9_]+\\s*"); + private final static Pattern whitespacePattern = Pattern.compile("\\s+"); + + private String name = null; + private final Type type = new Type(); + + private DefaultValue defaultValue = null; + + private String range = null; + private boolean restart = false; + + String enumString = null; + final String[] enumArray = null; + + private final static Pattern defaultNullPattern = Pattern.compile("^\\s*default\\s*=\\s*null"); + + public DefLine(String line) { + StringBuilder sb = new StringBuilder(line); + int parsed = parseNameType(sb); + sb.delete(0, parsed); + if (type.name.equals("enum")) { + parsed = parseEnum(sb); + sb.delete(0, parsed); + } + + while (sb.length() > 0) { + parsed = parseOptions(sb); + sb.delete(0, parsed); + } + validateName(); + validateReservedWords(); + } + + /** + * Currently (2012-03-05) not used. Ranges are not checked by the + */ + public String getRange() { + return range; + } + + public DefaultValue getDefault() { + return defaultValue; + } + + public String getName() { + return name; + } + + public Type getType() { + return type; + } + + public boolean getRestart() { + return restart; + } + + public String getEnumString() { + return enumString; + } + + public String[] getEnumArray() { + return enumArray; + } + + /** + * Special function that searches through s and returns the index + * of the first occurrence of " that is not escaped. + */ + private String findStringEnd(CharSequence s, int from) { + boolean escaped = false; + for (int i = from; i < s.length(); i++) { + switch (s.charAt(i)) { + case'\\': + escaped = !escaped; + break; + case'"': + if (!escaped) { + return s.subSequence(from, i).toString(); + } + break; + } + } + return null; + } + + + private int parseOptions(CharSequence string) { + Matcher defaultNullMatcher = defaultNullPattern.matcher(string); + Matcher defaultMatcher = defaultPattern.matcher(string); + Matcher rangeMatcher = rangePattern.matcher(string); + Matcher restartMatcher = restartPattern.matcher(string); + + if (defaultNullMatcher.find()) { + throw new IllegalArgumentException("Null default value is not allowed: " + string.toString()); + } else if (defaultMatcher.find()) { + String deflt = defaultMatcher.group(1); + if (deflt.charAt(0) == '"') { + int begin = defaultMatcher.start(1) + 1; + deflt = findStringEnd(string, begin); + if (deflt == null) { + throw new IllegalArgumentException(string.toString()); + } + defaultValue = new DefaultValue(deflt, type); + return begin + deflt.length() + 1; + } else { + defaultValue = new DefaultValue(deflt, type); + } + return defaultMatcher.end(); + } else if (rangeMatcher.find()) { + range = rangeMatcher.group(1); + return rangeMatcher.end(); + } else if (restartMatcher.find()) { + restart = true; + return restartMatcher.end(); + } else { + throw new IllegalArgumentException(string.toString()); + } + } + + private int parseNameType(CharSequence string) { + Matcher wordMatcher = wordPattern.matcher(string); + if (wordMatcher.find()) { + name = wordMatcher.group(); + } + if (wordMatcher.find()) { + type.name = wordMatcher.group(); + } + if (type.name == null || name == null) { + throw new IllegalArgumentException(string.toString()); + } + return wordMatcher.end(); + } + + private int parseEnum(CharSequence string) { + Matcher enumMatcher = enumPattern.matcher(string); + if (enumMatcher.find()) { + enumString = enumMatcher.group(0).trim(); + } + if (enumString == null) { + throw new IllegalArgumentException(string + " is not valid syntax"); + } + enumString = enumString.replaceFirst("\\{\\s*", ""); + enumString = enumString.replaceFirst("\\s*\\}", ""); + String result[] = enumPattern2.split(enumString); + type.enumArray = new String[result.length]; + for (int i = 0; i < result.length; i++) { + String s = result[i].trim(); + type.enumArray[i] = s; + Matcher wordMatcher2 = wordPattern2.matcher(s); + if (!wordMatcher2.matches()) { + throw new IllegalArgumentException(s + " is not valid syntax"); + } + } + return enumMatcher.end(); + } + + public static class Type { + String name; + String[] enumArray; + + public Type(String name) { + this.name=name; + } + + public Type() { + } + + public String getName() { + return name; + } + + public String[] getEnumArray() { + return enumArray; + } + + public Type setEnumArray(String[] enumArray) { + this.enumArray = enumArray; + return this; + } + + public String toString() { + return "type " + name; + } + + } + + // A naive approach to imitate the checking previously done in make-config-preproc.pl + // TODO: method too long + void validateName() { + Matcher digitMatcher; + Matcher nameMatcher; + Matcher whitespaceMatcher; + + boolean atStart = true; + boolean arrayOk = true; + boolean mapOk = true; + for (int i = 0; i < name.length(); i++) { + String s = name.substring(i, i + 1); + digitMatcher = digitPattern.matcher(s); + nameMatcher = namePattern.matcher(s); + whitespaceMatcher = whitespacePattern.matcher(s); + if (atStart) { + if (digitMatcher.matches()) { + throw new IllegalArgumentException(name + " must start with a non-digit character"); + } + if (!nameMatcher.matches()) { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + atStart = false; + } else { + if (nameMatcher.matches()) { + // do nothing + } else if (s.equals(".")) { + arrayOk = true; + mapOk = true; + atStart = true; + } else if (s.equals("[")) { + if (!arrayOk) { + throw new IllegalArgumentException(name + " Arrays cannot be multidimensional"); + } + arrayOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("]"))) { + throw new IllegalArgumentException(name + " Expected ] to terminate array definition"); + } + i++; + } else if (s.equals("{")) { + if (!mapOk) { + throw new IllegalArgumentException(name + " Maps cannot be multidimensional"); + } + mapOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("}"))) { + throw new IllegalArgumentException(name + " Expected } to terminate map definition"); + } + i++; + } else if (whitespaceMatcher.matches()) { + break; + } else { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + } + } + } + + void validateReservedWords() { + if (ReservedWords.isReservedWord(name)) { + throw new IllegalArgumentException(name + " is a reserved word in " + + ReservedWords.getLanguageForReservedWord(name)); + } + if (ReservedWords.capitalizedPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with an uppercase letter"); + } + if (ReservedWords.internalPrefixPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'"); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java new file mode 100644 index 00000000000..0a6c225fa19 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class generates a tree of CNodes from a .def file. + * + * @author gjoranv + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class DefParser { + static final Pattern commentPattern = Pattern.compile("^\\s*#+\\s*(.*?)\\s*$"); + public static final Pattern versionPattern = Pattern.compile("^(version\\s*=\\s*)([0-9][0-9-]*)$"); + // Namespace must start with a letter, since Java (Java language Spec, section 3.8) and C++ identifiers cannot start with a digit + public static final Pattern namespacePattern = Pattern.compile("^(namespace\\s*=\\s*)(([a-z][a-z0-9_]*)+([.][a-z][a-z0-9_]*)*)$"); + + private final BufferedReader reader; + private final String name; + private InnerCNode root = null; + private NormalizedDefinition normalizedDefinition = null; + + + private String comment = ""; + + /** + * Creates a new parser for a .def file with the given name and that can be accessed by the given reader. + * + * @param name The name of the .def file (not including version number and the '.def' suffix). + * @param defReader A reader to the .def file. + */ + public DefParser(String name, Reader defReader) { + this.name = createName(name); + if (defReader == null) { + throw new CodegenRuntimeException("Must have a non-null reader for a .def file."); + } + if (defReader instanceof BufferedReader) { + reader = (BufferedReader)defReader; + } else { + reader = new BufferedReader(defReader); + } + } + + // If name contains namespace, return just name + private String createName(String name) { + if (name.contains(".")) { + return name.substring(name.lastIndexOf(".") + 1); + } else { + return name; + } + } + + /** + * Parses the .def file upon the initial call. Subsequent calls returns the result from the initial call. + * + * @return A tree of CNodes representing this instance's .def file. + * @throws CodegenRuntimeException upon errors. + */ + public InnerCNode getTree() throws CodegenRuntimeException { + try { + if (root == null) parse(); + } catch (DefParserException | IOException e) { + throw new CodegenRuntimeException("Error parsing or reading config definition." + e.getMessage(), e); + } + return root; + } + + /** + * Parses the input from the reader and builds a tree of CNodes representing the .def file. + * + * @throws IOException upon reader errors. + * @throws DefParserException upon parsing errors. + */ + void parse() throws IOException, DefParserException { + root = new InnerCNode(name); + normalizedDefinition = new NormalizedDefinition(); + + String s; + List<String> originalInput = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + originalInput.add(s); + } + reader.close(); + + + // Parse and build tree of the original input + parseLines(root, originalInput, normalizedDefinition); + root.setMd5(normalizedDefinition.generateMd5Sum()); + } + + /** + * Parses one line from def-file and adds it to the tree. + * TODO: Method too long! + * + * @param root The root CNode in the tree. + * @param line A line from the def-file. + * @param nd A NormalizedDefinition object + * @throws IllegalArgumentException upon error in line. + */ + private void parseLine(CNode root, String line, NormalizedDefinition nd) throws IllegalArgumentException { + line = NormalizedDefinition.normalize(line); + line = line.trim(); + if (line.length() == 0) { + // If having empty lines in between comments, that is logically a break in the comment too + if (!comment.isEmpty()) { + comment += "\n"; + } + return; + } + Matcher commentMatch = commentPattern.matcher(line); + if (commentMatch.matches()) { + parseCommentLine(commentMatch); + return; + } + Matcher versionMatch = versionPattern.matcher(line); + if (versionMatch.matches()) { + parseVersionLine(versionMatch); + return; + } + Matcher namespaceMatcher = namespacePattern.matcher(line); + if (namespaceMatcher.matches()) { + parseNamespaceLine(namespaceMatcher.group(2)); + nd.addNormalizedLine(line); + return; + } + // Only add lines that are not version, namespace or comment lines + nd.addNormalizedLine(line); + DefLine defLine = new DefLine(line); + root.setLeaf(root.getName() + "." + defLine.getName(), defLine, comment); + comment = ""; + } + + private void parseCommentLine(Matcher commentMatch) { + if (!comment.isEmpty()) comment += "\n"; + String addition = commentMatch.group(1); + if (addition.isEmpty()) addition = " "; + comment += addition; + } + + private void parseVersionLine(Matcher matcher) { + root.setVersion(matcher.group(2)); + root.setComment(comment); + comment = ""; + } + + private void parseNamespaceLine(String namespace) { + if (namespace.startsWith("com.yahoo.")) + throw new IllegalArgumentException("Remove 'com.yahoo.' from the namespace '" + namespace + + "' - it will be automatically added to the java package name."); + root.setNamespace(namespace); + root.setComment(comment); + comment = ""; + } + + void parseLines(CNode root, List<String> defLines, NormalizedDefinition nd) throws DefParserException { + DefParserException failure = null; + int lineNumber = 1; + for (String line : defLines) { + try { + parseLine(root, line, nd); + lineNumber++; + } catch (IllegalArgumentException e) { + String msg = "Error when parsing line " + lineNumber + ": " + line + "\n" + e.getMessage(); + failure = new DefParserException(msg, e); + break; + } + } + + if (failure != null) { + throw (failure); + } + } + + public NormalizedDefinition getNormalizedDefinition() { + return normalizedDefinition; + } + + /** + * For debugging - dump the tree from the given root to System.out. + */ + public static void dumpTree(CNode root, String indent) { + StringBuilder sb = new StringBuilder(indent + root.getName()); + if (root instanceof LeafCNode) { + LeafCNode leaf = ((LeafCNode)root); + if (leaf.getDefaultValue() != null) { + sb.append(" = ").append(((LeafCNode)root).getDefaultValue().getValue()); + } + } + System.out.println(sb.toString()); + if (!root.getComment().isEmpty()) { + String comment = root.getComment(); + if (comment.contains("\n")) { + comment = comment.substring(0, comment.indexOf("\n")) + "..."; + } + if (comment.length() > 60) { + comment = comment.substring(0, 57) + "..."; + } + System.out.println(indent + " comment: " + comment); + } + CNode[] children = root.getChildren(); + for (CNode c : children) { + dumpTree(c, indent + " "); + } + + } + + class DefParserException extends Exception { + DefParserException(String s, Throwable cause) { + super(s, cause); + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java new file mode 100644 index 00000000000..8518ff04573 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * An immutable class representing a default value of a config variable + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DefaultValue { + + private String value = null; + + // The variable type. Always set UNLESS the value is null. + private DefLine.Type type = null; + + + /** + * Null value + */ + public DefaultValue() { + } + + /** + * A default value with the given value and type. + */ + public DefaultValue(String value, DefLine.Type type) { + this.value = value; + this.type = type; + } + + /** + * Returns the toString of the default value. + */ + public String getValue() { + return value; + } + + /** + * Returns the string representation of this value + */ + public String getStringRepresentation() { + if (value == null) + return "null"; + else if ("bool".equals(type.getName())) + return value; + else if ("int".equals(type.getName())) + return value; + else if ("long".equals(type.getName())) + return value; + else if ("double".equals(type.getName())) + return value; + else if ("enum".equals(type.getName())) + return value; + else { + // building a string, do unicode-escaping + StringBuilder sb = new StringBuilder(); + for (char c : value.toCharArray()) { + if (c > '\u007f') { + sb.append(String.format("\\u%04X", (int) c)); + } else { + sb.append(c); + } + } + return "\"" + sb.toString() + "\""; + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java new file mode 100644 index 00000000000..b7803fc8c22 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.Map; +import java.util.LinkedHashMap; + +/** + * Represents an inner node in the configuration tree. + * + * @author gjoranv + */ +public class InnerCNode extends CNode { + + /** + * The children of this Node. Mapped using their short name as + * string. This variable is null only if Node is a leaf Node. + */ + private final Map<String, CNode> children = new LinkedHashMap<String, CNode>(); + private boolean restart = false; + + /** + * Constructor for the root node. + */ + public InnerCNode(String name) { + super(null, name.split("\\.def")[0]); + defName = this.name; + } + + /** + * Constructor for inner nodes. + */ + private InnerCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public CNode[] getChildren() { + CNode[] ret = new CNode[children.size()]; + children.values().toArray(ret); + return ret; + } + + /** + * Access to children for testing + * @return modifiable children map + */ + public Map<String, CNode> children() { + return children; + } + + @Override + public CNode getChild(String name) { + return children.get(name); + } + + /** + * Returns and eventually creates the given subnode. + */ + private CNode createOrGetChild(DefLine.Type type, String name) throws IllegalArgumentException { + String key = name; + int split = name.indexOf('.'); + CNode newChild; + if (split != -1) { + key = name.substring(0, split).trim(); + newChild = new InnerCNode(this, key); + } else { + newChild = LeafCNode.newInstance(type, this, key); + if (newChild == null) + throw new IllegalArgumentException + ("Could not create " + type.name + " " + name); + } + return children.containsKey(newChild.getName()) + ? children.get(newChild.getName()) + : newChild; + } + + /** + * Adds a child to this node with the given type, name and value. Necessary children on the path + * to the given leaf node will be added as well. + * @param name the full name/path of the node to add. + * @param defLine the parsed .def-file line to add. + * @param comment comment extracted from the .def-file. + */ + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + if (name.indexOf('.') < 0) { + throw new IllegalArgumentException("Parameter with name '" + name + + "' cannot be a leaf node as it has already been declared as an inner node."); + } + checkMyName(name.substring(0, name.indexOf('.'))); + String childName = name.substring(name.indexOf('.') + 1); + + CNode child = createOrGetChild(defLine.getType(), childName); +/* + System.out.println("\nAdding child name: " + name); + System.out.println(" getName: " + child.getName()); + System.out.println(" full name: " + child.getFullName()); + System.out.println(" classname: " + child.getClassName()); + System.out.println(" full classname: " + child.getFullClassName()); +*/ + restart |= defLine.getRestart(); + child.setLeaf(childName, defLine, comment); + children.put(child.getName(), child); + } + + @Override + public boolean needRestart() { + return restart; + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java new file mode 100644 index 00000000000..3cd7b61ba55 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java @@ -0,0 +1,267 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author gjoranv + */ +public abstract class LeafCNode extends CNode { + + private boolean isInitialized = false; + private DefaultValue defaultValue = null; + private boolean restart = false; + + /** + * Constructor for the leaf nodes. + */ + protected LeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name) { + try { + switch (type.name) { + case "int": + return new IntegerLeaf(parent, name); + case "long": + return new LongLeaf(parent, name); + case "double": + return new DoubleLeaf(parent, name); + case "bool": + return new BooleanLeaf(parent, name); + case "string": + return new StringLeaf(parent, name); + case "reference": + return new ReferenceLeaf(parent, name); + case "file": + return new FileLeaf(parent, name); + case "path": + return new PathLeaf(parent, name); + case "enum": + return new EnumLeaf(parent, name, type.enumArray); + default: + return null; + } + } catch (NumberFormatException e) { + return null; + } + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name, String defVal) { + LeafCNode ret = newInstance(type, parent, name); + if (defVal!=null) { + DefaultValue def = new DefaultValue(defVal, type); + ret.setDefaultValue(def); + } + return ret; + } + + public abstract String getType(); + + @Override + public CNode[] getChildren() { + return new CNode[0]; + } + + @Override + public CNode getChild(String name) { + return null; + } + + public DefaultValue getDefaultValue() { + return defaultValue; + } + + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + /** + * @param defaultValue The value to check. + * @throws IllegalArgumentException if the value is illegal according to the node type. + */ + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + } + + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + DefLine.Type type = defLine.getType(); + // TODO: why the !is... conditions? + if (!isMap && !isArray && isInitialized) { + throw new IllegalArgumentException(name + " is already defined"); + } + isInitialized = true; + checkMyName(name); + if (!type.name.equalsIgnoreCase(getType())) { + throw new IllegalArgumentException("Type " + type.name + " does not match " + getType()); + } + setValue(defLine.getDefault()); + setComment(comment); + restart |= defLine.getRestart(); + } + + @Override + public boolean needRestart() { + return restart; + } + + public final void setValue(DefaultValue defaultValue) throws IllegalArgumentException { + try { + checkDefaultValue(defaultValue); + setDefaultValue(defaultValue); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException + ("Invalid default value", e); + } + } + + /** + * Superclass for leaf nodes that should not generate class. + */ + public static abstract class NoClassLeafCNode extends LeafCNode { + protected NoClassLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + } + + /** + * Superclass for no-class leaf nodes that cannot have a default. + */ + public static abstract class NoClassNoDefaultLeafCNode extends LeafCNode { + protected NoClassNoDefaultLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + if (defaultValue != null) + throw new IllegalArgumentException("Parameters of type '" + getType() + "' cannot have a default value."); + return this; + } + } + + public static class IntegerLeaf extends NoClassLeafCNode { + protected IntegerLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "int"; + } + } + + public static class LongLeaf extends NoClassLeafCNode { + protected LongLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "long"; + } + } + + public static class DoubleLeaf extends NoClassLeafCNode { + protected DoubleLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "double"; + } + } + + public static class BooleanLeaf extends NoClassLeafCNode { + protected BooleanLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "bool"; + } + } + + public static class StringLeaf extends NoClassLeafCNode { + protected StringLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "string"; + } + } + + public static class ReferenceLeaf extends StringLeaf { + ReferenceLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "reference"; + } + } + + public static class FileLeaf extends NoClassNoDefaultLeafCNode { + FileLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "file"; + } + } + + public static class PathLeaf extends NoClassNoDefaultLeafCNode { + PathLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "path"; + } + } + + public static class EnumLeaf extends LeafCNode { + + private final String[] legalValues; + + protected EnumLeaf(InnerCNode parent, String name, String[] valArray) { + super(parent, name); + this.legalValues = valArray; + } + + @Override + public String getType() { + return "enum"; + } + + /** @return This enum's legal values. */ + public String[] getLegalValues() { + return legalValues; + } + + @Override + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + if ((defaultValue != null) && (defaultValue.getValue() != null)) { + String defaultString = null; + String value = defaultValue.getValue(); + for (String val : legalValues) { + if (value.equals(val)) { + defaultString = val; + } + } + if (defaultString == null) + throw new IllegalArgumentException("Could not initialize enum with: " + value); + } + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java new file mode 100644 index 00000000000..f28549275ef --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.logging.Logger; + +/** + * This class generates code for a config class from a given def-file. + */ +public class MakeConfig { + + private final static Logger log = Logger.getLogger(MakeConfig.class.getName()); + + private final ClassBuilder classBuilder; + + public MakeConfig(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties properties) { + classBuilder = createClassBuilder(root, nd, path, properties); + } + + public static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties prop) { + if (prop.language.equals("cppng") || prop.language.equals("cpp")) + return new CppClassBuilder(root, nd, prop.destDir, prop.dirInRoot); + else + return new JavaClassBuilder(root, nd, prop.destDir); + } + + /** + * Generates the code and print it to this.out. + */ + void buildClasses() { + classBuilder.createConfigClasses(); + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java -Dconfig.dest=<dir> -Dconfig.spec=<path> [-Dconfig.lang=cpp -Dconfig.subdir=<dir>] [-Dconfig.dumpTree=true] MakeConfig"); + out.println(" (default language for generated code is Java)"); + } + + public static void main(String[] args) throws IOException, InterruptedException { + try { + MakeConfigProperties props = new MakeConfigProperties(); + for (File specFile : props.specFiles) { + String path = specFile.toURI().toString(); + String name = specFile.getName(); + if (name.endsWith(".def")) name = name.substring(0, name.length() - 4); + DefParser parser = new DefParser(name, new FileReader(specFile)); + InnerCNode configRoot = parser.getTree(); + checkNamespace(name, configRoot); + if (configRoot != null) { + MakeConfig mc = new MakeConfig(configRoot, parser.getNormalizedDefinition(), path, props); + mc.buildClasses(); + if (props.dumpTree) { + System.out.println("\nTree dump:"); + DefParser.dumpTree(configRoot, ""); + } + } else { + System.exit(1); + } + } + } catch (PropertyException e) { + System.out.println(Exceptions.toMessageString(e)); + printUsage(System.err); + System.exit(1); + } catch (CodegenRuntimeException e) { + System.out.println(Exceptions.toMessageString(e)); + System.exit(1); + } + } + + private static void checkNamespace(String name, InnerCNode configRoot) { + if (configRoot.defNamespace == null) + throw new IllegalArgumentException("In config definition '" + name + "': A namespace is required"); + } + + // The Exceptions class below is copied from vespajlib/com.yahoo.protect.Exceptions + + /** + * Helper methods for handling exceptions + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ + static class Exceptions { + + /** + * <p>Returns a use friendly error message string which includes information from all nested exceptions. + * + * <p>The form of this string is + * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> + * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. + */ + public static String toMessageString(Throwable t) { + StringBuilder b = new StringBuilder(); + String lastMessage = null; + String message; + for (; t != null; t = t.getCause(), lastMessage = message) { + message = getMessage(t); + if (message == null) continue; + if (lastMessage != null && lastMessage.equals(message)) continue; + if (b.length() > 0) + b.append(": "); + b.append(message); + } + return b.toString(); + } + + /** + * Returns a useful message from *this* exception, or null if none + */ + private static String getMessage(Throwable t) { + String message = t.getMessage(); + if (t.getCause() == null) { + if (message == null) return toShortClassName(t); + return message; + } else { + if (message == null) return null; + if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; + return message; + } + } + + private static String toShortClassName(Object o) { + String longName = o.getClass().getName(); + int lastDot = longName.lastIndexOf("."); + if (lastDot < 0) return longName; + return longName.substring(lastDot + 1); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java new file mode 100644 index 00000000000..d055a1fe1fb --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.StringTokenizer; + +/** + * Encapsulates data extracted from system properties. + * + * @author <a href="gv@yahoo-inc.com">Gjoran Voldengen</a> + */ +public class MakeConfigProperties { + + private final List<String> legalLanguages = Arrays.asList("java", "cpp", "cppng" ); + + final File destDir; + final File[] specFiles; + final String language; + final String dirInRoot; // Where within fileroot to store generated class files + final boolean dumpTree; + final boolean generateFrameworkCode; + + MakeConfigProperties() throws PropertyException { + destDir = checkDestinationDir(); + specFiles = checkSpecificationFiles(); + language = checkLanguage(); + dirInRoot = checkDirInRoot(); + dumpTree = System.getProperty("config.dumpTree") != null && + System.getProperty("config.dumpTree").equalsIgnoreCase("true"); + generateFrameworkCode = System.getProperty("config.useFramework") == null || + System.getProperty("config.useFramework").equalsIgnoreCase("true"); + } + + private File checkDestinationDir() throws PropertyException { + String destination = System.getProperty("config.dest"); + if (destination == null) + throw new PropertyException("Missing property: config.dest."); + + File dir = new File(destination); + if (!dir.isDirectory()) { + throw new PropertyException("Could not find directory: " + dir.getPath()); + } + return dir; + } + + private String checkDirInRoot() throws PropertyException { + String dirInRoot = System.getProperty("config.subdir"); + // Optional parameter + if (dirInRoot == null) { return null; } + File f = new File(destDir, dirInRoot); + if (!f.isDirectory()) { + throw new PropertyException("Could not find directory: " + f.getPath()); + } + return dirInRoot; + } + + /** + * @return Desired programming language of generated code, default is "java". + * @throws PropertyException if supplied language is not a legal language. + */ + private String checkLanguage() throws PropertyException { + String inputLang = System.getProperty("config.lang", "java").toLowerCase(); + if (! legalLanguages.contains(inputLang)) { + throw new PropertyException + ("Unsupported code language: '" + inputLang + "'. Supported languages are: " + legalLanguages); + } + return inputLang; + } + + private static File[] checkSpecificationFiles() throws PropertyException { + String string = System.getProperty("config.spec"); + if (string == null) + throw new PropertyException("Missing property: config.spec."); + + StringTokenizer st = new StringTokenizer(string, ","); + if (st.countTokens() == 0) + throw new PropertyException("Missing property: config.spec."); + + File[] files = new File[st.countTokens()]; + for (int i = 0; st.hasMoreElements(); i++) { + files[i] = new File((String) st.nextElement()); + if (!files[i].isFile()) + throw new PropertyException("Could not read file " + files[i].getPath()); + } + return files; + } + +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java new file mode 100644 index 00000000000..1847150d86d --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.text.DecimalFormat; +import java.security.MessageDigest; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * + * Does normalizing (removing comments, trimming whitespace etc.) and calculation of md5sum + * of config definitions + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class NormalizedDefinition { + /* Patterns used for finding ranges in config definitions */ + private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); + private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); + private MessageDigest md5; + + String defMd5 = null; + List<String> normalizedContent = null; + + public NormalizedDefinition() { + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (java.security.NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to create MD5 digest", e); + } + normalizedContent = new ArrayList<>(); + } + + public NormalizedDefinition normalize(BufferedReader reader) throws IOException { + String s; + List<String> input = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + String normalized = normalize(s); + if (normalized.length() > 0) { + input.add(normalized); + } + } + normalizedContent = input; + return this; + } + + /** + * Normalizes a config definition line. Each string is normalized according to the + * rules of config and definition files before they are used: + * <ul> + * <li>Remove trailing space.<li> + * <li>Remove trailing comments, and spaces before trailing comments.</li> + * <li>Remove empty lines</li> + * <li>Keep comment lines</li> + * </ul> + * The supplied list is changed in-place + * + * @param line A config definition line + * @return a normalized config definition line + */ + public static String normalize(String line) { + //System.out.println("before line=" + line + ";"); + // Normalize line + line = line.trim(); + Matcher m = intPattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(0x7fffffff); + String formattedMin = new DecimalFormat("#.#").format(-0x80000000); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + m = doublePattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(1e308); + String formattedMin = new DecimalFormat("#.#").format(-1e308); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + line = removeComment(line); + if (!line.isEmpty()) { + line = stripSpaces(line); + line = line.replaceAll("\\s,", ","); // Remove space before comma (for enums) + line += "\n"; + } + //System.out.println("after line=" + line + ";"); + return line; + } + + // Removes comment char and text after it, unless comment char is inside a string + // Keeps comment lines (lines that start with #) + private static String removeComment(String line) { + int index = line.indexOf("#"); + if (!line.contains("#") || index == 0) return line; + + int firstQuote = line.indexOf("\""); + if (firstQuote > 0) { + int secondQuote = line.indexOf("\"", firstQuote + 1); + if (index > secondQuote) { + line = line.substring(0, index); + line = line.trim(); + } + } else { + line = line.substring(0, index); + line = line.trim(); + } + + return line; + } + + public void addNormalizedLine(String line) { + normalizedContent.add(line); + } + + public String generateMd5Sum() { + for (String line : normalizedContent) { + String s = normalize(line); + if (!s.isEmpty()) { + md5.update(toBytes(s)); + } + } + defMd5 = toHexString(md5.digest()).toLowerCase(); + //System.out.println("md5=" + defMd5) ; + return defMd5; + } + + + // The two methods below are copied from vespajlib (com.yahoo.text.Utf8 and com.yahoo.io.HexDump) + // since configgen cannot depend on any other modules (at least not as it is done now) + public static byte[] toBytes(String str) { + Charset charset = Charset.forName("utf-8"); + + ByteBuffer b = charset.encode(str); + byte[] result = new byte[b.remaining()]; + b.get(result); + return result; + } + + private String toHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte aByte : bytes) { + sb.append(String.format("%02x", aByte)); + } + return sb.toString(); + } + + /** + * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * @param str String to strip spaces from + * @return String with spaces stripped + */ + public static String stripSpaces(String str) { + StringBuilder ret = new StringBuilder(""); + boolean inQuotes = false; + boolean inSpaceSequence = false; + for (char c : str.toCharArray()) { + if (Character.isWhitespace(c)) { + if (inQuotes) { + ret.append(c); + continue; + } + if (!inSpaceSequence) { + // start of space sequence + inSpaceSequence=true; + ret.append(" "); + } + } else { + if (inSpaceSequence) { + inSpaceSequence=false; + } + if (c=='\"') { + inQuotes=!inQuotes; + } + ret.append(c); + } + } + return ret.toString(); + } + + public List<String> getNormalizedContent() { + return normalizedContent; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String line : normalizedContent) { + builder.append(line.replace("\"", "\\\"")); + builder.append("\\n\\\n"); + } + return builder.toString(); + } + + public String getDefMd5() { + return defMd5; + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java new file mode 100644 index 00000000000..a42f76340de --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +public class PropertyException extends Exception { + PropertyException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java new file mode 100644 index 00000000000..145f2da3245 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.HashMap; +import java.util.regex.Pattern; + +/** + * Reserved words that cannot be used as variable names in a config definition file. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 2009-06-24 + */ + +public class ReservedWords { + + public static final String INTERNAL_PREFIX = "__"; + final static Pattern internalPrefixPattern = Pattern.compile("^" + INTERNAL_PREFIX + ".*"); + final static Pattern capitalizedPattern = Pattern.compile("^[A-Z].*"); + + private static final String[] cKeywords = + {"asm", "auto", "bool", "break", "case", "catch", + "char", "class", "const", "const_cast", "continue", "default", + "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", + "export", "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "item", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", "static_cast", + "struct", "switch", "template", "this", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", + "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", + "bitand"}; + + private static final String[] javaKeywords = + {"abstract", "boolean", "break", "byte", "case", + "catch", "char", "class","continue", "default", "do", "double", + "else", "extends","false", "final", "finally", "float", "for", + "if","implements", "import", "instanceof", "int", "interface", + "item", "long","native", "new", "null", "package", "private", + "protected","public", "return", "short", "static", + "strictfp","super","switch", "synchronized", "this", + "throw","throws","transient", "true", "try", "void", + "volatile","while", "byvalue", "cast", "const", "future", + "generic","goto", "inner", "operator", "outer", "rest", "var"}; + + private static final HashMap<String, String> allKeyWords; + + static { + allKeyWords = new HashMap<String, String>(); + for (String s : cKeywords) { + allKeyWords.put(s, "C"); + } + for (String s : javaKeywords) { + if (allKeyWords.containsKey(s)) { + allKeyWords.put(s, "C and Java"); + } else { + allKeyWords.put(s, "Java"); + } + } + } + + + public static boolean isReservedWord(String word) { + return allKeyWords.containsKey(word); + } + + public static String getLanguageForReservedWord(String word) { + return allKeyWords.get(word); + } + + +} diff --git a/configgen/src/main/manifest.mf b/configgen/src/main/manifest.mf new file mode 100644 index 00000000000..a998997f7c2 --- /dev/null +++ b/configgen/src/main/manifest.mf @@ -0,0 +1,9 @@ +Manifest-Version: 1.0 +Export-Package: com.yahoo.config.codegen +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/annotation-6-SNAPSHOT.jar,dependenc + ies/scala-library-2.9.1.jar +Bundle-ManifestVersion: 2 +Bundle-Name: vespa config generator +Bundle-SymbolicName: configgen +Main-Class: com.yahoo.config.codegen.MakeConfig diff --git a/configgen/src/main/resources/make-config-preproc.pl b/configgen/src/main/resources/make-config-preproc.pl new file mode 100755 index 00000000000..507b426457b --- /dev/null +++ b/configgen/src/main/resources/make-config-preproc.pl @@ -0,0 +1,952 @@ +#!/usr/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# This is the config pre-processor. +# It handles import statements, and does syntax checking etc. +# The idea is that it will be called directly from the script +# that does the code generation. +# +# Errors and warnings are printed in "next-error" compatible ways +# for emacs etc. +# +# Indented like this: +# (cperl-set-style "Whitesmith") +# (setq cperl-continued-brace-offset -4) + +require 5.006_001; +use strict; +use warnings; +use Digest::MD5; + +use Math::BigInt; +use Math::BigFloat; + +die "Usage: $0 <def-file>" unless $#ARGV == 0; + +my $defname = $ARGV[0]; + +my $md5 = Digest::MD5->new; + +my @c_keywords = + ("asm", "auto", "bool", "break", "case", "catch", + "char", "class", "const", "const_cast", "continue", "default", + "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", + "export", "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", "static_cast", + "struct", "switch", "template", "this", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", + "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", + "bitand"); + + +my @java_keywords = + ("abstract", "boolean", "break", "byte", "case", + "catch", "char", "class","continue", "default", "do", "double", + "else", "extends","false", "final", "finally", "float", "for", + "if","implements", "import", "instanceof", "int", "interface", + "long","native", "new", "null", "package", "private", + "protected","public", "return", "short", "static", + "strictfp","super","switch", "synchronized", "this", + "throw","throws","transient", "true", "try", "void", + "volatile","while", "byvalue", "cast", "const", "future", + "generic","goto", "inner", "operator", "outer", "rest", "var"); + +my %reserved_words; + +foreach my $word (@c_keywords) { + $reserved_words{$word} = "C"; +} + +foreach my $word (@java_keywords) { + my $x = $reserved_words{$word}; + if (defined($x)) { + $x = "$x, Java"; + } else { + $x = "Java"; + } + $reserved_words{$word} = $x; +} + +my $MIN_INT = -0x80000000; +my $MAX_INT = 0x7fffffff; +my $MIN_DOUBLE = -1e308; +my $MAX_DOUBLE = 1e308; + + +sub do_file { + my ($file, $prefix, $strip) = @_; + + local *FH; + open FH, "< $file" or die "Cannot open $file: $!\n"; + + local *COPY; + my $dir = $ENV{"VESPA_CONFIG_DEF_DIR"}; + my $copy; + my $file_version; + if (defined($dir)) { + $copy = $file; + $copy =~ s=.*/==; + $copy = "$dir/$copy"; + open COPY, ">$copy.new" or die "Cannot open file $copy.new: $!\n"; + } + + # Read line by line. + # 1. Strip away comments and trailing blanks + # 2. Report any errors + # 3. Handle import statements, disallow multi-level imports + # 4. Print everyting to stdout + + my $linenr = 0; + my $written_lines = 0; + my $quoted_strip = quotemeta($strip); + my $seen_version = 0; + + while (<FH>) { + print COPY $_ if $copy; + ++$linenr; + my $line = $_; + chomp $line; + + # Don't process comments or add them to md5 checksum, but print them + # such that codegen can include comments + if ($line =~ /^\s*#/) { + print "$line\n"; + next; + } + + # Strip away comments that are not at start of line + $line = &strip_trailing_comment($line, $linenr) + if ($line =~ m=[\\\#]=); + + if ($line eq "::error::") { + return -1; + } + + # Skip lines that are only whitespace + next if $line =~ m=^\s*$=; + + # Get rid of trailing whitespace + $line =~ s=\s+$==; + + if (!$seen_version) { + if ($line =~ m!^version=([a-zA-Z0-9][-a-zA-Z0-9_/?]*)!) { + $file_version = $1; + $seen_version = 1; + if ($prefix) { + print "$prefix imported $file"; + print ":$strip" if $strip; + print " "; + } + print "$line\n"; + next; + } else { + print STDERR "$file:$linenr: error: Definition file does not " + . "start with a valid version= identifier!\n"; + return -1; + } + } + + if ($strip) { + next unless $line =~ m=^${quoted_strip}[. \t]=; + } + + if (&check_syntax($line, $linenr, $file) == -1) { + return -1; + } + + # Handle import statements + my ($name, $type, $remains, $junk) = split(/\s+/, $line, 4); + if ($type eq "import") { + if ($strip || $prefix) { + my $col = index($line, $type, length("$name ")); + print STDERR "$file:$linenr:$col: error: Multi-level " + . "imports are disallowed.\n"; + return -1; + } + if ($junk) { + my $col = index($line, $junk, length("$name $type $remains")) + + 1; + print STDERR "$file:$linenr:$col: error: Junk after import " + . "target \"$remains\": \"$junk\"\n"; + return -1; + } + my ($impfile, $var) = split(/:/, $remains, 2); + $var = "" unless $var; # Make it defined. + + # Make sure only arrays can include arrays: + if ($name =~ m=\[\]$= && (!$var || $var !~ m=\[\]$=)) { + print STDERR "$file:$linenr: error: Array cannot import " + . "non-array in: $line\n"; + return -1; + } elsif ($name !~ m=\[\]$= && ($var && $var =~ m=\[\]$=)) { + print STDERR "$file:$linenr: error: Non-array cannot import " + . "array in: $line\n"; + return -1; + } + + local *X; + unless (open(X, "< $impfile")) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Cannot open " + . "\"$impfile\": $!\n"; + return -1; + } + close X; + my $imported_lines = &do_file("$impfile", "$name", "$var"); + if ($imported_lines == -1) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Imported from here " + . "as: $line\n"; + return -1; + } elsif ($imported_lines == 0) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Import target " + . "\"$var\" not found in \"$impfile\"\n"; + return -1; + } + $written_lines += $imported_lines; + } else { + ++$written_lines; + if ($strip) { + $line =~ s=^${quoted_strip}=${prefix}= + } elsif ($prefix) { + $line = $prefix . "." . $line; + } + + if (&check_name_sanity($line, $linenr, $file) == -1 + || &check_enum_sanity($line, $linenr, $file) == -1) { + return -1; + } + + $line = &normalize_line($line, $linenr); + if ($line eq "::error::") { + return -1; + } + print $line . "\n"; + } + # Add this line to the md5 checksum + $md5->add("$line\n") unless $prefix; + } + + print "md5=" . $md5->hexdigest . "\n" unless $prefix; + close FH; + if ($copy) { + close COPY; + # We have made a copy. It needs a new name.. + my $new_name = $copy; + $new_name =~ s=\.def==; + $new_name .= ".${file_version}.def"; + if (-f $new_name) { + system "cmp $copy.new $new_name 2>/dev/null" and die "$file:1: error: Definition file $file differs from ${new_name}!\n"; + unlink("$copy.new"); + } else { + rename("$copy.new", "$new_name") or die "Rename $copy.new -> $new_name failed: $!\n"; + } + } + return $written_lines; +} + +sub normalize_enum { + my($x, $linenr, $colnr) = @_; + my $len = length($x); + my $char = ''; + my $output = '{ '; + my $index; + my %enum = (); + my $current_variable = ''; + for ($index = $colnr + 1; $index < $len; ++$index) { + $char = substr($x, $index, 1); + if ($char eq '}') { + if (length($current_variable) < 2) { + print STDERR "$defname:$linenr:$index: error: ". + " variable must be at least two characters: $x\n" ; + return ('', 0); + } elsif ($enum{$current_variable}) { + print STDERR "$defname:$linenr:$index: error: ". + " enum variable declared twice: $x\n" ; + return ('', 0); + } elsif (!%enum && !$current_variable) { + print STDERR "$defname:$linenr:$index: error: ". + " enum cannot be empty: $x\n" ; + return ('', 0); + } + return ($output.$current_variable." } ", $index); + } elsif ($char eq ',') { + if (length($current_variable) < 2) { + print STDERR "$defname:$linenr:$index: error: ". + " variable must be at least two characters: $x\n" ; + return ('', 0); + } elsif ($enum{$current_variable}) { + print STDERR "$defname:$linenr:$index: error: ". + " enum variable declared twice: $x\n" ; + return ('', 0); + } + $enum{$current_variable} = 1; + $output .= "$current_variable, "; + $current_variable = ''; + } elsif ($char =~ m=[A-Z]=) { + $current_variable .= $char; + } elsif ($char =~ m=[0-9_]= && $current_variable) { + $current_variable .= $char; + } elsif ($char =~ m=\s=) { + if ($current_variable && !($x =~ /^.{$index}\s*[,\}]/)) { + print STDERR "$defname:$linenr:$index: error: ". + "expected ',' or '}': $x\n" ; + return ("", 0); + } else { + # skip whitespace + } + } else { + print STDERR "<$char> <$current_variable>\n"; + + print STDERR "$defname:$linenr:$index: error: ". + "Enum must match [A-Z][A-Z0-9_]+: $x\n"; + } + } + return ($output, $index); +} + +{ package Range; + + $Range::DOUBLE_RANGE = + new Range("a double range=[$MIN_DOUBLE,$MAX_DOUBLE] ",0,14); + $Range::INT_RANGE = new Range("a int range=[$MIN_INT,$MAX_INT] ",0,11); + + + sub in_range { + my($self, $value) = @_; + + if ($value =~ s/KB$//) { + $value *= 1024; + } elsif ($value =~ s/MB$//) { + $value *= (1024 * 1024); + } elsif ($value =~ s/GB$//) { + $value *= (1024*1024*1024); + } elsif ($value =~ s/k$//) { + $value *= 1000; + } elsif ($value =~ s/M$//) { + $value *= 1_000_000; + } elsif ($value =~ s/G$//) { + $value *= 1_000_000_000; + } elsif ($value =~ m=^0[xX]=) { + $value = hex($value); + } + + if ($self->{start_bracket} eq '(' ) { + return 0 if $value <= $self->{min}; + } elsif ($self->{start_bracket} eq '[' ) { + return 0 if $value < $self->{min}; + } else { + print STDERR "Illegal start_bracket '$self->{start_bracket}'\n"; + return undef; + } + if ($self->{end_bracket} eq ')' ) { + return 0 if $value >= $self->{max}; + } elsif ($self->{end_bracket} eq ']' ) { + return 0 if $value > $self->{max}; + } else { + print STDERR "Illegal end_bracket '$self->{start_bracket}'\n"; + return undef; + } + return 1; + } + + + sub new { + my($class, $x, $linenr, $colnr) = @_; + my $len = length($x); + my $self = {}; + bless($self, $class); + $self->{min_value} = ''; + my $index; + for ($index = $colnr + 1; $index < $len; ++$index) { + my $char = substr($x, $index, 1); + if (($char eq '(' || $char eq '[') && !$self->{start_bracket}) { + $self->{start_bracket} = $char; + } elsif (($char eq ')' || $char eq ']') && !$self->{end_bracket}) { + $self->{end_bracket} = $char; + last; + } elsif ($char =~ m=\s=) { + #ignore whitespace + } elsif ($char eq ',' && !defined($self->{max_value})) { + $self->{max_value} = ''; + } elsif ($char =~ m=[\d\.\+eE-]= ) { + (defined($self->{max_value}) + ? $self->{max_value} : $self->{min_value}) .= $char; + } else { + print STDERR "$defname:$linenr:$index: error: ". + " syntax error: $x\n" ; + return undef; + } + } + if ($self->{min_value} eq '' && $self->{max_value} eq '') { + print STDERR "$defname:$linenr:$colnr: error: ". + " range cannot be unbounded in both ends: $x\n" ; + return undef; + } + unless ($self->{start_bracket} && $self->{end_bracket}) { + print STDERR "$defname:$linenr:$colnr: error: ". + " missing bracket: $x\n" ; + return undef; + } + + + my @arr = split(/\s+/, $x, 3); + if ($arr[1] eq 'int') { + $self->{min} = Math::BigInt->new + ($self->{min_value} eq '' ? $MIN_INT : $self->{min_value}); + unless (defined($self->{min}) && $self->{min} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{min_value}: $x\n" ; + return undef; + } + my $min_val = + $self->{min} + ($self->{start_bracket} eq '('? 1 : 0); + + $self->{max} = Math::BigInt->new + ($self->{max_value} eq '' ? $MAX_INT : $self->{max_value}); + unless (defined($self->{max}) && $self->{max} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{max_value}: $x\n" ; + return undef; + } + my $max_val = + $self->{max} - ($self->{end_bracket} eq ')'? 1 : 0); + + if ($min_val < $MIN_INT ) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval less than MIN_INT: $x\n" ; + return undef; + } + if ($max_val > $MAX_INT) { + print STDERR "$self->{max} - 1 > $MAX_INT\n"; + print STDERR "$defname:$linenr:$colnr: error: ". + " end of interval greater than MAX_INT: $x\n" ; + return undef; + } + if ($max_val < $min_val) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + $self->{string} = + "$self->{start_bracket}$self->{min},$self->{max}$self->{end_bracket}"; + $self->{string} =~ s/\+//g; + $self->{index} = $index; + return $self; + } elsif ($arr[1] eq 'double') { + $self->{min} = Math::BigFloat->new + ($self->{min_value} eq '' ? $MIN_DOUBLE : $self->{min_value}); + unless (defined($self->{min}) && $self->{min} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{min_value}: $x\n" ; + return undef; + } + $self->{max} = Math::BigFloat->new + ($self->{max_value} eq '' ? $MAX_DOUBLE : $self->{max_value}); + unless (defined($self->{max}) && $self->{max} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{max_value}: $x\n" ; + return undef; + } + if ($self->{min} < $MIN_DOUBLE) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval less than MIN_DOUBLE: $x\n" ; + return undef; + } + if ($self->{max} > $MAX_DOUBLE) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval greater than MAX_DOUBLE: $x\n" ; + return undef; + } + if ($self->{max} < $self->{min}) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + if (($self->{start_bracket} eq '(' || $self->{end_bracket} eq ')') + && ($self->{min_value} + $self->{min_value} + >= $self->{min_value} + $self->{max_value}) + && ($self->{max_value} + $self->{max_value} + <= $self->{min_value} + $self->{max_value})) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + $self->{string} = $self->{start_bracket}.$self->{min}->fnorm. + ','.$self->{max}->fnorm.$self->{end_bracket}; + $self->{string} =~ s/\+//g; + $self->{index} = $index; + return $self; + } else { + print STDERR "$defname:$linenr:$colnr: error: ". + " range-option works only for type 'int' and 'double': $x\n" ; + return undef; + } + print STDERR "$defname:$linenr:$colnr: error: ". + " script error: $x\n" ; + return undef; + + } + +} + + + + +sub strip_trailing_comment { + my ($x, $linenr) = @_; + + my $index = 0; + my $len = length($x); + my $in_quotes = 0; + + # ### Support both " and ' quotes maybe? + + for ($index = 0; $index < $len; ++$index) { + if (substr($x, $index, 1) eq "\\") { + ++$index; + next; + } + if (substr($x, $index, 1) eq "\"") { + $in_quotes ^= 1; + } + if ($in_quotes == 0 && substr($x, $index, 1) eq "#") { + if (!(substr($x, $index - 1, 1) =~ m=\s=)) { + my $col = $index + 1; + print STDERR "$defname:$linenr:$col: warning: No whitespace " + . "before comment in line: $x\n"; + } + print substr($x, $index). "\n"; + $x = substr($x, 0, $index); + last; + } + } + if ($index > $len) { + print STDERR "$defname:$linenr:$len: error: syntax error, line " + . "ends with \\: \"$x\"\n"; + return "::error::"; + } + + return $x; +} + +sub normalize_line { + my ($x, $linenr) = @_; + + my $index = 0; + my $len = length($x); + my $in_quotes = 0; + my $char = ''; + my $output = ''; + my %hash = (); + + my @arr = split(/\s+/, $x, 3); + $hash{type} = $arr[1]; + + for ($index = 0; $index < length($x); ++$index) { + $char = substr($x, $index, 1); + if ($char eq "\\") { + $output .= substr($x, $index, 2); + ++$index; + next; + } + if ($char eq "\"") { + $in_quotes ^= 1; + $output .= $char; + next; + } + my $ends_with_whitespace = ($output =~ m= $=); + + if ($in_quotes == 0) { + if ($char =~ m=\s=) { + #delete multiple spaces + if (!$ends_with_whitespace) { # && ($output =~ !m=\=$=)) { + $output .= ' '; + } + } elsif ($char eq '{') { + my($enum, $i) = &normalize_enum($x, $linenr, $index); + return "::error::" unless $i; + $index = $i; + $output .= ($ends_with_whitespace) ? $enum : " $enum "; + } elsif ($char eq ',') { + chop $output if ($ends_with_whitespace); + $output .= ','; + } elsif ($char eq '=') { + chop $output if ($ends_with_whitespace); + $output .= '='; + if ($output =~ /range=$/) { + $hash{range} = + new Range($x, $linenr, $index); + return "::error::" unless $hash{range}; + $index = $hash{range}->{index}; + $output .= $hash{range}->{string}." "; + } + if ($output =~ /default=$/ + && ($hash{type} eq 'int' || $hash{type} eq 'double')) { + $x =~ /^.{$index}=\s*(\S+)/; + $hash{default} = $1; + if ($hash{type} eq 'int' && + !$Range::INT_RANGE->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + if ($hash{type} eq 'double' && + !$Range::DOUBLE_RANGE->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + } + if (defined($hash{default}) && $hash{range}) { + unless ($hash{range}->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + } + } else { + $output .= $char; + } + } else { + $output .= $char; + } + } + if ($index > $len) { + print STDERR "$defname:$linenr:$len: error: syntax error, line " + . "ends with \\: \"$x\"\n"; + return "::error::"; + } + chop $output if $output =~ m/ $/; + return $output; +} + +my %used_enum; +sub check_enum_sanity { + my ($line, $linenr, $file) = @_; + + my ($name, $type, $rest) = split(/\s+/, $line, 3); + return 0 unless ($type eq "enum"); + + $name =~ /(.*)\./; + my $prefix = $1; + $prefix = "" unless defined $prefix; # Make top level prefix + $used_enum{"$prefix"} = $used_enum{"$prefix"} || {}; + $rest = "" unless defined $rest; + $rest =~ /\{\s*(.*?)\}/; + my @values = split(/[,\s]+/, $1); + foreach my $value (@values) { + if ($used_enum{"$prefix"}->{$value}) { + print STDERR + "$file:$linenr: error: Name \"$value\" is already defined\n"; + my $prevdef = $used_enum{"$prefix"}->{$value}; + print STDERR "$prevdef: error: At this point\n"; + return -1; + } else { + $used_enum{"$prefix"}->{$value} = "$file:$linenr"; + } + } + return 0; +} + + +my %used_name; +my %used_component; +my %banned_prefixes; +my $cns_prev_name; +sub check_name_sanity { + my ($line, $linenr, $file) = @_; + my ($name, $junk) = split(/\s+/, $line, 2); + + my $plain_name = $name; + $plain_name =~ s=\[\]$==; + + # See if the name is already used. + if ($used_name{"$plain_name"}) { + print STDERR + "$file:$linenr: error: Name \"$name\" is already defined\n"; + my $prevdef = $used_name{$name}; + print STDERR "$prevdef: error: At this point\n"; + return -1; + } else { + $used_name{$name} = "$file:$linenr"; + } + + # Test for bans + my $banned = "${name}."; + do { + my $err = $banned_prefixes{$banned}; + if (defined($err)) { + print STDERR "$file:$linenr: error: The prefix \"$banned\" is illegal here\n"; + print STDERR "$err\n"; + return -1; + } + } while (($banned =~ s=[.][^.]+[.]$=.=)); + + # Add any new bans generated by this line + $banned_prefixes{"${name}."} = "$file:$linenr: error: \"${name}\" cannot " + . "be both a struct and a non-struct!"; + if ($cns_prev_name) { + my $prev = $cns_prev_name; + my $oldprev = $prev; + while (($prev =~ s=[.][^.]+[.]?$=.=)) { + if (substr($name, 0, length($prev)) eq $prev) { + $banned_prefixes{"$oldprev"} = "$file:" . ($linenr - 1) + . ": error: Last possible line is after this"; + last; + } + $oldprev = $prev; + } + } + $cns_prev_name = $name; + + # See if any of the components previously have a different "arrayness" + my $part_name = $name; + while (($part_name =~ s=[.][^.]+$==)) { + my $clashing_name = $part_name; + if ($part_name =~ m=\[\]$=) { + $clashing_name =~ s=\[\]$==; + } else { + $clashing_name .= "[]"; + } + my $clashline = $used_component{"$clashing_name"}; + if (defined $clashline) { + print STDERR "$file:$linenr: error: \"$clashing_name\" cannot be both array and non-array\n"; + print STDERR "$clashline: error: Previously defined here\n"; + return -1; + } elsif (!$used_component{"$part_name"}) { + $used_component{"$part_name"} = "$file:$linenr"; + } + } + return 0; +} + +# These are all the allowed types/commands +my %types = ( "int" => \&check_int, + "double" => \&check_double, + "string" => \&check_string, + "reference" => \&check_reference, + "enum" => \&check_enum, + "bool" => \&check_bool, + "properties" => \&check_properties, + "import" => \&check_import ); + +sub check_syntax { + my ($line, $linenr, $file) = @_; + + my $col = 0; + my $llen = length($line); + + # Step 1. Sanity check the name. + my $atstart = 1; + my $array_ok = 1; + + for ($col = 0; $col < $llen; ++$col) { + my $c = substr($line, $col, 1); + if ($atstart) { + if ($c !~ m=[a-zA-Z]=) { + print STDERR "$file:$linenr:$col: error: Non-alphabetic start " + . "of variable name in $line\n"; + return -1; + } + $atstart = 0; + } else { + if ($c =~ m=[a-zA-Z0-9_]=) { + 0; # Do nothing + } elsif ($c eq ".") { + $atstart = 1; + $array_ok = 1; + } elsif ($c eq "[") { + if (!$array_ok) { + ++$col; + print STDERR "$file:$linenr:$col: error: Arrays cannot be " + . "multidimensional in $line\n"; + return -1; + } + ++$col; + $array_ok = 0; + $c = substr($line, $col, 1); + if ($c ne "]") { + ++$col; + print STDERR "$file:$linenr:$col: error: Expected ] to " + . "terminate array definition in $line\n"; + return -1; + } + } elsif ($c =~ m=\s=) { + last; + } else { + ++$col; + print STDERR "$file:$linenr:$col: error: Syntax error, " + . "unexpected character in $line\n"; + return -1; + } + } + } + + my $name = substr($line, 0, $col); + $name =~ s=.*[.]==; + $name =~ s=[[]]$==; + + my $clash = $reserved_words{$name}; + if ($clash) { + $col -= (3 + length($name)); + $col = index($line, $name, $col) + 1; + print STDERR "$file:$linenr:$col: error: $name is a reserved word in: " + . "${clash}\n"; + return -1; + } + + while (substr($line, $col, 1) =~ m=\s=) { + ++$col; + } + + # At this point the name is sane. Next, check the type. + my ($type) = split(/\s/, substr($line, $col)); + + unless (defined $types{$type}) { + ++$col; + print STDERR "$file:$linenr:$col: error: Unknown type/command " + . "\"$type\"\n"; + return -1; + } + $col += length($type); + while (substr($line, $col, 1) =~ m=\s=) { + ++$col; + } + return $types{$type}($col, $line, $linenr, $file); +} + +sub reg_words_check { + my ($col, $line, $linenr, $file, $reg) = @_; + my $remainder = substr($line, $col); + my @options = split(/\s+/, $remainder); + + foreach my $option (@options) { + # Keep track of where we are for error reporting + $col = index($line, $option, $col) + 1; + unless ($option =~ m!${reg}!) { + print STDERR "$file:$linenr:$col: error: Bad option \"$option\" no match for m!${reg}!\n"; + return -1; + } + } + return 0; +} + +sub check_int { + my ($col, $line, $linenr, $file) = @_; + my $num = "(-?\\d+(KB|MB|GB|k|M|G)?|0x[0-9a-fA-F]+)"; # All legal numbers + my $optnum = "(${num})?"; # All legal optional numbers + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=${num}" + . "|range=[[(]${optnum},${optnum}"."[])]" + . "|restart" + . ")\$"); +} + +sub check_double { + my ($col, $line, $linenr, $file) = @_; + my $num = "-?(\\d+(\\.\\d*)?|\\.\\d+)([eE][+-]?\\d+)?"; # All legal doubles + my $optnum = "(${num})?"; # Optional doubles + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=${num}" + . "|range=[[(]${optnum},${optnum}"."[])]" + . "|restart" + . ")\$"); +} + +sub check_string { + my ($col, $line, $linenr, $file) = @_; + my $opts = substr($line, $col); + + # not entirely correct either for something like \\" + my $def = "default=((\"(\\\"|[^\"])*\")|null)"; + + my $res = "restart"; + my $reg = "^(${def}\\s+${res}|(${def})?|${res}|${res}\\s+${def})\$"; + + unless ($opts =~ m!${reg}!) { + print STDERR "$file:$linenr:$col: error: Bad options \"$opts\", no match for m!${reg}!\n"; + return -1; + } + return 0; +} + +sub check_reference { + my ($col, $line, $linenr, $file) = @_; + my $opts = substr($line, $col); + my $def = "default=((\"(\\\"|[^\"])*\")|null)"; + + unless ($opts eq "" || $opts =~m!${def}!) { + print STDERR "$file:$linenr:$col: error: reference can only " + . "take the 'default' option\n"; + return -1; + } + return 0; +} + + +sub check_enum { + my ($col, $line, $linenr, $file) = @_; + my $ret = ®_words_check($col, $line, $linenr, $file, + "^(" + . "[{},]" + . "|[A-Z][A-Z0-9_]+,?" + . "|default=[A-Z][A-Z0-9_]+" + . "|restart" + . ")\$"); + return -1 if $ret; + $col = index($line, '}', $col) + 1; #move $col to end of enum --> } + while (substr($line, $col, 1) =~ m=[\s\{]=) { + ++$col; + } + return 0 if $col >= length($line); + + + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=[A-Z][A-Z0-9_]+" + . "|restart" + . ")\$"); +} + +sub check_bool { + my ($col, $line, $linenr, $file) = @_; + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=(true|false)" + . "|restart" + . ")\$"); +} + +sub check_properties { + my ($col, $line, $linenr, $file) = @_; + return ®_words_check($col, $line, $linenr, $file, "^restart\$"); +} + +sub check_import { + my ($col, $line, $linenr, $file) = @_; + my $word = "[a-zA-Z][_a-zA-Z0-9]*"; + my $fnam = "${word}(\\.${word})*"; + my $var = "${word}((\\[\\])?\.${word})*(\\[\\])?"; + return ®_words_check($col, $line, $linenr, $file, + "^${fnam}\\.def:(${var})?\$"); + return 0; +} + + +my $lines = &do_file($defname, "", ""); + +if ($lines == -1) { + die "There were irrecoverable errors in \"$defname\"!\n"; +} +if ($lines == 0) { + die "$defname:1: error: Resulting definition is empty!\n"; +} + +exit 0; diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala new file mode 100644 index 00000000000..e4b9879f7f7 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala @@ -0,0 +1,350 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + +import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} +import JavaClassBuilder.{Indentation, createUniqueSymbol} +import ConfigGenerator.{indentCode, nodeClass, userDataType, boxedDataType} +import com.yahoo.config.codegen.LeafCNode._ + +/** + * @author gjoranv + */ + +object BuilderGenerator { + + def getBuilder(node: InnerCNode): String = { + getDeclaration(node) + "\n" + + indentCode(Indentation, + getUninitializedScalars(node) + "\n\n" + + node.getChildren.map(getBuilderFieldDefinition).mkString("\n") + "\n\n" + + getBuilderConstructors(node, nodeClass(node)) + "\n\n" + + getOverrideMethod(node) + "\n\n" + + getBuilderSetters(node) + "\n" + + getSpecialRootBuilderCode(node) + ) + + "}" + } + + private def getDeclaration(node: InnerCNode) = { + def getInterfaces = + if (node.getParent == null) "implements ConfigInstance.Builder" + else "implements ConfigBuilder" + + "public static class Builder " + getInterfaces + " {" + } + + private def getSpecialRootBuilderCode(node: InnerCNode) = { + if (node.getParent == null) "\n" + getDispatchCode(node) + "\n" + else "" + } + + private def getDispatchCode(node: InnerCNode) = { + // Use full path to @Override, as users are free to define an inner node called 'override'. (summarymap.def does) + // The generated inner 'Override' class would otherwise be mistaken for the annotation. + """ + |@java.lang.Override + |public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { + | if (producer instanceof Producer) { + | ((Producer)producer).getConfig(this); + | return true; + | } + | return false; + |} + | + |@java.lang.Override + |public final String getDefMd5() { return CONFIG_DEF_MD5; } + |@java.lang.Override + |public final String getDefName() { return CONFIG_DEF_NAME; } + |@java.lang.Override + |public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + """.stripMargin.trim + } + + private def getUninitializedScalars(node: InnerCNode): String = { + val scalarsWithoutDefault = { + node.getChildren.collect { + case leaf: LeafCNode if (!leaf.isArray && !leaf.isMap && leaf.getDefaultValue == null) => + "\"" + leaf.getName + "\"" + } + } + + val uninitializedList = + if (scalarsWithoutDefault.size > 0) + "Arrays.asList(\n" + indentCode(Indentation, scalarsWithoutDefault.mkString("",",\n","\n)")) + else + "" + + "private Set<String> " + InternalPrefix + "uninitialized = new HashSet<String>(" + uninitializedList + ");" + } + + private def getBuilderFieldDefinition(node: CNode): String = { + + (node match { + case array if node.isArray => + "public List<%s> %s = new ArrayList<>()".format(builderType(array), array.getName) + case map if node.isMap => + "public Map<String, %s> %s = new LinkedHashMap<>()".format(builderType(map), map.getName) + case struct: InnerCNode => + "private %s %s = new %s()".format(builderType(struct), struct.getName, builderType(struct)) + case scalar : LeafCNode => + "private " + boxedBuilderType(scalar) + " " + scalar.getName + " = null" + }) + ";" + } + + private def getBuilderSetters(node: CNode): String = { + val children: Array[CNode] = node.getChildren + + def structSetter(node: InnerCNode) = { + <code> + |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ + | {node.getName} = {InternalPrefix}builder; + | return this; + |}} + </code>.text.stripMargin.trim + } + + def innerArraySetters(node: InnerCNode) = { + <code> + |/** + | * Add the given builder to this builder's list of {nodeClass(node)} builders + | * @param {InternalPrefix}builder a builder + | * @return this builder + | */ + |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ + | {node.getName}.add({InternalPrefix}builder); + | return this; + |}} + | + |/** + | * Set the given list as this builder's list of {nodeClass(node)} builders + | * @param __builders a list of builders + | * @return this builder + | */ + |public Builder {node.getName}(List<{builderType(node)}> __builders) {{ + | {node.getName} = __builders; + | return this; + |}} + </code>.text.stripMargin.trim + } + + def leafArraySetters(node: LeafCNode) = { + val setters = + <code> + |public Builder {node.getName}({builderType(node)} {InternalPrefix}value) {{ + | {node.getName}.add({InternalPrefix}value); + | return this; + |}} + | + |public Builder {node.getName}(Collection<{builderType(node)}> {InternalPrefix}values) {{ + | {node.getName}.addAll({InternalPrefix}values); + | return this; + |}} + </code>.text.stripMargin.trim + + val privateSetter = + if (builderType(node) == "String" || builderType(node) == "FileReference") + "" + else + "\n\n" + + <code> + | + | + |private Builder {node.getName}(String {InternalPrefix}value) {{ + | return {node.getName}({builderType(node)}.valueOf({InternalPrefix}value)); + |}} + </code>.text.stripMargin.trim + + setters + privateSetter + } + + def innerMapSetters(node: CNode) = { + <code> + |public Builder {node.getName}(String {InternalPrefix}key, {builderType(node)} {InternalPrefix}value) {{ + | {node.getName}.put({InternalPrefix}key, {InternalPrefix}value); + | return this; + |}} + | + |public Builder {node.getName}(Map<String, {builderType(node)}> {InternalPrefix}values) {{ + | {node.getName}.putAll({InternalPrefix}values); + | return this; + |}} + </code>.text.stripMargin.trim + } + + def leafMapSetters(node: LeafCNode) = { + val privateSetter = + if (builderType(node) == "String" || builderType(node) == "FileReference") + "" + else + "\n\n" + + <code> + | + | + |private Builder {node.getName}(String {InternalPrefix}key, String {InternalPrefix}value) {{ + | return {node.getName}({InternalPrefix}key, {builderType(node)}.valueOf({InternalPrefix}value)); + |}} + </code>.text.stripMargin.trim + + innerMapSetters(node) + privateSetter + } + + def scalarSetters(node: LeafCNode): String = { + val name = node.getName + + val signalInitialized = + if (node.getDefaultValue == null) InternalPrefix + "uninitialized.remove(\"" + name + "\");\n" + else "" + + val stringSetter = + builderType(node) match { + case "String" => "" + case "FileReference" => "" + case _ => + """| + |private Builder %s(String %svalue) { + | return %s(%s.valueOf(%svalue)); + |}""".stripMargin.format(name, InternalPrefix, + name, boxedDataType(node), InternalPrefix) + } + + def getNullGuard = { + if (builderType(node) != boxedBuilderType(node)) + "" + else + "\n" + "if (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");" + .format(InternalPrefix) + } + + // TODO: check if 2.9.2 allows string to start with a newline + """|public Builder %s(%s %svalue) {%s + | %s = %svalue; + | %s + """.stripMargin.format(name, builderType(node), InternalPrefix, getNullGuard, + name, InternalPrefix, + signalInitialized).trim + + "\n return this;" + "\n}\n" + + stringSetter + } + + (children collect { + case innerArray: InnerCNode if innerArray.isArray => innerArraySetters(innerArray) + case innerMap: InnerCNode if innerMap.isMap => innerMapSetters(innerMap) + case leafArray: LeafCNode if leafArray.isArray => leafArraySetters(leafArray) + case leafMap: LeafCNode if leafMap.isMap => leafMapSetters(leafMap) + case struct: InnerCNode => structSetter(struct) + case scalar: LeafCNode => scalarSetters(scalar) + } ).mkString("\n\n") + } + + private def getBuilderConstructors(node: CNode, className: String): String = { + def setBuilderValueFromConfig(child: CNode) = { + val name = child.getName + val isArray = child.isArray + val isMap = child.isMap + + child match { + case fileArray: FileLeaf if isArray => name + "(" + userDataType(fileArray) + ".toValues(config." + name + "()));" + case fileMap: FileLeaf if isMap => name + "(" + userDataType(fileMap) + ".toValueMap(config." + name + "()));" + case file: FileLeaf => name + "(config." + name + "().value());" + case pathArray: PathLeaf if isArray => name + "(" + nodeClass(pathArray) + ".toFileReferences(config." + name + "));" + case pathMap: PathLeaf if isMap => name + "(" + nodeClass(pathMap) + ".toFileReferenceMap(config." + name + "));" + case path: PathLeaf => name + "(config." + name + ".getFileReference());" + case leaf: LeafCNode => name + "(config." + name + "());" + case innerArray: InnerCNode if isArray => setInnerArrayBuildersFromConfig(innerArray) + case innerMap: InnerCNode if isMap => setInnerMapBuildersFromConfig(innerMap) + case struct => name + "(new " + builderType(struct) + "(config." + name + "()));" + } + } + + def setInnerArrayBuildersFromConfig(innerArr: InnerCNode) = { + val elemName = createUniqueSymbol(node, innerArr.getName) + <code> + |for ({userDataType(innerArr)} {elemName} : config.{innerArr.getName}()) {{ + | {innerArr.getName}(new {builderType(innerArr)}({elemName})); + |}} + </code>.text.stripMargin.trim + } + + def setInnerMapBuildersFromConfig(innerMap: InnerCNode) = { + val entryName = InternalPrefix + "entry" + <code> + |for (Map.Entry<String, {userDataType(innerMap)}> {entryName} : config.{innerMap.getName}().entrySet()) {{ + | {innerMap.getName}({entryName}.getKey(), new {userDataType(innerMap)}.Builder({entryName}.getValue())); + |}} + </code>.text.stripMargin.trim + } + + <code> + |public Builder() {{ }} + | + |public Builder({className} config) {{ + |{indentCode(Indentation, node.getChildren.map(setBuilderValueFromConfig).mkString("\n"))} + |}} + </code>.text.stripMargin.trim + } + + def arrayOverride(name: String, superior: String): String = { + Indentation + name + ".addAll(" + superior + "." + name + ");" + } + + private def getOverrideMethod(node:CNode): String = { + val method = "override" + val superior = InternalPrefix + "superior" + + def callSetter(name: String): String = { + name + "(" + superior + "." + name + ");" + } + def overrideBuilderValue(child: CNode) = { + val name = child.getName + child match { + case leafArray: CNode if (child.isArray) => + conditionStatement(child) + "\n" + arrayOverride(name, superior) + case struct: InnerCNode if !(child.isArray || child.isMap) => + name + "(" + name + "." + method + "(" + superior + "." + name + "));" + case map: CNode if child.isMap => + callSetter(name) + case _ => + conditionStatement(child) + "\n" + + Indentation + callSetter(name) + } + } + + def conditionStatement(child: CNode) = { + val name = child.getName + val isArray = child.isArray + val isMap = child.isMap + child match { + case _ if isArray => "if (!" + superior + "." + name + ".isEmpty())" + case _ if isMap => "" + case scalar: LeafCNode => "if (" + superior + "." + name + " != null)" + case struct => "" + } + } + + <code> + |private Builder {method}(Builder {superior}) {{ + |{indentCode(Indentation, node.getChildren.map(overrideBuilderValue).mkString("\n"))} + | return this; + |}} + </code>.text.stripMargin.trim + } + + private def builderType(node: CNode): String = { + node match { + case inner: InnerCNode => boxedDataType(node) + ".Builder" + case file: FileLeaf => "String" + case path: PathLeaf => "FileReference" + case leafArray: LeafCNode if (node.isArray || node.isMap) => boxedDataType(node) + case _ => userDataType(node) + } + } + + private def boxedBuilderType(node: LeafCNode): String = { + node match { + case file: FileLeaf => "String" + case path: PathLeaf => "FileReference" + case _ => boxedDataType(node) + } + } + +}
\ No newline at end of file diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala new file mode 100644 index 00000000000..716f2a60c33 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala @@ -0,0 +1,459 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + + +import com.yahoo.config.codegen.LeafCNode._ +import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} +import JavaClassBuilder.Indentation +import BuilderGenerator.getBuilder +import util.parsing.combinator.JavaTokenParsers + +/** + * @author gjoranv + * @author tonytv + */ +// TODO: don't take indent as method param - the caller should indent +object ConfigGenerator { + + def generateContent(indent: String, node: InnerCNode, isOuter: Boolean = true): String = { + val children: Array[CNode] = node.getChildren + + def generateCodeForChildren: String = { + (children collect { + case enum: EnumLeaf => getEnumCode(enum, "") + "\n" + case inner: InnerCNode => getInnerDefinition(inner, indent) + "\n" + } ).mkString("\n") + } + + def getInnerDefinition(inner: InnerCNode, indent: String) = { + <code> + |{getClassDoc(inner, indent)} + |{getClassDeclaration(inner)} + |{generateContent(indent, inner, false)} + </code>.text.stripMargin.trim + "\n}" + } + + def getClassDeclaration(node: CNode): String = { + "public final static class " + nodeClass(node)+ " extends InnerNode { " + "\n" + } + + def getFieldDefinition(node: CNode): String = { + node.getCommentBlock("//") + "private final " + + (node match { + case _: LeafCNode if node.isArray => + "LeafNodeVector<%s, %s> %s;".format(boxedDataType(node), nodeClass(node), node.getName) + case _: InnerCNode if node.isArray => + "InnerNodeVector<%s> %s;".format(nodeClass(node), node.getName) + case _ if node.isMap => + "Map<String, %s> %s;".format(nodeClass(node), node.getName) + case _ => + "%s %s;".format(nodeClass(node), node.getName) + }) + } + + def getStaticMethods = { + if (node.isArray) getStaticMethodsForInnerArray(node) + "\n\n" + else if (node.isMap) getStaticMethodsForInnerMap(node) + "\n\n" + else "" + } + + def getContainsFieldsFlaggedWithRestart(node: CNode): String = { + if (isOuter) { + """ + |private static boolean containsFieldsFlaggedWithRestart() { + | return %b; + |} + """.stripMargin.trim.format(node.needRestart) + "\n\n" + } else "" + } + + indentCode(indent, + getBuilder(node) + "\n\n" + + children.map(getFieldDefinition).mkString("\n") + "\n\n" + + getConstructors(node) + "\n\n" + + getAccessors(children) + "\n\n" + + getGetChangesRequiringRestart(node) + "\n\n" + + getContainsFieldsFlaggedWithRestart(node) + + getStaticMethods + + generateCodeForChildren + ) + } + + private def getGetChangesRequiringRestart(node: InnerCNode): String = { + def quotedComment(node: CNode): String = { + node.getComment.replace("\n", "\\n").replace("\"", "\\\"") + } + + def getComparison(node: CNode): String = node match { + case inner: InnerCNode if inner.isArray => + <code> + | changes.compareArray(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", + | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); + </code>.text.stripMargin.trim + case inner: InnerCNode if inner.isMap => + <code> + | changes.compareMap(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", + | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); + </code>.text.stripMargin.trim + case inner: InnerCNode => + <code> + | changes.mergeChanges("{inner.getName}", this.{inner.getName}.getChangesRequiringRestart(newConfig.{inner.getName})); + </code>.text.stripMargin.trim + case node: CNode if node.isArray => + <code> + | changes.compareArray(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", + | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); + </code>.text.stripMargin.trim + case node: CNode if node.isMap => + <code> + | changes.compareMap(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", + | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); + </code>.text.stripMargin.trim + case node: CNode => + <code> + | changes.compare(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}"); + </code>.text.stripMargin.trim + } + + val comparisons = + for { + c <- node.getChildren if c.needRestart + } yield "\n " + getComparison(c) + + <code> + |private ChangesRequiringRestart getChangesRequiringRestart({nodeClass(node)} newConfig) {{ + | ChangesRequiringRestart changes = new ChangesRequiringRestart("{node.getName}");{comparisons.mkString("")} + | return changes; + |}} + </code>.text.stripMargin.trim + } + + + private def scalarDefault(scalar: LeafCNode): String = { + scalar match { + case _ if scalar.getDefaultValue == null => "" + case enumWithNullDefault: EnumLeaf if enumWithNullDefault.getDefaultValue.getValue == null => "" + case enum: EnumLeaf => nodeClass(enum) + "." + enum.getDefaultValue.getStringRepresentation + case long: LongLeaf => long.getDefaultValue.getStringRepresentation + "L" + case double: DoubleLeaf => double.getDefaultValue.getStringRepresentation + "D" + case _ => scalar.getDefaultValue.getStringRepresentation + } + } + + private def getConstructors(inner: InnerCNode) = { + + def assignFromBuilder(child: CNode) = { + val name = child.getName + val className = nodeClass(child) + val dataType = boxedDataType(child) + val isArray = child.isArray + val isMap = child.isMap + + def assignIfInitialized(leaf: LeafCNode) = { + <code> + |{name} = (builder.{name} == null) ? + | new {className}({scalarDefault(leaf)}) : new {className}(builder.{name}); + </code>.text.stripMargin.trim + } + + child match { + case fileArray: FileLeaf if isArray => + name + " = LeafNodeVector.createFileNodeVector(builder."+ name +");" + case pathArray: PathLeaf if isArray => + name + " = LeafNodeVector.createPathNodeVector(builder."+ name +");" + case leafArray: LeafCNode if isArray => + name + " = new LeafNodeVector<>(builder."+ name +", new " + className + "());" + case fileMap: LeafCNode if isMap && child.isInstanceOf[FileLeaf] => + name + " = LeafNodeMaps.asFileNodeMap(builder."+ name +");" + case pathMap: LeafCNode if isMap && child.isInstanceOf[PathLeaf] => + name + " = LeafNodeMaps.asPathNodeMap(builder."+ name +");" + case leafMap: LeafCNode if isMap => + name + " = LeafNodeMaps.asNodeMap(builder."+ name +", new " + className + "());" + case innerArray: InnerCNode if isArray => + name + " = " + className + ".createVector(builder." + name + ");" + case innerMap: InnerCNode if isMap => + name + " = " + className + ".createMap(builder." + name + ");" + case struct: InnerCNode => + name + " = new " + className + "(builder." + name + ", throwIfUninitialized);" + case leaf: LeafCNode => + assignIfInitialized(leaf) + } + } + + // TODO: The default ctor can be removed if the config library uses builders to set values from payload, but ... + // a default ctor is also needed for all innerArrays, because of InnerNodeVector.createNew() + def defaultConstructor = { + // TODO @link gives javadoc warnings, although the syntax seems to be valid + //def link = "{@link " + {nodeClass(inner)} + "#" + {nodeClass(inner)} + "(Builder)}" + def link = {nodeClass(inner)} + "(Builder)" + + <code> + |/** + | * @deprecated Not for public use. + | * Does not check for uninitialized fields. + | * Replaced by {link} + | */ + |@Deprecated + |public {nodeClass(inner)}() {{ + | this(new Builder(), false); + |}} + </code>.text.stripMargin.trim + } + + // TODO: merge these two constructors into one when the config library uses builders to set values from payload. + <code> + |{defaultConstructor} + | + |public {nodeClass(inner)}(Builder builder) {{ + | this(builder, true); + |}} + | + |private {nodeClass(inner)}(Builder builder, boolean throwIfUninitialized) {{ + | if (throwIfUninitialized && ! builder.{InternalPrefix}uninitialized.isEmpty()) + | throw new IllegalArgumentException("The following builder parameters for " + + | "{inner.getFullName} must be initialized: " + builder.{InternalPrefix}uninitialized); + | + |{indentCode(Indentation, inner.getChildren.map(assignFromBuilder).mkString("\n"))} + |}} + </code>.text.stripMargin.trim + } + + private def getAccessors(children: Array[CNode]): String = { + + def getAccessorCode(indent: String, node: CNode): String = { + indentCode(indent, + if (node.isArray) + accessorsForArray(node) + else if (node.isMap) + accessorsForMap(node) + else + accessorForStructOrScalar(node)) + } + + def valueAccessor(node: CNode) = node match { + case leaf: LeafCNode => ".value()" + case inner => "" + } + + def listAccessor(node: CNode) = node match { + case leaf: LeafCNode => "%s.asList()".format(leaf.getName) + case inner => inner.getName + } + + def mapAccessor(node: CNode) = node match { + case leaf: LeafCNode => "LeafNodeMaps.asValueMap(%s)".format(leaf.getName) + case inner => "Collections.unmodifiableMap(%s)".format(inner.getName) + } + + def accessorsForArray(node: CNode): String = { + val name = node.getName + val fullName = node.getFullName + <code> + |/** + | * @return {fullName} + | */ + |public List<{boxedDataType(node)}> {name}() {{ + | return {listAccessor(node)}; + |}} + | + |/** + | * @param i the index of the value to return + | * @return {fullName} + | */ + |public {userDataType(node)} {name}(int i) {{ + | return {name}.get(i){valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + def accessorsForMap(node: CNode): String = { + val name = node.getName + val fullName = node.getFullName + <code> + |/** + | * @return {fullName} + | */ + |public Map<String, {boxedDataType(node)}> {name}() {{ + | return {mapAccessor(node)}; + |}} + | + |/** + | * @param key the key of the value to return + | * @return {fullName} + | */ + |public {userDataType(node)} {name}(String key) {{ + | return {name}.get(key){valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + def accessorForStructOrScalar(node: CNode): String = { + <code> + |/** + | * @return {node.getFullName} + | */ + |public {userDataType(node)} {node.getName}() {{ + | return {node.getName}{valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + val accessors = + for { + c <- children + accessor = getAccessorCode("", c) + if (accessor.length > 0) + } yield (accessor + "\n") + accessors.mkString("\n").trim + } + + private def getStaticMethodsForInnerArray(inner: InnerCNode) = { + """ + |private static InnerNodeVector<%s> createVector(List<Builder> builders) { + | List<%s> elems = new ArrayList<>(); + | for (Builder b : builders) { + | elems.add(new %s(b)); + | } + | return new InnerNodeVector<%s>(elems, new %s()); + |} + """.stripMargin.format(List.fill(5)(nodeClass(inner)): _*).trim + } + + private def getStaticMethodsForInnerMap(inner: InnerCNode) = { + """ + |private static Map<String, %s> createMap(Map<String, Builder> builders) { + | Map<String, %s> ret = new LinkedHashMap<>(); + | for(String key : builders.keySet()) { + | ret.put(key, new %s(builders.get(key))); + | } + | return Collections.unmodifiableMap(ret); + |} + """.stripMargin.format(List.fill(3)(nodeClass(inner)): _*).trim + } + + private def getEnumCode(enum: EnumLeaf, indent: String): String = { + + def getEnumValues(enum: EnumLeaf): String = { + val enumValues = + for (value <- enum.getLegalValues) yield + """ public final static Enum %s = Enum.%s;""".format(value, value) + enumValues.mkString("\n") + } + + // TODO: try to rewrite to xml + val code = + """ + |%s + |public final static class %s extends EnumNode<%s> { + + | public %s(){ + | this.value = null; + | } + + | public %s(Enum enumValue) { + | super(enumValue != null); + | this.value = enumValue; + | } + + | public enum Enum {%s} + |%s + + | @Override + | protected boolean doSetValue(@NonNull String name) { + | try { + | value = Enum.valueOf(name); + | return true; + | } catch (IllegalArgumentException e) { + | } + | return false; + | } + |} + |""" + .stripMargin.format(getClassDoc(enum, indent), + nodeClass(enum), + nodeClass(enum)+".Enum", + nodeClass(enum), + nodeClass(enum), + enum.getLegalValues.mkString(", "), + getEnumValues(enum)) + + indentCode(indent, code).trim + } + + def getClassDoc(node: CNode, indent: String): String = { + val header = "/**\n" + " * This class represents " + node.getFullName + val nodeComment = node.getCommentBlock(" *") match { + case "" => "" + case s => "\n *\n" + s.stripLineEnd // TODO: strip trailing \n in CNode.getCommentBlock + } + header + nodeComment + "\n */" + } + + def indentCode(indent: String, code: String): String = { + val indentedLines = + for (s <- code.split("\n", -1)) yield + if (s.length() > 0) (indent + s) else s + indentedLines.mkString("\n") + } + + /** + * @return the name of the class that is generated by this node. + */ + def nodeClass(node: CNode): String = { + node match { + case emptyName: CNode if node.getName.length == 0 => + throw new CodegenRuntimeException("Node with empty name, under parent " + emptyName.getParent.getName) + case root: InnerCNode if root.getParent == null => createClassName(root.getName) + case b: BooleanLeaf => "BooleanNode" + case d: DoubleLeaf => "DoubleNode" + case f: FileLeaf => "FileNode" + case p: PathLeaf => "PathNode" + case i: IntegerLeaf => "IntegerNode" + case l: LongLeaf => "LongNode" + case r: ReferenceLeaf => "ReferenceNode" + case s: StringLeaf => "StringNode" + case _ => node.getName.capitalize + } + } + + def userDataType(node: CNode): String = { + node match { + case inner: InnerCNode => nodeClass(node) + case enum: EnumLeaf => nodeClass(enum) + ".Enum" + case b: BooleanLeaf => "boolean" + case d: DoubleLeaf => "double" + case f: FileLeaf => "FileReference" + case p: PathLeaf => "Path" + case i: IntegerLeaf => "int" + case l: LongLeaf => "long" + case s: StringLeaf => "String" + } + } + + /** + * @return the boxed java data type, e.g. Integer for int + */ + def boxedDataType(node: CNode): String = { + val rawType = userDataType(node) + + rawType match { + case "int" => "Integer" + case _ if rawType == rawType.toLowerCase => rawType.capitalize + case _ => rawType + } + } + + /** + * Create class name from def name + * @param defName The file name without the '.def' suffix + */ + def createClassName(defName: String): String = { + val className = defName.split("-").map (_.capitalize).mkString + "Config" + val parser = new JavaTokenParsers {} + parser.parseAll(parser.ident, className) match { + case parser.NoSuccess(msg, _) => + throw new CodegenRuntimeException("Illegal config definition file name '" + defName + "': " + msg) + case success => success.get + } + } +} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala new file mode 100644 index 00000000000..c346338e543 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + +import java.io.{FileNotFoundException, FileOutputStream, PrintStream, File} +import ConfigGenerator.{indentCode, createClassName} +import util.Random +import scala.collection.JavaConversions._ + +/** + * Builds one Java class based on the given CNode tree. + * + * @author gjoranv + * @author tonytv + */ +class JavaClassBuilder( + root: InnerCNode, + nd: NormalizedDefinition, + destDir: File) + extends ClassBuilder +{ + import JavaClassBuilder._ + + val javaPackage = PackagePrefix + root.getNamespace + val className = createClassName(root.getName) + + override def createConfigClasses() { + try { + val outFile = new File(getDestPath(destDir, root.getNamespace), className + ".java") + val out = new PrintStream(new FileOutputStream(outFile)) + out.print(getConfigClass(className)) + System.err.println(outFile.getPath + " successfully written.") + } + catch { + case e: FileNotFoundException => { + throw new CodegenRuntimeException(e) + } + } + } + + def getConfigClass(className:String): String = { + val ret = new StringBuilder + + ret.append(getHeader).append("\n\n") + ret.append(getRootClassDeclaration(root, className)).append("\n\n") + ret.append(indentCode(Indentation, getFrameworkCode(className))).append("\n\n") + ret.append(ConfigGenerator.generateContent(Indentation, root)).append("\n") + ret.append("}\n") + + ret.toString() + } + + private def getHeader: String = { + <code> + |/** + | * This file is generated from a config definition file. + | * ------------ D O N O T E D I T ! ------------ + | */ + | + |package {javaPackage}; + | + |import java.util.*; + |import java.nio.file.Path; + |import edu.umd.cs.findbugs.annotations.NonNull; + |{getImportFrameworkClasses(root.getNamespace)} + </code>.text.stripMargin.trim + } + + private def getImportFrameworkClasses(namespace: String): String = { + if (namespace != CNode.DEFAULT_NAMESPACE) + "import " + PackagePrefix + CNode.DEFAULT_NAMESPACE + ".*;\n" + else + "" + } + + // TODO: remove the extra comment line " *" if root.getCommentBlock is empty + private def getRootClassDeclaration(root:InnerCNode, className: String): String = { + <code> + |/** + | * This class represents the root node of {root.getFullName} + | * + |{root.getCommentBlock(" *")} */ + |public final class {className} extends ConfigInstance {{ + | + | public final static String CONFIG_DEF_MD5 = "{root.getMd5}"; + | public final static String CONFIG_DEF_NAME = "{root.getName}"; + | public final static String CONFIG_DEF_NAMESPACE = "{root.getNamespace}"; + | public final static String CONFIG_DEF_VERSION = "{root.getVersion}"; + | public final static String[] CONFIG_DEF_SCHEMA = {{ + |{indentCode(Indentation * 2, getDefSchema)} + | }}; + | + | public static String getDefMd5() {{ return CONFIG_DEF_MD5; }} + | public static String getDefName() {{ return CONFIG_DEF_NAME; }} + | public static String getDefNamespace() {{ return CONFIG_DEF_NAMESPACE; }} + | public static String getDefVersion() {{ return CONFIG_DEF_VERSION; }} + </code>.text.stripMargin.trim + } + + private def getDefSchema: String = { + nd.getNormalizedContent.map { line => + "\"" + + line.replace("\"", "\\\"") + + "\"" + }.mkString(",\n") + } + + private def getFrameworkCode(className: String): String = { + getProducerBase + } + + private def getProducerBase = { + """ + |public interface Producer extends ConfigInstance.Producer { + | void getConfig(Builder builder); + |} + """.stripMargin.trim + } +} + + +object JavaClassBuilder { + private val PackagePrefix: String = System.getProperty("config.packagePrefix", "com.yahoo.") + + val Indentation = " " + + /** + * Returns a name that can be safely used as a local variable in the generated config class + * for the given node. The name will be based on the given basis string, but the basis itself is + * not a possible return value. + * + * @param node The node to find a unused symbol name for. + * @param basis The basis for the generated symbol name. + * @return A name that is not used in the given config node. + */ + def createUniqueSymbol(node: CNode, basis: String) = { + + def getCandidate(cnt: Int) = { + if (cnt < basis.length()) + basis.substring(0, cnt) + else + ReservedWords.INTERNAL_PREFIX + basis + Random.nextInt().abs + } + + def getUsedSymbols: Set[String] = { + (node.getChildren map (child => child.getName)).toSet + } + + // TODO: refactoring potential + val usedSymbols = getUsedSymbols + var count = 1 + var candidate = getCandidate(count) + while (usedSymbols contains(candidate)) { + count += 1 + candidate = getCandidate(count) + } + candidate + } + + /** + * @param rootDir The root directory for the destination path. + * @param namespace The namespace from the def file + * @return the destination path for the generated config file, including the given rootDir. + */ + private def getDestPath(rootDir: File, namespace: String): File = { + var dir: File = rootDir + val subDirs: Array[String] = (PackagePrefix + namespace).split("""\.""") + for (subDir <- subDirs) { + dir = new File(dir, subDir) + if (!dir.isDirectory && !dir.mkdir) throw new CodegenRuntimeException("Could not create " + dir.getPath) + } + dir + } +} |