diff options
Diffstat (limited to 'configgen')
32 files changed, 6160 insertions, 0 deletions
diff --git a/configgen/.gitignore b/configgen/.gitignore new file mode 100644 index 00000000000..503990bce0c --- /dev/null +++ b/configgen/.gitignore @@ -0,0 +1,4 @@ +configgen.iml +target +/deploy +/pom.xml.build diff --git a/configgen/OWNERS b/configgen/OWNERS new file mode 100644 index 00000000000..b690845e701 --- /dev/null +++ b/configgen/OWNERS @@ -0,0 +1,2 @@ +musum +gjoranv diff --git a/configgen/README b/configgen/README new file mode 100644 index 00000000000..cc122f6deb6 --- /dev/null +++ b/configgen/README @@ -0,0 +1,16 @@ +Vespa Config Generation +======================= + +The configgen module is used to generate config-classes from .def files. + +Userguide +--------- + +Usually you will want to use this module through the config-class-plugin +maven plugin - see the documentation for that module. + +This module can be used stand-alone by building the jar file (mvn package) +and then calling MakeConfig from that file: + +java -Dconfig.spec=<def-file_1,def-file_2,...> -Dconfig.dest=<dest-dir> -jar target/configgen.jar + diff --git a/configgen/bin/make-config.pl b/configgen/bin/make-config.pl new file mode 100755 index 00000000000..4629e6e3240 --- /dev/null +++ b/configgen/bin/make-config.pl @@ -0,0 +1,134 @@ +#!/usr/local/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This script transforms a .def file into a .h and .cpp file for config +# +# TODO: Remove this script and use the java code directly. BTW, why +# does this script have the same limitations as the old make-config.pl +# in that the .def-file must reside in the root directory? The java +# code supports reading the .def-file from any directory. This script +# should have the same input parameters as the java code, and just +# map them directly to the java system properties + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +require 5.006_001; +use strict; +use warnings; + +use Cwd 'abs_path'; + +# Now this uses the new java codegen library. But the script still exist to +# map be able to call java the right way, setting the necessary properties + +my ($root, $def) = @ARGV; + +if (!defined $root || !defined $def) { + print "Usage make-config.pl <source root dir> <def file>\n"; + exit(1); +} + +#print "Root: $root\n" +# . "Def: $def\n"; + +my $subdir = &getRelativePath($root, &getPath($def)); + +my $cmd = "java" + . " -Dconfig.spec=$def" + . " -Dconfig.dest=$root" + . " -Dconfig.lang=cppng" + . " -Dconfig.requireNamespace=false" + . " -Dconfig.subdir=$subdir" + . " -Dconfig.dumpTree=false" + . " -Xms64m -Xmx64m" + . " -jar $VESPA_HOME/lib/jars/configgen.jar"; + +print "Generating config: $cmd\n"; +exec($cmd); + +exit(0); # Will never be called due to exec above, but just to indicate end + +sub getRelativePath { + my ($from, $to) = @_; + + $from = abs_path($from); + $to = abs_path($to); + + # Escape $from so it can contain regex special characters in path + $from =~ s/([\+\*\(\)\{\}\.\?\[\]\$\&])/\\$1/g; + + $to =~ /^$from\/(.*)$/ + or die "The def file must be contained within the root"; + return $1; +} + +sub getPath { + my $file = $_[0]; + if ($file =~ /^(.*)\/[^\/]*$/) { + return $1; + } else { + return "."; + } +} diff --git a/configgen/bin/make-configold.pl b/configgen/bin/make-configold.pl new file mode 100755 index 00000000000..5ba2d88f67e --- /dev/null +++ b/configgen/bin/make-configold.pl @@ -0,0 +1,133 @@ +#!/usr/local/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This script transforms a .def file into a .h and .cpp file for config +# +# TODO: Remove this script and use the java code directly. BTW, why +# does this script have the same limitations as the old make-config.pl +# in that the .def-file must reside in the root directory? The java +# code supports reading the .def-file from any directory. This script +# should have the same input parameters as the java code, and just +# map them directly to the java system properties + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +require 5.006_001; +use strict; +use warnings; + +use Cwd 'abs_path'; + +# Now this uses the new java codegen library. But the script still exist to +# map be able to call java the right way, setting the necessary properties + +my ($root, $def) = @ARGV; + +if (!defined $root || !defined $def) { + print "Usage make-config.pl <source root dir> <def file>\n"; + exit(1); +} + +#print "Root: $root\n" +# . "Def: $def\n"; + +my $subdir = &getRelativePath($root, &getPath($def)); + +my $cmd = "java" + . " -Dconfig.spec=$def" + . " -Dconfig.dest=$root" + . " -Dconfig.lang=cpp" + . " -Dconfig.subdir=$subdir" + . " -Dconfig.dumpTree=false" + . " -Xms64m -Xmx64m" + . " -jar $VESPA_HOME/lib/jars/configgen.jar"; + +print "Generating config: $cmd\n"; +exec($cmd); + +exit(0); # Will never be called due to exec above, but just to indicate end + +sub getRelativePath { + my ($from, $to) = @_; + + $from = abs_path($from); + $to = abs_path($to); + + # Escape $from so it can contain regex special characters in path + $from =~ s/([\+\*\(\)\{\}\.\?\[\]\$\&])/\\$1/g; + + $to =~ /^$from\/(.*)$/ + or die "The def file must be contained within the root"; + return $1; +} + +sub getPath { + my $file = $_[0]; + if ($file =~ /^(.*)\/[^\/]*$/) { + return $1; + } else { + return "."; + } +} diff --git a/configgen/pom.xml b/configgen/pom.xml new file mode 100644 index 00000000000..d8fc4703694 --- /dev/null +++ b/configgen/pom.xml @@ -0,0 +1,109 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configgen</artifactId> + <packaging>jar</packaging> + <version>6-SNAPSHOT</version> + <name>configgen</name> + <description>Config java code generation from defintion files for Java Vespa components.</description> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-compiler</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-xml_${scala.major-version}</artifactId> + </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parser-combinators_${scala.major-version}</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestFile>src/main/manifest.mf</manifestFile> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>org.scala-tools</groupId> + <artifactId>maven-scala-plugin</artifactId> + <configuration> + <args> + <arg>-unchecked</arg> + <arg>-deprecation</arg> + <arg>-feature</arg> + </args> + </configuration> + <executions> + <execution> + <id>compile</id> + <goals> + <goal>compile</goal> + </goals> + <phase>compile</phase> + </execution> + <execution> + <id>test-compile</id> + <goals> + <goal>testCompile</goal> + </goals> + <phase>test-compile</phase> + </execution> + <execution> + <phase>process-resources</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CNode.java b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java new file mode 100644 index 00000000000..02fec548b40 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CNode.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.StringTokenizer; + +/** + * Abstract superclass for all nodes representing a definition file. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public abstract class CNode { + + public static final String DEFAULT_NAMESPACE = "config"; + + // TODO: replace by "type" enum + public final boolean isArray; + public final boolean isMap; + final String name; + final InnerCNode parent; + + // TODO: remove! Only set for the root node, and root.getName() returns the same thing! + String defName = null; + String defVersion = ""; + String defNamespace = null; + String defMd5 = "MISSING MD5"; + String comment = ""; + + + protected CNode(InnerCNode parent, String name) { + this.parent = parent; + int bracketIdx = name.indexOf('['); + int curlyIdx = name.indexOf('{'); + if (bracketIdx != -1) { + this.name = name.substring(0, bracketIdx); + isArray = true; + isMap = false; + } else if (curlyIdx != -1) { + this.name = name.substring(0, curlyIdx); + isMap = true; + isArray = false; + } else { + this.name = name; + isMap = false; + isArray = false; + } + } + + /** + * Returns the simple name of this node. + * @return the simple name of this node + */ + public String getName() { + return name; + } + + public InnerCNode getParent() { + return parent; + } + + public abstract CNode[] getChildren(); + + public abstract CNode getChild(String name); + + public String getMd5() { + return defMd5; + } + + void setMd5(String md5) { + defMd5 = md5; + } + + public String getVersion() { + return defVersion; + } + + void setVersion(String version) { + defVersion = version; + } + + public String getNamespace() { + return defNamespace; + } + + void setNamespace(String namespace) { + defNamespace = namespace; + } + + public String getComment() { + return comment; + } + + void setComment(String comment) { + this.comment = comment; + } + + protected abstract void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException; + + public abstract boolean needRestart(); + + protected void checkMyName(String myName) throws IllegalArgumentException { + if (isArray) { + int n1 = myName.indexOf('['); + int n2 = myName.indexOf(']'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid array syntax: " + myName); + myName = myName.substring(0, n1); + } else if (isMap) { + int n1 = myName.indexOf('{'); + int n2 = myName.indexOf('}'); + if (n1 == -1 || n2 < n1) + throw new IllegalArgumentException("Invalid map syntax: " + myName); + myName = myName.substring(0, n1); + } else if (myName.contains("[]")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-array type."); + } else if (myName.contains("{}")) { + throw new IllegalArgumentException("Parameter with name '" + getName() + "' has already been declared as a non-map type."); + } + if (!myName.equals(getName())) + throw new IllegalArgumentException(myName + " does not match " + getName() + "."); + } + + /** + * @return the full name as a config path of this node. + */ + public String getFullName() { + StringBuilder buf = new StringBuilder(); + if (parent != null) + buf.append(parent.getFullName()); + if (buf.length() > 0) + buf.append('.'); + StringBuilder theName = new StringBuilder(this.getName()); + + if (isArray) theName.append("[]"); + else if (isMap) theName.append("{}"); + + return buf.append(theName).toString(); + } + + /** + * @param prefix The prefix to use, usually an indent (spaces) followed by either '*' or "//" + * @return a comment block where each line is prefixed, but the caller must close it if using '*'. + */ + public String getCommentBlock(String prefix) { + prefix = prefix + " "; + StringBuilder ret = new StringBuilder(); + if (getComment().length() > 0) { + StringTokenizer st = new StringTokenizer(getComment(), "\n"); + while (st.hasMoreTokens()) { + ret.append(prefix).append(st.nextToken()).append("\n"); + } + } + return ret.toString(); + } + + @Override + public String toString() { + return getNamespace()+"."+getName()+","+getVersion(); + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java new file mode 100644 index 00000000000..dab2ce98510 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ClassBuilder.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public interface ClassBuilder { + + /** + * Generate config class file(s). + */ + public void createConfigClasses(); +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java new file mode 100644 index 00000000000..56a87c0506e --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CodegenRuntimeException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * This exception is thrown on internal errors. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class CodegenRuntimeException extends RuntimeException { + public CodegenRuntimeException(String s, Throwable cause) { + super(s, cause); + } + + public CodegenRuntimeException(Throwable cause) { + super(cause); + } + + public CodegenRuntimeException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java new file mode 100644 index 00000000000..e6d7f29ad36 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -0,0 +1,1117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; + + +/** + * This class autogenerates C++ code for the C++ config, based on a CNode tree given. + */ +public class CppClassBuilder implements ClassBuilder { + private final CNode root; + private final NormalizedDefinition nd; + private final File rootDir; + private final String relativePathUnderRoot; + private static final Map<String, String> vectorTypeDefs; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "BoolVector"); + map.put("int32_t", "IntVector"); + map.put("int64_t", "LongVector"); + map.put("double", "DoubleVector"); + map.put("vespalib::string", "StringVector"); + vectorTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> mapTypeDefs; + static { + Map<String, String> map = new HashMap<>(); + map.put("bool", "BoolMap"); + map.put("int32_t", "IntMap"); + map.put("int64_t", "LongMap"); + map.put("double", "DoubleMap"); + map.put("vespalib::string", "StringMap"); + mapTypeDefs = Collections.unmodifiableMap(map); + } + private static final Map<String, String> slimeTypeMap; + static { + Map<String, String> map = new HashMap<String, String>(); + map.put("bool", "Bool"); + map.put("int", "Long"); + map.put("long", "Long"); + map.put("double", "Double"); + map.put("string", "String"); + map.put("enum", "String"); + map.put("file", "String"); + map.put("reference", "String"); + slimeTypeMap = Collections.unmodifiableMap(map); + } + + public CppClassBuilder(CNode root, NormalizedDefinition nd, File rootDir, String relativePathUnderRoot) { + this.root = root; + this.nd = nd; + this.rootDir = rootDir; + this.relativePathUnderRoot = relativePathUnderRoot; + } + + public void createConfigClasses() { + generateConfig(root, nd); + } + + String readFile(File f) throws IOException { + if (!f.isFile()) return null; + StringBuilder sb = new StringBuilder(); + BufferedReader sr = new BufferedReader(new FileReader(f)); + while (true) { + String line = sr.readLine(); + if (line == null) break; + sb.append(line).append("\n"); + } + return sb.toString(); + } + + void writeFile(File f, String content) throws IOException { + FileWriter fw = new FileWriter(f); + fw.write(content); + fw.close(); + } + + void generateConfig(CNode root, NormalizedDefinition nd) { + try{ + StringWriter headerWriter = new StringWriter(); + StringWriter bodyWriter = new StringWriter(); + writeHeaderFile(headerWriter, root); + writeBodyFile(bodyWriter, root, relativePathUnderRoot, nd); + + String newHeader = headerWriter.toString(); + String newBody = bodyWriter.toString(); + + File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h")); + File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp")); + + String oldHeader = readFile(headerFile); + String oldBody = readFile(bodyFile); + + if (oldHeader == null || !oldHeader.equals(newHeader)) { + writeFile(headerFile, newHeader); + } + if (oldBody == null || !oldBody.equals(newBody)) { + writeFile(bodyFile, newBody); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + String getFileName(CNode node, String extension) { + return "config-" + node.getName() + "." + extension; + } + + static String removeDashesAndUpperCaseAllFirstChars(String source, boolean capitalizeFirst) { + // Create upper case chars after each dash + String parts[] = source.split("[-_]"); + StringBuilder sb = new StringBuilder(); + for (String s : parts) { + sb.append(s.substring(0, 1).toUpperCase()).append(s.substring(1)); + } + String result = sb.toString(); + if (!capitalizeFirst) { + result = result.substring(0,1).toLowerCase() + result.substring(1); + } + return result; + } + + /** Convert name of type to the name we want to use in macro ifdefs in file. */ + String getDefineName(String name) { + return name.toUpperCase().replace("-", ""); + } + + /** Convert name of type to the name we want to use as type name in the generated code. */ + static String getTypeName(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, true); + } + + /** Convert name of an identifier from value in def file to name to use in C++ file. */ + String getIdentifier(String name) { + return removeDashesAndUpperCaseAllFirstChars(name, false); + } + + void writeHeaderFile(Writer w, CNode root) throws IOException { + writeHeaderHeader(w, root); + writeHeaderPublic(w, root); + writeHeaderFooter(w, root); + } + + void writeHeaderPublic(Writer w, CNode root) throws IOException { + w.write("public:\n"); + writeHeaderTypeDefs(w, root, " "); + writeTypeDeclarations(w, root, " "); + writeHeaderFunctionDeclarations(w, getTypeName(root, false), root, " "); + writeStaticMemberDeclarations(w, " "); + writeMembers(w, root, " "); + } + + String [] generateCppNameSpace(CNode root) { + String namespace = root.getNamespace(); + if (namespace.contains(".")) { + return namespace.split("\\."); + } + return new String[]{namespace}; + } + + String generateCppNameSpaceString(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i]); + str.append("::"); + } + str.append(namespaceList[namespaceList.length - 1]); + return str.toString(); + } + + String generateCppNameSpaceDefine(String[] namespaceList) { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < namespaceList.length - 1; i++) { + str.append(namespaceList[i].toUpperCase()); + str.append("_"); + } + str.append(namespaceList[namespaceList.length - 1].toUpperCase()); + return str.toString(); + } + + void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("namespace " + namespaceList[i] + " {\n\n"); + } + } + + void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException { + for (int i = 0; i < namespaceList.length; i++) { + w.write("} // namespace " + namespaceList[i] + "\n\n"); + } + } + + void writeHeaderHeader(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespacePrint = generateCppNameSpaceString(namespaceList); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + w.write("" + + "/**\n" + + " * @class " + namespacePrint + "::" + className + "\n" + + " * @ingroup config\n" + + " *\n" + + " * @brief This is an autogenerated class for handling VESPA config.\n" + + " *\n" + + " * This class is autogenerated by vespa from a config definition file.\n" + + " * To subscribe to config, you need to include the config/config.h header, \n" + + " * and create a ConfigSubscriber in order to subscribe for config.\n" + ); + if (root.getComment().length() > 0) { + w.write(" *\n"); + StringTokenizer st = new StringTokenizer(root.getComment(), "\n"); + while (st.hasMoreTokens()) { + w.write(" * " + st.nextToken() + "\n"); + } + } + w.write("" + + " */\n" + + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n" + + "#define CLOUD_CONFIG_" + defineName + "_H\n" + + "\n" + + "#include <vespa/config/common/configvalue.h>\n" + + "#include <vespa/config/configgen/configpayload.h>\n" + + "#include <vespa/config/configgen/configinstance.h>\n" + + "#include <vespa/config/print/configdatabuffer.h>\n" + + "#include <vespa/vespalib/stllike/string.h>\n" + + "#include <vector>\n" + + "#include <map>\n" + + "\n"); + writeNameSpaceBegin(w, namespaceList); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("" + + "/**\n" + + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n" + + " * versions after this class declaration.\n" + + " */\n" + + "class Internal" + className + "Type : public ::config::ConfigInstance\n" + + "{\n" + ); + + } + + + + void writeTypeDeclarations(Writer w, CNode node, String indent) throws IOException { + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(indent + "enum " + typeName + " { "); + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + for (int i=0; i<leaf.getLegalValues().length; ++i) { + if (i != 0) { + w.write(", "); + } + w.write(leaf.getLegalValues()[i]); + } + w.write(" };\n" + + indent + "typedef std::vector<" + typeName + "> " + + typeName + "Vector;" + + "\n" + + indent + "typedef std::map<vespalib::string, " + typeName + "> " + + typeName + "Map;" + + "\n" + + indent + "static " + typeName + " get" + typeName + "(const vespalib::string&);\n" + + indent + "static vespalib::string get" + typeName + "Name(" + typeName + " e);\n" + + "\n" + ); + w.write(indent + "struct Internal" + typeName + "Converter {\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + " " + typeName + " operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault);\n"); + w.write(indent + "};\n"); + } else { + w.write(indent + "class " + typeName + " {\n"); + w.write(indent + "public:\n"); + writeTypeDeclarations(w, child, indent + " "); + writeStructFunctionDeclarations(w, getTypeName(child, false), child, indent + " "); + writeMembers(w, child, indent + " "); + w.write(indent + "};\n"); + w.write(indent + "typedef std::vector<" + typeName + "> " + typeName + "Vector;\n\n"); + w.write(indent + "typedef std::map<vespalib::string, " + typeName + "> " + typeName + "Map;\n\n"); + } + } + } + } + + void writeHeaderFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + "const vespalib::string & defName() const { return CONFIG_DEF_NAME; }\n" + + indent + "const vespalib::string & defVersion() const { return CONFIG_DEF_VERSION; }\n" + + indent + "const vespalib::string & defMd5() const { return CONFIG_DEF_MD5; }\n" + + indent + "const vespalib::string & defNamespace() const { return CONFIG_DEF_NAMESPACE; }\n" + + indent + "void serialize(::config::ConfigDataBuffer & __buffer) const;\n"); + writeConfigClassFunctionDeclarations(w, "Internal" + className + "Type", node, indent); + } + + void writeConfigClassFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const ::config::ConfigValue & __value);\n"); + w.write(indent + className + "(const ::config::ConfigDataBuffer & __value);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + } + + void writeStructFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const std::vector<vespalib::string> & __lines);\n"); + w.write(indent + className + "(const vespalib::slime::Inspector & __inspector);\n"); + w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); + writeCommonFunctionDeclarations(w, className, node, indent); + w.write(indent + "void serialize(vespalib::slime::Cursor & __cursor) const;\n"); + } + + void writeClassCopyConstructorDeclaration(Writer w, String className, CNode node, String indent) throws IOException { + w.write(indent + className + "(const " + className + " & __rhs);\n"); + } + + + void writeClassCopyConstructorDefinitionCommon(Writer w, CNode node) throws IOException { + for (int i = 0; i < node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" " + childName + "(__rhs." + childName + ")"); + } else { + w.write(",\n " + childName + "(__rhs." + childName + ")"); + } + } + } + + void writeConfigClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + w.write(parent + className + "(const " + className + " & __rhs)\n"); + w.write(" : ConfigInstance(),\n "); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeClassCopyConstructorDefinition(Writer w, String parent, String className, CNode node) throws IOException { + String typeName = getTypeName(node, false); + // Write empty constructor + w.write(parent + typeName + "(const " + typeName + " & __rhs)\n"); + w.write(" :"); + writeClassCopyConstructorDefinitionCommon(w, node); + w.write("\n" + + "{\n" + + "}\n" + + "\n" + ); + } + + void writeCommonFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { + w.write("" + + indent + className + "();\n"); + writeClassCopyConstructorDeclaration(w, className, node, indent); + w.write("" + + "\n" + + indent + "bool operator==(const " + className + "& __rhs) const;\n" + + indent + "bool operator!=(const " + className + "& __rhs) const;\n" + + "\n" + ); + } + + static String getTypeName(CNode node, boolean includeArray) { + String type = null; + if (node instanceof InnerCNode) { + InnerCNode innerNode = (InnerCNode) node; + type = getTypeName(innerNode.getName()); + } else if (node instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) node; + if (leaf.getType().equals("bool")) { + type = "bool"; + } else if (leaf.getType().equals("int")) { + type = "int32_t"; + } else if (leaf.getType().equals("long")) { + type = "int64_t"; + } else if (leaf.getType().equals("double")) { + type = "double"; + } else if (leaf.getType().equals("enum")) { + type = getTypeName(node.getName()); + } else if (leaf.getType().equals("string")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("reference")) { + type = "vespalib::string"; + } else if (leaf.getType().equals("file")) { + type = "vespalib::string"; + } else { + throw new IllegalArgumentException("Unknown leaf datatype " + leaf.getType()); + } + } + if (type == null) { + throw new IllegalArgumentException("Unknown node " + node); + } + if (node.isArray && includeArray) { + if (vectorTypeDefs.containsKey(type)) { + type = vectorTypeDefs.get(type); + } else { + type = type + "Vector"; + } + } else if (node.isMap && includeArray) { + if (mapTypeDefs.containsKey(type)) { + type = mapTypeDefs.get(type); + } else { + type = type + "Map"; + } + } + return type; + } + + void writeStaticMemberDeclarations(Writer w, String indent) throws IOException { + w.write("" + + indent + "static const vespalib::string CONFIG_DEF_MD5;\n" + + indent + "static const vespalib::string CONFIG_DEF_VERSION;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAME;\n" + + indent + "static const vespalib::string CONFIG_DEF_NAMESPACE;\n" + + indent + "static const std::vector<vespalib::string> CONFIG_DEF_SCHEMA;\n" + + indent + "static const int64_t CONFIG_DEF_SERIALIZE_VERSION;\n" + + "\n" + ); + } + + void writeComment(Writer w, String indent, String comment, boolean javadoc) + throws IOException + { + /** If simple one liner comment, write on one line. */ + if (javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 7)) + { + w.write(indent + "/** " + comment + " */\n"); + return; + } else if (!javadoc && comment.indexOf('\n') == -1 + && comment.length() <= 80 - (indent.length() + 3)) + { + w.write(indent + "// " + comment + "\n"); + return; + } + /** If not we need to write multi line comment. */ + int maxLineLen = 80 - (indent.length() + 3); + if (javadoc) w.write(indent + "/**\n"); + do { + String current; + // Extract first line to write + int newLine = comment.indexOf('\n'); + if (newLine == -1) { + current = comment; + comment = ""; + } else { + current = comment.substring(0, newLine); + comment = comment.substring(newLine + 1); + } + // If line too long, cut it in two + if (current.length() > maxLineLen) { + int spaceIndex = current.lastIndexOf(' ', maxLineLen); + if (spaceIndex >= maxLineLen - 15) { + comment = current.substring(spaceIndex + 1) + + "\n" + comment; + current = current.substring(0, spaceIndex); + } else { + comment = current.substring(maxLineLen) + "\n" + comment; + current = current.substring(0, maxLineLen) + "-"; + } + } + w.write(indent + (javadoc ? " * " : "// ") + current + "\n"); + } while (comment.length() > 0); + if (javadoc) w.write(indent + " */\n"); + } + + void writeMembers(Writer w, CNode node, String indent) throws IOException { + for (CNode child : node.getChildren()) { + String typeName = getTypeName(child, true); + if (child.getComment().length() > 0) { + String comment = child.getComment(); + int index; + do { + index = comment.indexOf("\n\n"); + if (index == -1) break; + String next = comment.substring(0, index); + comment = comment.substring(index + 2); + w.write("\n"); + writeComment(w, indent, next, false); + } while (true); + w.write("\n"); + writeComment(w, indent, comment, true); + } + w.write(indent + typeName + " " + getIdentifier(child.getName()) + ";"); + if (child instanceof LeafCNode) { + LeafCNode leaf = (LeafCNode) child; + DefaultValue value = leaf.getDefaultValue(); + if (value != null) { + w.write(" // Default: " + value.getStringRepresentation()); + } + } + w.write("\n"); + } + } + + void writeHeaderTypeDefs(Writer w, CNode root, String indent) throws IOException { + w.write(indent + "typedef std::unique_ptr<const " + getInternalClassName(root) + "> UP;\n"); + for (Map.Entry<String, String> entry : vectorTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String vectorName = entry.getValue(); + String typeDef = "typedef std::vector<" + typeName + "> " + vectorName; + w.write(indent + typeDef + ";\n"); + } + for (Map.Entry<String, String> entry : mapTypeDefs.entrySet()) { + String typeName = entry.getKey(); + String mapName = entry.getValue(); + String typeDef = "typedef std::map<vespalib::string, " + typeName + "> " + mapName; + w.write(indent + typeDef + ";\n"); + } + } + + private static String getInternalClassName(CNode root) { + return "Internal" + getTypeName(root, false) + "Type"; + } + + void writeHeaderFooter(Writer w, CNode root) throws IOException { + String [] namespaceList = generateCppNameSpace(root); + String namespaceDefine = generateCppNameSpaceDefine(namespaceList); + + String className = getTypeName(root, false); + String defineName = namespaceDefine + "_" + getDefineName(className); + + w.write("" + + "};\n" + + "\n" + + "} // namespace internal\n\n"); + + w.write("typedef internal::" + getInternalClassName(root) + " " + className + "ConfigBuilder;\n"); + w.write("typedef const internal::" + getInternalClassName(root) + " " + className + "Config;\n"); + w.write("\n"); + writeNameSpaceEnd(w, namespaceList); + w.write("#endif // VESPA_config_" + defineName + "_H\n"); + } + + void writeBodyFile(Writer w, CNode root, String subdir, NormalizedDefinition nd) throws IOException { + writeBodyHeader(w, root, subdir); + writeStaticMemberDefinitions(w, root, nd); + writeDefinition(w, root, null); + writeBodyFooter(w, root); + } + + void writeBodyHeader(Writer w, CNode root, String subdir) throws IOException { + if (subdir == null) { + w.write("#include \"" + getFileName(root, "h") + "\""); + } else { + w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">"); + } + w.write("\n"); + w.write("#include <set>\n"); + w.write("#include <vespa/config/common/configparser.h>\n"); + w.write("#include <vespa/vespalib/data/slime/convenience.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/config/configgen/vector_inserter.h>\n"); + w.write("#include <vespa/config/configgen/map_inserter.h>\n"); + w.write("\n\n"); + writeNameSpaceBegin(w, generateCppNameSpace(root)); + w.write("\n"); + w.write("namespace internal {\n\n"); + w.write("using ::config::ConfigParser;\n"); + w.write("using ::config::InvalidConfigException;\n"); + w.write("using ::config::ConfigInstance;\n"); + w.write("using ::config::ConfigValue;\n"); + w.write("using namespace vespalib::slime::convenience;\n"); + w.write("\n"); + } + + void writeStaticMemberDefinitions(Writer w, CNode root, NormalizedDefinition nd) throws IOException { + String typeName = getInternalClassName(root); + w.write("const vespalib::string " + typeName + "::CONFIG_DEF_MD5(\"" + root.defMd5 + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_VERSION(\"" + root.defVersion + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAME(\"" + root.defName + "\");\n" + + "const vespalib::string " + typeName + "::CONFIG_DEF_NAMESPACE(\"" + root.getNamespace() + "\");\n" + + "const int64_t " + typeName + "::CONFIG_DEF_SERIALIZE_VERSION(1);\n"); + w.write("const static vespalib::string __internalDefSchema[] = {\n"); + for (String line : nd.getNormalizedContent()) { + w.write("\"" + line.replace("\"", "\\\"") + "\",\n"); + } + w.write("};\n"); + w.write("const std::vector<vespalib::string> " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n"); + w.write(" __internalDefSchema + (sizeof(__internalDefSchema) / \n"); + w.write(" sizeof(__internalDefSchema[0])));\n"); + w.write("\n"); + } + + void writeDefinition(Writer w, CNode node, String parent) throws IOException { + boolean root = false; + if (parent == null) { + parent = getInternalClassName(node) + "::"; + root = true; + } + java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + for (CNode child : node.getChildren()) { + boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); + if (complexType && !declaredTypes.contains(child.getName())) { + String typeName = getTypeName(child, false); + declaredTypes.add(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + LeafCNode.EnumLeaf leaf = (LeafCNode.EnumLeaf) child; + // Definition of getType(string) + w.write(parent + typeName + "\n" + + parent + "get" + typeName + "(const vespalib::string& name)\n" + + "{\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" " + (i != 0 ? "} else " : "")); + w.write("if (name == \"" + leaf.getLegalValues()[i] + "\") {\n" + + " return " + leaf.getLegalValues()[i] + ";\n"); + } + w.write(" } else {\n" + + " throw InvalidConfigException(\"Illegal enum value '\" + name + \"'\");\n" + + " }\n" + + "}\n" + + "\n" + ); + // Definition of getTypeName(enum) + w.write("vespalib::string\n" + + parent + "get" + typeName + "Name(" + typeName + " t)\n" + + "{\n" + + " switch (t) {\n" + ); + for (int i=0; i<leaf.getLegalValues().length; ++i) { + w.write(" case " + leaf.getLegalValues()[i] + ": return \"" + leaf.getLegalValues()[i] + "\";\n"); + } + w.write(" default:\n" + + " {\n" + + " vespalib::asciistream ost;\n" + + " ost << \"UNKNOWN(\" << t << \")\";\n" + + " return ost.str();\n" + + " }\n" + + " }\n" + + "}\n" + + "\n" + ); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" throw InvalidConfigException(\"Value for '\" + __fieldName + \"' required but not found\");\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write("}\n"); + w.write(parent + typeName + " " + parent + "Internal" + typeName + "Converter::operator()(const ::vespalib::slime::Inspector & __inspector, " + typeName + " __eDefault) {\n"); + w.write(" if (__inspector.valid()) {\n"); + w.write(" return " + parent + "get" + typeName + "(__inspector.asString().make_string());\n"); + w.write(" }\n"); + w.write(" return __eDefault;\n"); + w.write("}\n\n"); + } else { + writeDefinition(w, child, parent + typeName + "::"); + } + } + } + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + // Write empty constructor + w.write(parent + typeName + "()\n"); + for (int i=0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i == 0) { + w.write(" : " + childName + "("); + } else { + w.write("),\n " + childName + "("); + } + if (child.isArray || child.isMap) { + // Default array for empty constructor is empty array. + } else if (child instanceof LeafCNode) { // If we have a default value, use that.. + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue() != null) { + w.write(getDefaultValue(leaf)); + } else { + // Defines empty constructor defaults for primitives without default set + if (leaf.getType().equals("bool")) { + w.write("false"); + } else if (leaf.getType().equals("int")) { + w.write("0"); + } else if (leaf.getType().equals("double")) { + w.write("0"); + } else if (leaf.getType().equals("string")) { + } else if (leaf.getType().equals("enum")) { + LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) leaf; + w.write(enumNode.getLegalValues()[0]); + } else if (leaf.getType().equals("reference")) { + } else if (leaf.getType().equals("file")) { + } + } + } + // If we hit neither else, we're an inner node, thus special type that has its own empty constructor + } + if (node.getChildren().length > 0) + w.write(")\n"); + w.write("" + + "{\n" + + "}\n" + + "\n" + ); + // Write copy constructor + if (root) + writeConfigClassCopyConstructorDefinition(w, parent, typeName, node); + else + writeClassCopyConstructorDefinition(w, parent, typeName, node); + + // Write parsing constructor + String indent = " "; + if (root) { + w.write(typeName + "::" + typeName + "(const ConfigValue & __value)\n" + + "{\n" + + indent + "try {\n"); + indent = " "; + w.write(indent + "const std::vector<vespalib::string> & __lines(__value.getLines());\n"); + } else { + w.write(parent + typeName + "(const std::vector<vespalib::string> & __lines)\n" + + "{\n"); + } + w.write("" + + indent + "std::set<vespalib::string> __remainingValuesToParse(" + + "__lines.begin(), __lines.end());\n"); + w.write(indent + "for(std::set<vespalib::string>::iterator __rVTPiter = __remainingValuesToParse.begin();\n" + + indent + " __rVTPiter != __remainingValuesToParse.end();)\n" + + indent + "{\n" + + indent + " if (ConfigParser::stripWhitespace(*__rVTPiter).empty()) {\n" + + indent + " std::set<vespalib::string>::iterator __rVTPiter2 = __rVTPiter++;\n" + + indent + " __remainingValuesToParse.erase(__rVTPiter2);\n" + + indent + " } else {\n" + + indent + " ++__rVTPiter;\n" + + indent + " }\n" + + indent + "}\n"); + for (CNode child : node.getChildren()) { + String childType = getTypeName(child, false); + String childName = getIdentifier(child.getName()); + if (child instanceof LeafCNode.EnumLeaf) { + if (child.isArray) { + w.write(indent + "std::vector<vespalib::string> " + childName + "__ValueList(\n "); + } else if (child.isMap) { + w.write(indent + "std::map<vespalib::string, vespalib::string> " + childName + "__ValueMap(\n "); + } else { + w.write(indent + childName + " = get" + childType + "("); + } + childType = "vespalib::string"; + } else { + w.write(indent + childName + " = "); + } + if (child.isArray) { + w.write("ConfigParser::parseArray<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else if (child.isMap) { + w.write("ConfigParser::parseMap<" + childType + ">(\"" + + child.getName() + "\", __lines)"); + } else { + if (child instanceof LeafCNode) { + w.write("ConfigParser::parse<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } else { + w.write("ConfigParser::parseStruct<" + childType + ">(\"" + + child.getName() + "\", __lines"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + if (leaf.getDefaultValue().getValue() != null) { + String defaultVal = getDefaultValue(leaf); + if (leaf instanceof LeafCNode.EnumLeaf) { + defaultVal = '"' + defaultVal + '"'; + } + w.write(", " + defaultVal); + } + } + w.write(")"); + } + if (child instanceof LeafCNode.EnumLeaf) { + childType = getTypeName(child, false); + w.write(");\n"); + if (child.isArray) { + w.write(indent + childName + ".reserve(" + childName + "__ValueList.size());\n" + + indent + "for (std::vector<vespalib::string>::const_iterator __it\n" + + indent + " = " + childName + "__ValueList.begin();\n" + + indent + " __it != " + childName + "__ValueList.end(); ++__it)\n" + + indent + "{\n" + + indent + " " + childName + ".push_back(get" + childType + "(*__it));\n" + + indent + "}\n" + ); + } else if (child.isMap) { + w.write(indent + "typedef std::map<vespalib::string, vespalib::string> __ValueMap;\n"); + w.write(indent + "for (__ValueMap::iterator __it(" + childName + "__ValueMap.begin()), __mt(" + childName + "__ValueMap.end()); __it != __mt; __it++) {\n" + + " " + childName + "[__it->first] = get" + childType + "(__it->second);\n" + + "}\n" + ); + } + } else { + w.write(";\n"); + } + w.write(indent + "ConfigParser::stripLinesForKey(\"" + + child.getName() + "\", " + + "__remainingValuesToParse);\n"); + } + if (root) { + indent = " "; + w.write(indent + "} catch (InvalidConfigException & __ice) {\n"); + w.write(indent + " throw InvalidConfigException(\"Error parsing config '\" + CONFIG_DEF_NAME + \"' in namespace '\" + CONFIG_DEF_NAMESPACE + \"'" + + ": \" + __ice.getMessage());\n" + + indent + "}\n"); + } + w.write("}\n" + + "\n" + ); + // Write operator== + String lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator==(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return (" + ); + for (int i = 0; i<node.getChildren().length; ++i) { + CNode child = node.getChildren()[i]; + String childName = getIdentifier(child.getName()); + if (i != 0) { + w.write(" &&\n "); + } + w.write(childName + " == __rhs." + childName); + } + w.write(");\n" + + "}\n" + + "\n" + ); + // Write operator!= + lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); + w.write("bool\n" + + parent + lineBreak + "operator!=(const " + typeName + "& __rhs) const\n" + + "{\n" + + " return !(operator==(__rhs));\n" + + "}\n" + + "\n" + ); + writeSlimeEncoder(w, node, parent, root); + writeSlimeDecoder(w, node, parent, root); + writeSlimeConstructor(w, node, parent, root); + } + + public void writeSlimeEncoder(Writer w, CNode node, String parent, boolean root) throws IOException + { + String indent = " "; + if (root) { + w.write("void\n" + + parent + "serialize(::config::ConfigDataBuffer & __buffer) const\n" + + "{\n"); + w.write(indent + "vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Cursor & __croot = __slime.setObject();\n"); + w.write(indent + "__croot.setDouble(\"version\", CONFIG_DEF_SERIALIZE_VERSION);\n"); + w.write(indent + "vespalib::slime::Cursor & __key = __croot.setObject(\"configKey\");\n"); + w.write(indent + "__key.setString(\"defName\", vespalib::slime::Memory(CONFIG_DEF_NAME));\n"); + w.write(indent + "__key.setString(\"defNamespace\", vespalib::slime::Memory(CONFIG_DEF_NAMESPACE));\n"); + w.write(indent + "__key.setString(\"defMd5\", vespalib::slime::Memory(CONFIG_DEF_MD5));\n"); + w.write(indent + "vespalib::slime::Cursor & __keySchema =__key.setArray(\"defSchema\");\n"); + w.write(indent + "for (size_t i = 0; i < CONFIG_DEF_SCHEMA.size(); i++) {\n"); + w.write(indent + " __keySchema.addString(vespalib::slime::Memory(CONFIG_DEF_SCHEMA[i]));\n"); + w.write(indent + "}\n"); + w.write(indent + "vespalib::slime::Cursor & __cursor = __croot.setObject(\"configPayload\");\n"); + } else { + w.write("void\n" + + parent + "serialize(vespalib::slime::Cursor & __cursor) const\n" + + "{\n"); + } + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + w.write(indent + "{\n"); + indent = " "; + w.write(indent + "vespalib::slime::Cursor & __c = __cursor.setObject(\"" + child.getName() + "\");\n"); + if (child.isArray) { + w.write(indent + "__c.setString(\"type\", \"array\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + w.write(indent + "for (size_t __i = 0; __i < " + childName + ".size(); __i++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "[__i]));\n"); + } else { + w.write("(\"value\", " + childName + "[__i]);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " " + childName + "[__i].serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "__c.setString(\"type\", \"map\");\n"); + w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); + String childMapType = getTypeName(child, true); + w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n"); + w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); + w.write(indent + " __c3.setString(\"key\", vespalib::slime::Memory(it->first));\n"); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); + w.write(indent + " __c3.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(it->second)));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + " __c3.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(it->second));\n"); + } else { + w.write("(\"value\", it->second);\n"); + } + } else { + w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); + w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); + w.write(indent + " it->second.serialize(__c4);\n"); + } + w.write(indent + "}\n"); + } else { + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write(indent + "__c.setString(\"type\", \"enum\");\n"); + w.write(indent + "__c.set" + repType); + w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + ")));\n"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + w.write(indent + "__c.setString(\"type\", \"" + type + "\");\n"); + w.write(indent + "__c.set" + repType); + if ("String".equals(repType)) { + w.write("(\"value\", vespalib::slime::Memory(" + childName + "));\n"); + } else { + w.write("(\"value\", " + childName + ");\n"); + } + } else { + w.write(indent + "__c.setString(\"type\", \"struct\");\n"); + w.write(indent + "Cursor & __c2 = __c.setObject(\"value\");\n"); + w.write(indent + childName + ".serialize(__c2);\n"); + } + } + indent = " "; + w.write(indent + "}\n"); + + } + w.write("}\n\n"); + } + + public void writeSlimeDecoder(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigDataBuffer & __buffer)\n" + + "{\n"); + w.write(indent + "const vespalib::Slime & __slime(__buffer.slimeObject());\n"); + w.write(indent + "vespalib::slime::Inspector & __croot = __slime.get();\n"); + w.write(indent + "vespalib::slime::Inspector & __inspector = __croot[\"configPayload\"];\n"); + } else { + w.write("" + + parent + typeName + "(const vespalib::slime::Inspector & __inspector)\n" + + "{\n"); + } + + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String inspectorLine = "__inspector[\"" + child.getName() + "\"][\"value\"]"; + if (child.isArray) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + ".push_back("); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(");\n"); + w.write(indent + "}\n"); + } else if (child.isMap) { + w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); + w.write(indent + " " + childName + "[" + inspectorLine + "[__i][\"key\"].asString().make_string()] = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + w.write(";\n"); + w.write(indent + "}\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + ".as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + ".as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + ".as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + ")"); + } + w.write(";\n"); + } + } + w.write("}\n\n"); + } + + public void writeSlimeConstructor(Writer w, CNode node, String parent, boolean root) throws IOException { + String tmpName = getTypeName(node, false); + String typeName = root ? getInternalClassName(node) : tmpName; + String indent = " "; + if (root) { + w.write("" + + typeName + "::" + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } else { + w.write("" + + parent + typeName + "(const ::config::ConfigPayload & __payload)\n" + + "{\n"); + } + w.write(indent + "const vespalib::slime::Inspector & __inspector(__payload.get());\n"); + for (CNode child : node.getChildren()) { + String childName = getIdentifier(child.getName()); + String childType = getTypeName(child, false); + String childInspector = "__inspector[\"" + child.getName() + "\"]"; + if (child.isArray) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::VectorInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else if (child.isMap) { + String inserterName = "__" + childName + "Inserter"; + w.write(indent + "::config::internal::MapInserter<" + childType); + if (child instanceof LeafCNode.EnumLeaf) { + w.write(", Internal" + childType + "Converter"); + } + w.write("> " + inserterName + "(" + childName + ");\n"); + w.write(indent + childInspector + ".traverse(" + inserterName + ");\n"); + } else { + w.write(indent + childName + " = "); + if (child instanceof LeafCNode.EnumLeaf) { + w.write("Internal" + childType + "Converter"); + } else { + w.write("::config::internal::ValueConverter<" + childType + ">"); + } + if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { + LeafCNode leaf = (LeafCNode) child; + String defaultValue = getDefaultValue(leaf); + w.write("()(" + childInspector + ", " + defaultValue + ");\n"); + } else if (child instanceof InnerCNode) { + w.write("()(" + childInspector + ");\n"); + } else { + w.write("()(\"" + child.getName() + "\", " + childInspector + ");\n"); + } + } + } + w.write("}\n\n"); + } + + void writeBodyFooter(Writer w, CNode root) throws IOException { + w.write("} // namespace internal\n\n"); + writeNameSpaceEnd(w, generateCppNameSpace(root)); + } + + String getDefaultValue(LeafCNode leaf) { + String defaultVal = leaf.getDefaultValue().getStringRepresentation(); + if (leaf.getType().equals("string") && defaultVal.equals("null")) + throw new CodegenRuntimeException("Default value null not allowed for C++ config"); + if (leaf.getType().equals("long") && "-9223372036854775808".equals(defaultVal)) { + return "LONG_MIN"; + } else if (leaf.getType().equals("int") && "-2147483648".equals(defaultVal)) { + return "INT_MIN"; + } else { + return defaultVal; + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java new file mode 100644 index 00000000000..f58c202f70f --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java @@ -0,0 +1,274 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + */ +public class DefLine { + private final static Pattern defaultPattern = Pattern.compile("^\\s*default\\s*=\\s*(\\S+)"); + private final static Pattern rangePattern = Pattern.compile("^\\s*range\\s*=\\s*([\\(\\[].*?[\\)\\]])"); + private final static Pattern restartPattern = Pattern.compile("^\\s*restart\\s*"); + private final static Pattern wordPattern = Pattern.compile("\\S+"); + private final static Pattern enumPattern = Pattern.compile("\\s*\\{(\\s*\\w+\\s*)+(\\s*,\\s*\\w+\\s*)*\\s*\\}"); + private final static Pattern enumPattern2 = Pattern.compile("\\s*,\\s*"); + private final static Pattern wordPattern2 = Pattern.compile("\\w+"); + private final static Pattern digitPattern = Pattern.compile("\\d"); + private final static Pattern namePattern = Pattern.compile("\\s*[a-zA-Z0-9_]+\\s*"); + private final static Pattern whitespacePattern = Pattern.compile("\\s+"); + + private String name = null; + private final Type type = new Type(); + + private DefaultValue defaultValue = null; + + private String range = null; + private boolean restart = false; + + String enumString = null; + final String[] enumArray = null; + + private final static Pattern defaultNullPattern = Pattern.compile("^\\s*default\\s*=\\s*null"); + + public DefLine(String line) { + StringBuilder sb = new StringBuilder(line); + int parsed = parseNameType(sb); + sb.delete(0, parsed); + if (type.name.equals("enum")) { + parsed = parseEnum(sb); + sb.delete(0, parsed); + } + + while (sb.length() > 0) { + parsed = parseOptions(sb); + sb.delete(0, parsed); + } + validateName(); + validateReservedWords(); + } + + /** + * Currently (2012-03-05) not used. Ranges are not checked by the + */ + public String getRange() { + return range; + } + + public DefaultValue getDefault() { + return defaultValue; + } + + public String getName() { + return name; + } + + public Type getType() { + return type; + } + + public boolean getRestart() { + return restart; + } + + public String getEnumString() { + return enumString; + } + + public String[] getEnumArray() { + return enumArray; + } + + /** + * Special function that searches through s and returns the index + * of the first occurrence of " that is not escaped. + */ + private String findStringEnd(CharSequence s, int from) { + boolean escaped = false; + for (int i = from; i < s.length(); i++) { + switch (s.charAt(i)) { + case'\\': + escaped = !escaped; + break; + case'"': + if (!escaped) { + return s.subSequence(from, i).toString(); + } + break; + } + } + return null; + } + + + private int parseOptions(CharSequence string) { + Matcher defaultNullMatcher = defaultNullPattern.matcher(string); + Matcher defaultMatcher = defaultPattern.matcher(string); + Matcher rangeMatcher = rangePattern.matcher(string); + Matcher restartMatcher = restartPattern.matcher(string); + + if (defaultNullMatcher.find()) { + throw new IllegalArgumentException("Null default value is not allowed: " + string.toString()); + } else if (defaultMatcher.find()) { + String deflt = defaultMatcher.group(1); + if (deflt.charAt(0) == '"') { + int begin = defaultMatcher.start(1) + 1; + deflt = findStringEnd(string, begin); + if (deflt == null) { + throw new IllegalArgumentException(string.toString()); + } + defaultValue = new DefaultValue(deflt, type); + return begin + deflt.length() + 1; + } else { + defaultValue = new DefaultValue(deflt, type); + } + return defaultMatcher.end(); + } else if (rangeMatcher.find()) { + range = rangeMatcher.group(1); + return rangeMatcher.end(); + } else if (restartMatcher.find()) { + restart = true; + return restartMatcher.end(); + } else { + throw new IllegalArgumentException(string.toString()); + } + } + + private int parseNameType(CharSequence string) { + Matcher wordMatcher = wordPattern.matcher(string); + if (wordMatcher.find()) { + name = wordMatcher.group(); + } + if (wordMatcher.find()) { + type.name = wordMatcher.group(); + } + if (type.name == null || name == null) { + throw new IllegalArgumentException(string.toString()); + } + return wordMatcher.end(); + } + + private int parseEnum(CharSequence string) { + Matcher enumMatcher = enumPattern.matcher(string); + if (enumMatcher.find()) { + enumString = enumMatcher.group(0).trim(); + } + if (enumString == null) { + throw new IllegalArgumentException(string + " is not valid syntax"); + } + enumString = enumString.replaceFirst("\\{\\s*", ""); + enumString = enumString.replaceFirst("\\s*\\}", ""); + String result[] = enumPattern2.split(enumString); + type.enumArray = new String[result.length]; + for (int i = 0; i < result.length; i++) { + String s = result[i].trim(); + type.enumArray[i] = s; + Matcher wordMatcher2 = wordPattern2.matcher(s); + if (!wordMatcher2.matches()) { + throw new IllegalArgumentException(s + " is not valid syntax"); + } + } + return enumMatcher.end(); + } + + public static class Type { + String name; + String[] enumArray; + + public Type(String name) { + this.name=name; + } + + public Type() { + } + + public String getName() { + return name; + } + + public String[] getEnumArray() { + return enumArray; + } + + public Type setEnumArray(String[] enumArray) { + this.enumArray = enumArray; + return this; + } + + public String toString() { + return "type " + name; + } + + } + + // A naive approach to imitate the checking previously done in make-config-preproc.pl + // TODO: method too long + void validateName() { + Matcher digitMatcher; + Matcher nameMatcher; + Matcher whitespaceMatcher; + + boolean atStart = true; + boolean arrayOk = true; + boolean mapOk = true; + for (int i = 0; i < name.length(); i++) { + String s = name.substring(i, i + 1); + digitMatcher = digitPattern.matcher(s); + nameMatcher = namePattern.matcher(s); + whitespaceMatcher = whitespacePattern.matcher(s); + if (atStart) { + if (digitMatcher.matches()) { + throw new IllegalArgumentException(name + " must start with a non-digit character"); + } + if (!nameMatcher.matches()) { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + atStart = false; + } else { + if (nameMatcher.matches()) { + // do nothing + } else if (s.equals(".")) { + arrayOk = true; + mapOk = true; + atStart = true; + } else if (s.equals("[")) { + if (!arrayOk) { + throw new IllegalArgumentException(name + " Arrays cannot be multidimensional"); + } + arrayOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("]"))) { + throw new IllegalArgumentException(name + " Expected ] to terminate array definition"); + } + i++; + } else if (s.equals("{")) { + if (!mapOk) { + throw new IllegalArgumentException(name + " Maps cannot be multidimensional"); + } + mapOk = false; + if ((i > (name.length() - 2)) || !(name.substring(i + 1, i + 2).equals("}"))) { + throw new IllegalArgumentException(name + " Expected } to terminate map definition"); + } + i++; + } else if (whitespaceMatcher.matches()) { + break; + } else { + throw new IllegalArgumentException(name + " contains unexpected character"); + } + } + } + } + + void validateReservedWords() { + if (ReservedWords.isReservedWord(name)) { + throw new IllegalArgumentException(name + " is a reserved word in " + + ReservedWords.getLanguageForReservedWord(name)); + } + if (ReservedWords.capitalizedPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with an uppercase letter"); + } + if (ReservedWords.internalPrefixPattern.matcher(name).matches()) { + throw new IllegalArgumentException("'" + name + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'"); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java new file mode 100644 index 00000000000..0a6c225fa19 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class generates a tree of CNodes from a .def file. + * + * @author gjoranv + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class DefParser { + static final Pattern commentPattern = Pattern.compile("^\\s*#+\\s*(.*?)\\s*$"); + public static final Pattern versionPattern = Pattern.compile("^(version\\s*=\\s*)([0-9][0-9-]*)$"); + // Namespace must start with a letter, since Java (Java language Spec, section 3.8) and C++ identifiers cannot start with a digit + public static final Pattern namespacePattern = Pattern.compile("^(namespace\\s*=\\s*)(([a-z][a-z0-9_]*)+([.][a-z][a-z0-9_]*)*)$"); + + private final BufferedReader reader; + private final String name; + private InnerCNode root = null; + private NormalizedDefinition normalizedDefinition = null; + + + private String comment = ""; + + /** + * Creates a new parser for a .def file with the given name and that can be accessed by the given reader. + * + * @param name The name of the .def file (not including version number and the '.def' suffix). + * @param defReader A reader to the .def file. + */ + public DefParser(String name, Reader defReader) { + this.name = createName(name); + if (defReader == null) { + throw new CodegenRuntimeException("Must have a non-null reader for a .def file."); + } + if (defReader instanceof BufferedReader) { + reader = (BufferedReader)defReader; + } else { + reader = new BufferedReader(defReader); + } + } + + // If name contains namespace, return just name + private String createName(String name) { + if (name.contains(".")) { + return name.substring(name.lastIndexOf(".") + 1); + } else { + return name; + } + } + + /** + * Parses the .def file upon the initial call. Subsequent calls returns the result from the initial call. + * + * @return A tree of CNodes representing this instance's .def file. + * @throws CodegenRuntimeException upon errors. + */ + public InnerCNode getTree() throws CodegenRuntimeException { + try { + if (root == null) parse(); + } catch (DefParserException | IOException e) { + throw new CodegenRuntimeException("Error parsing or reading config definition." + e.getMessage(), e); + } + return root; + } + + /** + * Parses the input from the reader and builds a tree of CNodes representing the .def file. + * + * @throws IOException upon reader errors. + * @throws DefParserException upon parsing errors. + */ + void parse() throws IOException, DefParserException { + root = new InnerCNode(name); + normalizedDefinition = new NormalizedDefinition(); + + String s; + List<String> originalInput = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + originalInput.add(s); + } + reader.close(); + + + // Parse and build tree of the original input + parseLines(root, originalInput, normalizedDefinition); + root.setMd5(normalizedDefinition.generateMd5Sum()); + } + + /** + * Parses one line from def-file and adds it to the tree. + * TODO: Method too long! + * + * @param root The root CNode in the tree. + * @param line A line from the def-file. + * @param nd A NormalizedDefinition object + * @throws IllegalArgumentException upon error in line. + */ + private void parseLine(CNode root, String line, NormalizedDefinition nd) throws IllegalArgumentException { + line = NormalizedDefinition.normalize(line); + line = line.trim(); + if (line.length() == 0) { + // If having empty lines in between comments, that is logically a break in the comment too + if (!comment.isEmpty()) { + comment += "\n"; + } + return; + } + Matcher commentMatch = commentPattern.matcher(line); + if (commentMatch.matches()) { + parseCommentLine(commentMatch); + return; + } + Matcher versionMatch = versionPattern.matcher(line); + if (versionMatch.matches()) { + parseVersionLine(versionMatch); + return; + } + Matcher namespaceMatcher = namespacePattern.matcher(line); + if (namespaceMatcher.matches()) { + parseNamespaceLine(namespaceMatcher.group(2)); + nd.addNormalizedLine(line); + return; + } + // Only add lines that are not version, namespace or comment lines + nd.addNormalizedLine(line); + DefLine defLine = new DefLine(line); + root.setLeaf(root.getName() + "." + defLine.getName(), defLine, comment); + comment = ""; + } + + private void parseCommentLine(Matcher commentMatch) { + if (!comment.isEmpty()) comment += "\n"; + String addition = commentMatch.group(1); + if (addition.isEmpty()) addition = " "; + comment += addition; + } + + private void parseVersionLine(Matcher matcher) { + root.setVersion(matcher.group(2)); + root.setComment(comment); + comment = ""; + } + + private void parseNamespaceLine(String namespace) { + if (namespace.startsWith("com.yahoo.")) + throw new IllegalArgumentException("Remove 'com.yahoo.' from the namespace '" + namespace + + "' - it will be automatically added to the java package name."); + root.setNamespace(namespace); + root.setComment(comment); + comment = ""; + } + + void parseLines(CNode root, List<String> defLines, NormalizedDefinition nd) throws DefParserException { + DefParserException failure = null; + int lineNumber = 1; + for (String line : defLines) { + try { + parseLine(root, line, nd); + lineNumber++; + } catch (IllegalArgumentException e) { + String msg = "Error when parsing line " + lineNumber + ": " + line + "\n" + e.getMessage(); + failure = new DefParserException(msg, e); + break; + } + } + + if (failure != null) { + throw (failure); + } + } + + public NormalizedDefinition getNormalizedDefinition() { + return normalizedDefinition; + } + + /** + * For debugging - dump the tree from the given root to System.out. + */ + public static void dumpTree(CNode root, String indent) { + StringBuilder sb = new StringBuilder(indent + root.getName()); + if (root instanceof LeafCNode) { + LeafCNode leaf = ((LeafCNode)root); + if (leaf.getDefaultValue() != null) { + sb.append(" = ").append(((LeafCNode)root).getDefaultValue().getValue()); + } + } + System.out.println(sb.toString()); + if (!root.getComment().isEmpty()) { + String comment = root.getComment(); + if (comment.contains("\n")) { + comment = comment.substring(0, comment.indexOf("\n")) + "..."; + } + if (comment.length() > 60) { + comment = comment.substring(0, 57) + "..."; + } + System.out.println(indent + " comment: " + comment); + } + CNode[] children = root.getChildren(); + for (CNode c : children) { + dumpTree(c, indent + " "); + } + + } + + class DefParserException extends Exception { + DefParserException(String s, Throwable cause) { + super(s, cause); + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java new file mode 100644 index 00000000000..8518ff04573 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefaultValue.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * An immutable class representing a default value of a config variable + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DefaultValue { + + private String value = null; + + // The variable type. Always set UNLESS the value is null. + private DefLine.Type type = null; + + + /** + * Null value + */ + public DefaultValue() { + } + + /** + * A default value with the given value and type. + */ + public DefaultValue(String value, DefLine.Type type) { + this.value = value; + this.type = type; + } + + /** + * Returns the toString of the default value. + */ + public String getValue() { + return value; + } + + /** + * Returns the string representation of this value + */ + public String getStringRepresentation() { + if (value == null) + return "null"; + else if ("bool".equals(type.getName())) + return value; + else if ("int".equals(type.getName())) + return value; + else if ("long".equals(type.getName())) + return value; + else if ("double".equals(type.getName())) + return value; + else if ("enum".equals(type.getName())) + return value; + else { + // building a string, do unicode-escaping + StringBuilder sb = new StringBuilder(); + for (char c : value.toCharArray()) { + if (c > '\u007f') { + sb.append(String.format("\\u%04X", (int) c)); + } else { + sb.append(c); + } + } + return "\"" + sb.toString() + "\""; + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java new file mode 100644 index 00000000000..b7803fc8c22 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/InnerCNode.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.Map; +import java.util.LinkedHashMap; + +/** + * Represents an inner node in the configuration tree. + * + * @author gjoranv + */ +public class InnerCNode extends CNode { + + /** + * The children of this Node. Mapped using their short name as + * string. This variable is null only if Node is a leaf Node. + */ + private final Map<String, CNode> children = new LinkedHashMap<String, CNode>(); + private boolean restart = false; + + /** + * Constructor for the root node. + */ + public InnerCNode(String name) { + super(null, name.split("\\.def")[0]); + defName = this.name; + } + + /** + * Constructor for inner nodes. + */ + private InnerCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public CNode[] getChildren() { + CNode[] ret = new CNode[children.size()]; + children.values().toArray(ret); + return ret; + } + + /** + * Access to children for testing + * @return modifiable children map + */ + public Map<String, CNode> children() { + return children; + } + + @Override + public CNode getChild(String name) { + return children.get(name); + } + + /** + * Returns and eventually creates the given subnode. + */ + private CNode createOrGetChild(DefLine.Type type, String name) throws IllegalArgumentException { + String key = name; + int split = name.indexOf('.'); + CNode newChild; + if (split != -1) { + key = name.substring(0, split).trim(); + newChild = new InnerCNode(this, key); + } else { + newChild = LeafCNode.newInstance(type, this, key); + if (newChild == null) + throw new IllegalArgumentException + ("Could not create " + type.name + " " + name); + } + return children.containsKey(newChild.getName()) + ? children.get(newChild.getName()) + : newChild; + } + + /** + * Adds a child to this node with the given type, name and value. Necessary children on the path + * to the given leaf node will be added as well. + * @param name the full name/path of the node to add. + * @param defLine the parsed .def-file line to add. + * @param comment comment extracted from the .def-file. + */ + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + if (name.indexOf('.') < 0) { + throw new IllegalArgumentException("Parameter with name '" + name + + "' cannot be a leaf node as it has already been declared as an inner node."); + } + checkMyName(name.substring(0, name.indexOf('.'))); + String childName = name.substring(name.indexOf('.') + 1); + + CNode child = createOrGetChild(defLine.getType(), childName); +/* + System.out.println("\nAdding child name: " + name); + System.out.println(" getName: " + child.getName()); + System.out.println(" full name: " + child.getFullName()); + System.out.println(" classname: " + child.getClassName()); + System.out.println(" full classname: " + child.getFullClassName()); +*/ + restart |= defLine.getRestart(); + child.setLeaf(childName, defLine, comment); + children.put(child.getName(), child); + } + + @Override + public boolean needRestart() { + return restart; + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java new file mode 100644 index 00000000000..3cd7b61ba55 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/LeafCNode.java @@ -0,0 +1,267 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +/** + * @author gjoranv + */ +public abstract class LeafCNode extends CNode { + + private boolean isInitialized = false; + private DefaultValue defaultValue = null; + private boolean restart = false; + + /** + * Constructor for the leaf nodes. + */ + protected LeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name) { + try { + switch (type.name) { + case "int": + return new IntegerLeaf(parent, name); + case "long": + return new LongLeaf(parent, name); + case "double": + return new DoubleLeaf(parent, name); + case "bool": + return new BooleanLeaf(parent, name); + case "string": + return new StringLeaf(parent, name); + case "reference": + return new ReferenceLeaf(parent, name); + case "file": + return new FileLeaf(parent, name); + case "path": + return new PathLeaf(parent, name); + case "enum": + return new EnumLeaf(parent, name, type.enumArray); + default: + return null; + } + } catch (NumberFormatException e) { + return null; + } + } + + public static LeafCNode newInstance(DefLine.Type type, InnerCNode parent, String name, String defVal) { + LeafCNode ret = newInstance(type, parent, name); + if (defVal!=null) { + DefaultValue def = new DefaultValue(defVal, type); + ret.setDefaultValue(def); + } + return ret; + } + + public abstract String getType(); + + @Override + public CNode[] getChildren() { + return new CNode[0]; + } + + @Override + public CNode getChild(String name) { + return null; + } + + public DefaultValue getDefaultValue() { + return defaultValue; + } + + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + /** + * @param defaultValue The value to check. + * @throws IllegalArgumentException if the value is illegal according to the node type. + */ + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + } + + @Override + protected void setLeaf(String name, DefLine defLine, String comment) + throws IllegalArgumentException { + DefLine.Type type = defLine.getType(); + // TODO: why the !is... conditions? + if (!isMap && !isArray && isInitialized) { + throw new IllegalArgumentException(name + " is already defined"); + } + isInitialized = true; + checkMyName(name); + if (!type.name.equalsIgnoreCase(getType())) { + throw new IllegalArgumentException("Type " + type.name + " does not match " + getType()); + } + setValue(defLine.getDefault()); + setComment(comment); + restart |= defLine.getRestart(); + } + + @Override + public boolean needRestart() { + return restart; + } + + public final void setValue(DefaultValue defaultValue) throws IllegalArgumentException { + try { + checkDefaultValue(defaultValue); + setDefaultValue(defaultValue); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException + ("Invalid default value", e); + } + } + + /** + * Superclass for leaf nodes that should not generate class. + */ + public static abstract class NoClassLeafCNode extends LeafCNode { + protected NoClassLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + } + + /** + * Superclass for no-class leaf nodes that cannot have a default. + */ + public static abstract class NoClassNoDefaultLeafCNode extends LeafCNode { + protected NoClassNoDefaultLeafCNode(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public LeafCNode setDefaultValue(DefaultValue defaultValue) { + if (defaultValue != null) + throw new IllegalArgumentException("Parameters of type '" + getType() + "' cannot have a default value."); + return this; + } + } + + public static class IntegerLeaf extends NoClassLeafCNode { + protected IntegerLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "int"; + } + } + + public static class LongLeaf extends NoClassLeafCNode { + protected LongLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "long"; + } + } + + public static class DoubleLeaf extends NoClassLeafCNode { + protected DoubleLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "double"; + } + } + + public static class BooleanLeaf extends NoClassLeafCNode { + protected BooleanLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "bool"; + } + } + + public static class StringLeaf extends NoClassLeafCNode { + protected StringLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "string"; + } + } + + public static class ReferenceLeaf extends StringLeaf { + ReferenceLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "reference"; + } + } + + public static class FileLeaf extends NoClassNoDefaultLeafCNode { + FileLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "file"; + } + } + + public static class PathLeaf extends NoClassNoDefaultLeafCNode { + PathLeaf(InnerCNode parent, String name) { + super(parent, name); + } + + @Override + public String getType() { + return "path"; + } + } + + public static class EnumLeaf extends LeafCNode { + + private final String[] legalValues; + + protected EnumLeaf(InnerCNode parent, String name, String[] valArray) { + super(parent, name); + this.legalValues = valArray; + } + + @Override + public String getType() { + return "enum"; + } + + /** @return This enum's legal values. */ + public String[] getLegalValues() { + return legalValues; + } + + @Override + public void checkDefaultValue(DefaultValue defaultValue) throws IllegalArgumentException { + if ((defaultValue != null) && (defaultValue.getValue() != null)) { + String defaultString = null; + String value = defaultValue.getValue(); + for (String val : legalValues) { + if (value.equals(val)) { + defaultString = val; + } + } + if (defaultString == null) + throw new IllegalArgumentException("Could not initialize enum with: " + value); + } + } + } + +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java new file mode 100644 index 00000000000..f28549275ef --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.logging.Logger; + +/** + * This class generates code for a config class from a given def-file. + */ +public class MakeConfig { + + private final static Logger log = Logger.getLogger(MakeConfig.class.getName()); + + private final ClassBuilder classBuilder; + + public MakeConfig(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties properties) { + classBuilder = createClassBuilder(root, nd, path, properties); + } + + public static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, String path, MakeConfigProperties prop) { + if (prop.language.equals("cppng") || prop.language.equals("cpp")) + return new CppClassBuilder(root, nd, prop.destDir, prop.dirInRoot); + else + return new JavaClassBuilder(root, nd, prop.destDir); + } + + /** + * Generates the code and print it to this.out. + */ + void buildClasses() { + classBuilder.createConfigClasses(); + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java -Dconfig.dest=<dir> -Dconfig.spec=<path> [-Dconfig.lang=cpp -Dconfig.subdir=<dir>] [-Dconfig.dumpTree=true] MakeConfig"); + out.println(" (default language for generated code is Java)"); + } + + public static void main(String[] args) throws IOException, InterruptedException { + try { + MakeConfigProperties props = new MakeConfigProperties(); + for (File specFile : props.specFiles) { + String path = specFile.toURI().toString(); + String name = specFile.getName(); + if (name.endsWith(".def")) name = name.substring(0, name.length() - 4); + DefParser parser = new DefParser(name, new FileReader(specFile)); + InnerCNode configRoot = parser.getTree(); + checkNamespace(name, configRoot); + if (configRoot != null) { + MakeConfig mc = new MakeConfig(configRoot, parser.getNormalizedDefinition(), path, props); + mc.buildClasses(); + if (props.dumpTree) { + System.out.println("\nTree dump:"); + DefParser.dumpTree(configRoot, ""); + } + } else { + System.exit(1); + } + } + } catch (PropertyException e) { + System.out.println(Exceptions.toMessageString(e)); + printUsage(System.err); + System.exit(1); + } catch (CodegenRuntimeException e) { + System.out.println(Exceptions.toMessageString(e)); + System.exit(1); + } + } + + private static void checkNamespace(String name, InnerCNode configRoot) { + if (configRoot.defNamespace == null) + throw new IllegalArgumentException("In config definition '" + name + "': A namespace is required"); + } + + // The Exceptions class below is copied from vespajlib/com.yahoo.protect.Exceptions + + /** + * Helper methods for handling exceptions + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ + static class Exceptions { + + /** + * <p>Returns a use friendly error message string which includes information from all nested exceptions. + * + * <p>The form of this string is + * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> + * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. + */ + public static String toMessageString(Throwable t) { + StringBuilder b = new StringBuilder(); + String lastMessage = null; + String message; + for (; t != null; t = t.getCause(), lastMessage = message) { + message = getMessage(t); + if (message == null) continue; + if (lastMessage != null && lastMessage.equals(message)) continue; + if (b.length() > 0) + b.append(": "); + b.append(message); + } + return b.toString(); + } + + /** + * Returns a useful message from *this* exception, or null if none + */ + private static String getMessage(Throwable t) { + String message = t.getMessage(); + if (t.getCause() == null) { + if (message == null) return toShortClassName(t); + return message; + } else { + if (message == null) return null; + if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; + return message; + } + } + + private static String toShortClassName(Object o) { + String longName = o.getClass().getName(); + int lastDot = longName.lastIndexOf("."); + if (lastDot < 0) return longName; + return longName.substring(lastDot + 1); + } + } +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java new file mode 100644 index 00000000000..d055a1fe1fb --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.StringTokenizer; + +/** + * Encapsulates data extracted from system properties. + * + * @author <a href="gv@yahoo-inc.com">Gjoran Voldengen</a> + */ +public class MakeConfigProperties { + + private final List<String> legalLanguages = Arrays.asList("java", "cpp", "cppng" ); + + final File destDir; + final File[] specFiles; + final String language; + final String dirInRoot; // Where within fileroot to store generated class files + final boolean dumpTree; + final boolean generateFrameworkCode; + + MakeConfigProperties() throws PropertyException { + destDir = checkDestinationDir(); + specFiles = checkSpecificationFiles(); + language = checkLanguage(); + dirInRoot = checkDirInRoot(); + dumpTree = System.getProperty("config.dumpTree") != null && + System.getProperty("config.dumpTree").equalsIgnoreCase("true"); + generateFrameworkCode = System.getProperty("config.useFramework") == null || + System.getProperty("config.useFramework").equalsIgnoreCase("true"); + } + + private File checkDestinationDir() throws PropertyException { + String destination = System.getProperty("config.dest"); + if (destination == null) + throw new PropertyException("Missing property: config.dest."); + + File dir = new File(destination); + if (!dir.isDirectory()) { + throw new PropertyException("Could not find directory: " + dir.getPath()); + } + return dir; + } + + private String checkDirInRoot() throws PropertyException { + String dirInRoot = System.getProperty("config.subdir"); + // Optional parameter + if (dirInRoot == null) { return null; } + File f = new File(destDir, dirInRoot); + if (!f.isDirectory()) { + throw new PropertyException("Could not find directory: " + f.getPath()); + } + return dirInRoot; + } + + /** + * @return Desired programming language of generated code, default is "java". + * @throws PropertyException if supplied language is not a legal language. + */ + private String checkLanguage() throws PropertyException { + String inputLang = System.getProperty("config.lang", "java").toLowerCase(); + if (! legalLanguages.contains(inputLang)) { + throw new PropertyException + ("Unsupported code language: '" + inputLang + "'. Supported languages are: " + legalLanguages); + } + return inputLang; + } + + private static File[] checkSpecificationFiles() throws PropertyException { + String string = System.getProperty("config.spec"); + if (string == null) + throw new PropertyException("Missing property: config.spec."); + + StringTokenizer st = new StringTokenizer(string, ","); + if (st.countTokens() == 0) + throw new PropertyException("Missing property: config.spec."); + + File[] files = new File[st.countTokens()]; + for (int i = 0; st.hasMoreElements(); i++) { + files[i] = new File((String) st.nextElement()); + if (!files[i].isFile()) + throw new PropertyException("Could not read file " + files[i].getPath()); + } + return files; + } + +} + diff --git a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java new file mode 100644 index 00000000000..1847150d86d --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.text.DecimalFormat; +import java.security.MessageDigest; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * + * Does normalizing (removing comments, trimming whitespace etc.) and calculation of md5sum + * of config definitions + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class NormalizedDefinition { + /* Patterns used for finding ranges in config definitions */ + private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); + private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); + private MessageDigest md5; + + String defMd5 = null; + List<String> normalizedContent = null; + + public NormalizedDefinition() { + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (java.security.NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to create MD5 digest", e); + } + normalizedContent = new ArrayList<>(); + } + + public NormalizedDefinition normalize(BufferedReader reader) throws IOException { + String s; + List<String> input = new ArrayList<>(); + while ((s = reader.readLine()) != null) { + String normalized = normalize(s); + if (normalized.length() > 0) { + input.add(normalized); + } + } + normalizedContent = input; + return this; + } + + /** + * Normalizes a config definition line. Each string is normalized according to the + * rules of config and definition files before they are used: + * <ul> + * <li>Remove trailing space.<li> + * <li>Remove trailing comments, and spaces before trailing comments.</li> + * <li>Remove empty lines</li> + * <li>Keep comment lines</li> + * </ul> + * The supplied list is changed in-place + * + * @param line A config definition line + * @return a normalized config definition line + */ + public static String normalize(String line) { + //System.out.println("before line=" + line + ";"); + // Normalize line + line = line.trim(); + Matcher m = intPattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(0x7fffffff); + String formattedMin = new DecimalFormat("#.#").format(-0x80000000); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + m = doublePattern.matcher(line); + if (m.matches()) { + String formattedMax = new DecimalFormat("#.#").format(1e308); + String formattedMin = new DecimalFormat("#.#").format(-1e308); + line = line.replaceFirst("\\[,", "["+formattedMin+","); + line = line.replaceFirst(",\\]", ","+formattedMax+"]"); + } + line = removeComment(line); + if (!line.isEmpty()) { + line = stripSpaces(line); + line = line.replaceAll("\\s,", ","); // Remove space before comma (for enums) + line += "\n"; + } + //System.out.println("after line=" + line + ";"); + return line; + } + + // Removes comment char and text after it, unless comment char is inside a string + // Keeps comment lines (lines that start with #) + private static String removeComment(String line) { + int index = line.indexOf("#"); + if (!line.contains("#") || index == 0) return line; + + int firstQuote = line.indexOf("\""); + if (firstQuote > 0) { + int secondQuote = line.indexOf("\"", firstQuote + 1); + if (index > secondQuote) { + line = line.substring(0, index); + line = line.trim(); + } + } else { + line = line.substring(0, index); + line = line.trim(); + } + + return line; + } + + public void addNormalizedLine(String line) { + normalizedContent.add(line); + } + + public String generateMd5Sum() { + for (String line : normalizedContent) { + String s = normalize(line); + if (!s.isEmpty()) { + md5.update(toBytes(s)); + } + } + defMd5 = toHexString(md5.digest()).toLowerCase(); + //System.out.println("md5=" + defMd5) ; + return defMd5; + } + + + // The two methods below are copied from vespajlib (com.yahoo.text.Utf8 and com.yahoo.io.HexDump) + // since configgen cannot depend on any other modules (at least not as it is done now) + public static byte[] toBytes(String str) { + Charset charset = Charset.forName("utf-8"); + + ByteBuffer b = charset.encode(str); + byte[] result = new byte[b.remaining()]; + b.get(result); + return result; + } + + private String toHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte aByte : bytes) { + sb.append(String.format("%02x", aByte)); + } + return sb.toString(); + } + + /** + * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * @param str String to strip spaces from + * @return String with spaces stripped + */ + public static String stripSpaces(String str) { + StringBuilder ret = new StringBuilder(""); + boolean inQuotes = false; + boolean inSpaceSequence = false; + for (char c : str.toCharArray()) { + if (Character.isWhitespace(c)) { + if (inQuotes) { + ret.append(c); + continue; + } + if (!inSpaceSequence) { + // start of space sequence + inSpaceSequence=true; + ret.append(" "); + } + } else { + if (inSpaceSequence) { + inSpaceSequence=false; + } + if (c=='\"') { + inQuotes=!inQuotes; + } + ret.append(c); + } + } + return ret.toString(); + } + + public List<String> getNormalizedContent() { + return normalizedContent; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String line : normalizedContent) { + builder.append(line.replace("\"", "\\\"")); + builder.append("\\n\\\n"); + } + return builder.toString(); + } + + public String getDefMd5() { + return defMd5; + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java new file mode 100644 index 00000000000..a42f76340de --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/PropertyException.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +public class PropertyException extends Exception { + PropertyException(String s) { + super(s); + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java new file mode 100644 index 00000000000..145f2da3245 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ReservedWords.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.util.HashMap; +import java.util.regex.Pattern; + +/** + * Reserved words that cannot be used as variable names in a config definition file. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 2009-06-24 + */ + +public class ReservedWords { + + public static final String INTERNAL_PREFIX = "__"; + final static Pattern internalPrefixPattern = Pattern.compile("^" + INTERNAL_PREFIX + ".*"); + final static Pattern capitalizedPattern = Pattern.compile("^[A-Z].*"); + + private static final String[] cKeywords = + {"asm", "auto", "bool", "break", "case", "catch", + "char", "class", "const", "const_cast", "continue", "default", + "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", + "export", "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "item", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", "static_cast", + "struct", "switch", "template", "this", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", + "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", + "bitand"}; + + private static final String[] javaKeywords = + {"abstract", "boolean", "break", "byte", "case", + "catch", "char", "class","continue", "default", "do", "double", + "else", "extends","false", "final", "finally", "float", "for", + "if","implements", "import", "instanceof", "int", "interface", + "item", "long","native", "new", "null", "package", "private", + "protected","public", "return", "short", "static", + "strictfp","super","switch", "synchronized", "this", + "throw","throws","transient", "true", "try", "void", + "volatile","while", "byvalue", "cast", "const", "future", + "generic","goto", "inner", "operator", "outer", "rest", "var"}; + + private static final HashMap<String, String> allKeyWords; + + static { + allKeyWords = new HashMap<String, String>(); + for (String s : cKeywords) { + allKeyWords.put(s, "C"); + } + for (String s : javaKeywords) { + if (allKeyWords.containsKey(s)) { + allKeyWords.put(s, "C and Java"); + } else { + allKeyWords.put(s, "Java"); + } + } + } + + + public static boolean isReservedWord(String word) { + return allKeyWords.containsKey(word); + } + + public static String getLanguageForReservedWord(String word) { + return allKeyWords.get(word); + } + + +} diff --git a/configgen/src/main/manifest.mf b/configgen/src/main/manifest.mf new file mode 100644 index 00000000000..a998997f7c2 --- /dev/null +++ b/configgen/src/main/manifest.mf @@ -0,0 +1,9 @@ +Manifest-Version: 1.0 +Export-Package: com.yahoo.config.codegen +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/annotation-6-SNAPSHOT.jar,dependenc + ies/scala-library-2.9.1.jar +Bundle-ManifestVersion: 2 +Bundle-Name: vespa config generator +Bundle-SymbolicName: configgen +Main-Class: com.yahoo.config.codegen.MakeConfig diff --git a/configgen/src/main/resources/make-config-preproc.pl b/configgen/src/main/resources/make-config-preproc.pl new file mode 100755 index 00000000000..507b426457b --- /dev/null +++ b/configgen/src/main/resources/make-config-preproc.pl @@ -0,0 +1,952 @@ +#!/usr/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# This is the config pre-processor. +# It handles import statements, and does syntax checking etc. +# The idea is that it will be called directly from the script +# that does the code generation. +# +# Errors and warnings are printed in "next-error" compatible ways +# for emacs etc. +# +# Indented like this: +# (cperl-set-style "Whitesmith") +# (setq cperl-continued-brace-offset -4) + +require 5.006_001; +use strict; +use warnings; +use Digest::MD5; + +use Math::BigInt; +use Math::BigFloat; + +die "Usage: $0 <def-file>" unless $#ARGV == 0; + +my $defname = $ARGV[0]; + +my $md5 = Digest::MD5->new; + +my @c_keywords = + ("asm", "auto", "bool", "break", "case", "catch", + "char", "class", "const", "const_cast", "continue", "default", + "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", + "export", "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", "static_cast", + "struct", "switch", "template", "this", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", + "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", + "bitand"); + + +my @java_keywords = + ("abstract", "boolean", "break", "byte", "case", + "catch", "char", "class","continue", "default", "do", "double", + "else", "extends","false", "final", "finally", "float", "for", + "if","implements", "import", "instanceof", "int", "interface", + "long","native", "new", "null", "package", "private", + "protected","public", "return", "short", "static", + "strictfp","super","switch", "synchronized", "this", + "throw","throws","transient", "true", "try", "void", + "volatile","while", "byvalue", "cast", "const", "future", + "generic","goto", "inner", "operator", "outer", "rest", "var"); + +my %reserved_words; + +foreach my $word (@c_keywords) { + $reserved_words{$word} = "C"; +} + +foreach my $word (@java_keywords) { + my $x = $reserved_words{$word}; + if (defined($x)) { + $x = "$x, Java"; + } else { + $x = "Java"; + } + $reserved_words{$word} = $x; +} + +my $MIN_INT = -0x80000000; +my $MAX_INT = 0x7fffffff; +my $MIN_DOUBLE = -1e308; +my $MAX_DOUBLE = 1e308; + + +sub do_file { + my ($file, $prefix, $strip) = @_; + + local *FH; + open FH, "< $file" or die "Cannot open $file: $!\n"; + + local *COPY; + my $dir = $ENV{"VESPA_CONFIG_DEF_DIR"}; + my $copy; + my $file_version; + if (defined($dir)) { + $copy = $file; + $copy =~ s=.*/==; + $copy = "$dir/$copy"; + open COPY, ">$copy.new" or die "Cannot open file $copy.new: $!\n"; + } + + # Read line by line. + # 1. Strip away comments and trailing blanks + # 2. Report any errors + # 3. Handle import statements, disallow multi-level imports + # 4. Print everyting to stdout + + my $linenr = 0; + my $written_lines = 0; + my $quoted_strip = quotemeta($strip); + my $seen_version = 0; + + while (<FH>) { + print COPY $_ if $copy; + ++$linenr; + my $line = $_; + chomp $line; + + # Don't process comments or add them to md5 checksum, but print them + # such that codegen can include comments + if ($line =~ /^\s*#/) { + print "$line\n"; + next; + } + + # Strip away comments that are not at start of line + $line = &strip_trailing_comment($line, $linenr) + if ($line =~ m=[\\\#]=); + + if ($line eq "::error::") { + return -1; + } + + # Skip lines that are only whitespace + next if $line =~ m=^\s*$=; + + # Get rid of trailing whitespace + $line =~ s=\s+$==; + + if (!$seen_version) { + if ($line =~ m!^version=([a-zA-Z0-9][-a-zA-Z0-9_/?]*)!) { + $file_version = $1; + $seen_version = 1; + if ($prefix) { + print "$prefix imported $file"; + print ":$strip" if $strip; + print " "; + } + print "$line\n"; + next; + } else { + print STDERR "$file:$linenr: error: Definition file does not " + . "start with a valid version= identifier!\n"; + return -1; + } + } + + if ($strip) { + next unless $line =~ m=^${quoted_strip}[. \t]=; + } + + if (&check_syntax($line, $linenr, $file) == -1) { + return -1; + } + + # Handle import statements + my ($name, $type, $remains, $junk) = split(/\s+/, $line, 4); + if ($type eq "import") { + if ($strip || $prefix) { + my $col = index($line, $type, length("$name ")); + print STDERR "$file:$linenr:$col: error: Multi-level " + . "imports are disallowed.\n"; + return -1; + } + if ($junk) { + my $col = index($line, $junk, length("$name $type $remains")) + + 1; + print STDERR "$file:$linenr:$col: error: Junk after import " + . "target \"$remains\": \"$junk\"\n"; + return -1; + } + my ($impfile, $var) = split(/:/, $remains, 2); + $var = "" unless $var; # Make it defined. + + # Make sure only arrays can include arrays: + if ($name =~ m=\[\]$= && (!$var || $var !~ m=\[\]$=)) { + print STDERR "$file:$linenr: error: Array cannot import " + . "non-array in: $line\n"; + return -1; + } elsif ($name !~ m=\[\]$= && ($var && $var =~ m=\[\]$=)) { + print STDERR "$file:$linenr: error: Non-array cannot import " + . "array in: $line\n"; + return -1; + } + + local *X; + unless (open(X, "< $impfile")) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Cannot open " + . "\"$impfile\": $!\n"; + return -1; + } + close X; + my $imported_lines = &do_file("$impfile", "$name", "$var"); + if ($imported_lines == -1) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Imported from here " + . "as: $line\n"; + return -1; + } elsif ($imported_lines == 0) { + my $col = index($line, $remains, length("$name $type")) + 1; + print STDERR "$file:$linenr:$col: error: Import target " + . "\"$var\" not found in \"$impfile\"\n"; + return -1; + } + $written_lines += $imported_lines; + } else { + ++$written_lines; + if ($strip) { + $line =~ s=^${quoted_strip}=${prefix}= + } elsif ($prefix) { + $line = $prefix . "." . $line; + } + + if (&check_name_sanity($line, $linenr, $file) == -1 + || &check_enum_sanity($line, $linenr, $file) == -1) { + return -1; + } + + $line = &normalize_line($line, $linenr); + if ($line eq "::error::") { + return -1; + } + print $line . "\n"; + } + # Add this line to the md5 checksum + $md5->add("$line\n") unless $prefix; + } + + print "md5=" . $md5->hexdigest . "\n" unless $prefix; + close FH; + if ($copy) { + close COPY; + # We have made a copy. It needs a new name.. + my $new_name = $copy; + $new_name =~ s=\.def==; + $new_name .= ".${file_version}.def"; + if (-f $new_name) { + system "cmp $copy.new $new_name 2>/dev/null" and die "$file:1: error: Definition file $file differs from ${new_name}!\n"; + unlink("$copy.new"); + } else { + rename("$copy.new", "$new_name") or die "Rename $copy.new -> $new_name failed: $!\n"; + } + } + return $written_lines; +} + +sub normalize_enum { + my($x, $linenr, $colnr) = @_; + my $len = length($x); + my $char = ''; + my $output = '{ '; + my $index; + my %enum = (); + my $current_variable = ''; + for ($index = $colnr + 1; $index < $len; ++$index) { + $char = substr($x, $index, 1); + if ($char eq '}') { + if (length($current_variable) < 2) { + print STDERR "$defname:$linenr:$index: error: ". + " variable must be at least two characters: $x\n" ; + return ('', 0); + } elsif ($enum{$current_variable}) { + print STDERR "$defname:$linenr:$index: error: ". + " enum variable declared twice: $x\n" ; + return ('', 0); + } elsif (!%enum && !$current_variable) { + print STDERR "$defname:$linenr:$index: error: ". + " enum cannot be empty: $x\n" ; + return ('', 0); + } + return ($output.$current_variable." } ", $index); + } elsif ($char eq ',') { + if (length($current_variable) < 2) { + print STDERR "$defname:$linenr:$index: error: ". + " variable must be at least two characters: $x\n" ; + return ('', 0); + } elsif ($enum{$current_variable}) { + print STDERR "$defname:$linenr:$index: error: ". + " enum variable declared twice: $x\n" ; + return ('', 0); + } + $enum{$current_variable} = 1; + $output .= "$current_variable, "; + $current_variable = ''; + } elsif ($char =~ m=[A-Z]=) { + $current_variable .= $char; + } elsif ($char =~ m=[0-9_]= && $current_variable) { + $current_variable .= $char; + } elsif ($char =~ m=\s=) { + if ($current_variable && !($x =~ /^.{$index}\s*[,\}]/)) { + print STDERR "$defname:$linenr:$index: error: ". + "expected ',' or '}': $x\n" ; + return ("", 0); + } else { + # skip whitespace + } + } else { + print STDERR "<$char> <$current_variable>\n"; + + print STDERR "$defname:$linenr:$index: error: ". + "Enum must match [A-Z][A-Z0-9_]+: $x\n"; + } + } + return ($output, $index); +} + +{ package Range; + + $Range::DOUBLE_RANGE = + new Range("a double range=[$MIN_DOUBLE,$MAX_DOUBLE] ",0,14); + $Range::INT_RANGE = new Range("a int range=[$MIN_INT,$MAX_INT] ",0,11); + + + sub in_range { + my($self, $value) = @_; + + if ($value =~ s/KB$//) { + $value *= 1024; + } elsif ($value =~ s/MB$//) { + $value *= (1024 * 1024); + } elsif ($value =~ s/GB$//) { + $value *= (1024*1024*1024); + } elsif ($value =~ s/k$//) { + $value *= 1000; + } elsif ($value =~ s/M$//) { + $value *= 1_000_000; + } elsif ($value =~ s/G$//) { + $value *= 1_000_000_000; + } elsif ($value =~ m=^0[xX]=) { + $value = hex($value); + } + + if ($self->{start_bracket} eq '(' ) { + return 0 if $value <= $self->{min}; + } elsif ($self->{start_bracket} eq '[' ) { + return 0 if $value < $self->{min}; + } else { + print STDERR "Illegal start_bracket '$self->{start_bracket}'\n"; + return undef; + } + if ($self->{end_bracket} eq ')' ) { + return 0 if $value >= $self->{max}; + } elsif ($self->{end_bracket} eq ']' ) { + return 0 if $value > $self->{max}; + } else { + print STDERR "Illegal end_bracket '$self->{start_bracket}'\n"; + return undef; + } + return 1; + } + + + sub new { + my($class, $x, $linenr, $colnr) = @_; + my $len = length($x); + my $self = {}; + bless($self, $class); + $self->{min_value} = ''; + my $index; + for ($index = $colnr + 1; $index < $len; ++$index) { + my $char = substr($x, $index, 1); + if (($char eq '(' || $char eq '[') && !$self->{start_bracket}) { + $self->{start_bracket} = $char; + } elsif (($char eq ')' || $char eq ']') && !$self->{end_bracket}) { + $self->{end_bracket} = $char; + last; + } elsif ($char =~ m=\s=) { + #ignore whitespace + } elsif ($char eq ',' && !defined($self->{max_value})) { + $self->{max_value} = ''; + } elsif ($char =~ m=[\d\.\+eE-]= ) { + (defined($self->{max_value}) + ? $self->{max_value} : $self->{min_value}) .= $char; + } else { + print STDERR "$defname:$linenr:$index: error: ". + " syntax error: $x\n" ; + return undef; + } + } + if ($self->{min_value} eq '' && $self->{max_value} eq '') { + print STDERR "$defname:$linenr:$colnr: error: ". + " range cannot be unbounded in both ends: $x\n" ; + return undef; + } + unless ($self->{start_bracket} && $self->{end_bracket}) { + print STDERR "$defname:$linenr:$colnr: error: ". + " missing bracket: $x\n" ; + return undef; + } + + + my @arr = split(/\s+/, $x, 3); + if ($arr[1] eq 'int') { + $self->{min} = Math::BigInt->new + ($self->{min_value} eq '' ? $MIN_INT : $self->{min_value}); + unless (defined($self->{min}) && $self->{min} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{min_value}: $x\n" ; + return undef; + } + my $min_val = + $self->{min} + ($self->{start_bracket} eq '('? 1 : 0); + + $self->{max} = Math::BigInt->new + ($self->{max_value} eq '' ? $MAX_INT : $self->{max_value}); + unless (defined($self->{max}) && $self->{max} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{max_value}: $x\n" ; + return undef; + } + my $max_val = + $self->{max} - ($self->{end_bracket} eq ')'? 1 : 0); + + if ($min_val < $MIN_INT ) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval less than MIN_INT: $x\n" ; + return undef; + } + if ($max_val > $MAX_INT) { + print STDERR "$self->{max} - 1 > $MAX_INT\n"; + print STDERR "$defname:$linenr:$colnr: error: ". + " end of interval greater than MAX_INT: $x\n" ; + return undef; + } + if ($max_val < $min_val) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + $self->{string} = + "$self->{start_bracket}$self->{min},$self->{max}$self->{end_bracket}"; + $self->{string} =~ s/\+//g; + $self->{index} = $index; + return $self; + } elsif ($arr[1] eq 'double') { + $self->{min} = Math::BigFloat->new + ($self->{min_value} eq '' ? $MIN_DOUBLE : $self->{min_value}); + unless (defined($self->{min}) && $self->{min} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{min_value}: $x\n" ; + return undef; + } + $self->{max} = Math::BigFloat->new + ($self->{max_value} eq '' ? $MAX_DOUBLE : $self->{max_value}); + unless (defined($self->{max}) && $self->{max} ne 'NaN') { + print STDERR "$defname:$linenr:$colnr: error: ". + " parse error $self->{max_value}: $x\n" ; + return undef; + } + if ($self->{min} < $MIN_DOUBLE) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval less than MIN_DOUBLE: $x\n" ; + return undef; + } + if ($self->{max} > $MAX_DOUBLE) { + print STDERR "$defname:$linenr:$colnr: error: ". + " start of interval greater than MAX_DOUBLE: $x\n" ; + return undef; + } + if ($self->{max} < $self->{min}) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + if (($self->{start_bracket} eq '(' || $self->{end_bracket} eq ')') + && ($self->{min_value} + $self->{min_value} + >= $self->{min_value} + $self->{max_value}) + && ($self->{max_value} + $self->{max_value} + <= $self->{min_value} + $self->{max_value})) { + print STDERR "$defname:$linenr:$colnr: error: ". + " illegal range: $x\n" ; + return undef; + } + $self->{string} = $self->{start_bracket}.$self->{min}->fnorm. + ','.$self->{max}->fnorm.$self->{end_bracket}; + $self->{string} =~ s/\+//g; + $self->{index} = $index; + return $self; + } else { + print STDERR "$defname:$linenr:$colnr: error: ". + " range-option works only for type 'int' and 'double': $x\n" ; + return undef; + } + print STDERR "$defname:$linenr:$colnr: error: ". + " script error: $x\n" ; + return undef; + + } + +} + + + + +sub strip_trailing_comment { + my ($x, $linenr) = @_; + + my $index = 0; + my $len = length($x); + my $in_quotes = 0; + + # ### Support both " and ' quotes maybe? + + for ($index = 0; $index < $len; ++$index) { + if (substr($x, $index, 1) eq "\\") { + ++$index; + next; + } + if (substr($x, $index, 1) eq "\"") { + $in_quotes ^= 1; + } + if ($in_quotes == 0 && substr($x, $index, 1) eq "#") { + if (!(substr($x, $index - 1, 1) =~ m=\s=)) { + my $col = $index + 1; + print STDERR "$defname:$linenr:$col: warning: No whitespace " + . "before comment in line: $x\n"; + } + print substr($x, $index). "\n"; + $x = substr($x, 0, $index); + last; + } + } + if ($index > $len) { + print STDERR "$defname:$linenr:$len: error: syntax error, line " + . "ends with \\: \"$x\"\n"; + return "::error::"; + } + + return $x; +} + +sub normalize_line { + my ($x, $linenr) = @_; + + my $index = 0; + my $len = length($x); + my $in_quotes = 0; + my $char = ''; + my $output = ''; + my %hash = (); + + my @arr = split(/\s+/, $x, 3); + $hash{type} = $arr[1]; + + for ($index = 0; $index < length($x); ++$index) { + $char = substr($x, $index, 1); + if ($char eq "\\") { + $output .= substr($x, $index, 2); + ++$index; + next; + } + if ($char eq "\"") { + $in_quotes ^= 1; + $output .= $char; + next; + } + my $ends_with_whitespace = ($output =~ m= $=); + + if ($in_quotes == 0) { + if ($char =~ m=\s=) { + #delete multiple spaces + if (!$ends_with_whitespace) { # && ($output =~ !m=\=$=)) { + $output .= ' '; + } + } elsif ($char eq '{') { + my($enum, $i) = &normalize_enum($x, $linenr, $index); + return "::error::" unless $i; + $index = $i; + $output .= ($ends_with_whitespace) ? $enum : " $enum "; + } elsif ($char eq ',') { + chop $output if ($ends_with_whitespace); + $output .= ','; + } elsif ($char eq '=') { + chop $output if ($ends_with_whitespace); + $output .= '='; + if ($output =~ /range=$/) { + $hash{range} = + new Range($x, $linenr, $index); + return "::error::" unless $hash{range}; + $index = $hash{range}->{index}; + $output .= $hash{range}->{string}." "; + } + if ($output =~ /default=$/ + && ($hash{type} eq 'int' || $hash{type} eq 'double')) { + $x =~ /^.{$index}=\s*(\S+)/; + $hash{default} = $1; + if ($hash{type} eq 'int' && + !$Range::INT_RANGE->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + if ($hash{type} eq 'double' && + !$Range::DOUBLE_RANGE->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + } + if (defined($hash{default}) && $hash{range}) { + unless ($hash{range}->in_range($hash{default})) { + print STDERR "$defname:$linenr:$index: error: ". + "Default not in range: $x\n"; + return "::error::"; + } + } + } else { + $output .= $char; + } + } else { + $output .= $char; + } + } + if ($index > $len) { + print STDERR "$defname:$linenr:$len: error: syntax error, line " + . "ends with \\: \"$x\"\n"; + return "::error::"; + } + chop $output if $output =~ m/ $/; + return $output; +} + +my %used_enum; +sub check_enum_sanity { + my ($line, $linenr, $file) = @_; + + my ($name, $type, $rest) = split(/\s+/, $line, 3); + return 0 unless ($type eq "enum"); + + $name =~ /(.*)\./; + my $prefix = $1; + $prefix = "" unless defined $prefix; # Make top level prefix + $used_enum{"$prefix"} = $used_enum{"$prefix"} || {}; + $rest = "" unless defined $rest; + $rest =~ /\{\s*(.*?)\}/; + my @values = split(/[,\s]+/, $1); + foreach my $value (@values) { + if ($used_enum{"$prefix"}->{$value}) { + print STDERR + "$file:$linenr: error: Name \"$value\" is already defined\n"; + my $prevdef = $used_enum{"$prefix"}->{$value}; + print STDERR "$prevdef: error: At this point\n"; + return -1; + } else { + $used_enum{"$prefix"}->{$value} = "$file:$linenr"; + } + } + return 0; +} + + +my %used_name; +my %used_component; +my %banned_prefixes; +my $cns_prev_name; +sub check_name_sanity { + my ($line, $linenr, $file) = @_; + my ($name, $junk) = split(/\s+/, $line, 2); + + my $plain_name = $name; + $plain_name =~ s=\[\]$==; + + # See if the name is already used. + if ($used_name{"$plain_name"}) { + print STDERR + "$file:$linenr: error: Name \"$name\" is already defined\n"; + my $prevdef = $used_name{$name}; + print STDERR "$prevdef: error: At this point\n"; + return -1; + } else { + $used_name{$name} = "$file:$linenr"; + } + + # Test for bans + my $banned = "${name}."; + do { + my $err = $banned_prefixes{$banned}; + if (defined($err)) { + print STDERR "$file:$linenr: error: The prefix \"$banned\" is illegal here\n"; + print STDERR "$err\n"; + return -1; + } + } while (($banned =~ s=[.][^.]+[.]$=.=)); + + # Add any new bans generated by this line + $banned_prefixes{"${name}."} = "$file:$linenr: error: \"${name}\" cannot " + . "be both a struct and a non-struct!"; + if ($cns_prev_name) { + my $prev = $cns_prev_name; + my $oldprev = $prev; + while (($prev =~ s=[.][^.]+[.]?$=.=)) { + if (substr($name, 0, length($prev)) eq $prev) { + $banned_prefixes{"$oldprev"} = "$file:" . ($linenr - 1) + . ": error: Last possible line is after this"; + last; + } + $oldprev = $prev; + } + } + $cns_prev_name = $name; + + # See if any of the components previously have a different "arrayness" + my $part_name = $name; + while (($part_name =~ s=[.][^.]+$==)) { + my $clashing_name = $part_name; + if ($part_name =~ m=\[\]$=) { + $clashing_name =~ s=\[\]$==; + } else { + $clashing_name .= "[]"; + } + my $clashline = $used_component{"$clashing_name"}; + if (defined $clashline) { + print STDERR "$file:$linenr: error: \"$clashing_name\" cannot be both array and non-array\n"; + print STDERR "$clashline: error: Previously defined here\n"; + return -1; + } elsif (!$used_component{"$part_name"}) { + $used_component{"$part_name"} = "$file:$linenr"; + } + } + return 0; +} + +# These are all the allowed types/commands +my %types = ( "int" => \&check_int, + "double" => \&check_double, + "string" => \&check_string, + "reference" => \&check_reference, + "enum" => \&check_enum, + "bool" => \&check_bool, + "properties" => \&check_properties, + "import" => \&check_import ); + +sub check_syntax { + my ($line, $linenr, $file) = @_; + + my $col = 0; + my $llen = length($line); + + # Step 1. Sanity check the name. + my $atstart = 1; + my $array_ok = 1; + + for ($col = 0; $col < $llen; ++$col) { + my $c = substr($line, $col, 1); + if ($atstart) { + if ($c !~ m=[a-zA-Z]=) { + print STDERR "$file:$linenr:$col: error: Non-alphabetic start " + . "of variable name in $line\n"; + return -1; + } + $atstart = 0; + } else { + if ($c =~ m=[a-zA-Z0-9_]=) { + 0; # Do nothing + } elsif ($c eq ".") { + $atstart = 1; + $array_ok = 1; + } elsif ($c eq "[") { + if (!$array_ok) { + ++$col; + print STDERR "$file:$linenr:$col: error: Arrays cannot be " + . "multidimensional in $line\n"; + return -1; + } + ++$col; + $array_ok = 0; + $c = substr($line, $col, 1); + if ($c ne "]") { + ++$col; + print STDERR "$file:$linenr:$col: error: Expected ] to " + . "terminate array definition in $line\n"; + return -1; + } + } elsif ($c =~ m=\s=) { + last; + } else { + ++$col; + print STDERR "$file:$linenr:$col: error: Syntax error, " + . "unexpected character in $line\n"; + return -1; + } + } + } + + my $name = substr($line, 0, $col); + $name =~ s=.*[.]==; + $name =~ s=[[]]$==; + + my $clash = $reserved_words{$name}; + if ($clash) { + $col -= (3 + length($name)); + $col = index($line, $name, $col) + 1; + print STDERR "$file:$linenr:$col: error: $name is a reserved word in: " + . "${clash}\n"; + return -1; + } + + while (substr($line, $col, 1) =~ m=\s=) { + ++$col; + } + + # At this point the name is sane. Next, check the type. + my ($type) = split(/\s/, substr($line, $col)); + + unless (defined $types{$type}) { + ++$col; + print STDERR "$file:$linenr:$col: error: Unknown type/command " + . "\"$type\"\n"; + return -1; + } + $col += length($type); + while (substr($line, $col, 1) =~ m=\s=) { + ++$col; + } + return $types{$type}($col, $line, $linenr, $file); +} + +sub reg_words_check { + my ($col, $line, $linenr, $file, $reg) = @_; + my $remainder = substr($line, $col); + my @options = split(/\s+/, $remainder); + + foreach my $option (@options) { + # Keep track of where we are for error reporting + $col = index($line, $option, $col) + 1; + unless ($option =~ m!${reg}!) { + print STDERR "$file:$linenr:$col: error: Bad option \"$option\" no match for m!${reg}!\n"; + return -1; + } + } + return 0; +} + +sub check_int { + my ($col, $line, $linenr, $file) = @_; + my $num = "(-?\\d+(KB|MB|GB|k|M|G)?|0x[0-9a-fA-F]+)"; # All legal numbers + my $optnum = "(${num})?"; # All legal optional numbers + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=${num}" + . "|range=[[(]${optnum},${optnum}"."[])]" + . "|restart" + . ")\$"); +} + +sub check_double { + my ($col, $line, $linenr, $file) = @_; + my $num = "-?(\\d+(\\.\\d*)?|\\.\\d+)([eE][+-]?\\d+)?"; # All legal doubles + my $optnum = "(${num})?"; # Optional doubles + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=${num}" + . "|range=[[(]${optnum},${optnum}"."[])]" + . "|restart" + . ")\$"); +} + +sub check_string { + my ($col, $line, $linenr, $file) = @_; + my $opts = substr($line, $col); + + # not entirely correct either for something like \\" + my $def = "default=((\"(\\\"|[^\"])*\")|null)"; + + my $res = "restart"; + my $reg = "^(${def}\\s+${res}|(${def})?|${res}|${res}\\s+${def})\$"; + + unless ($opts =~ m!${reg}!) { + print STDERR "$file:$linenr:$col: error: Bad options \"$opts\", no match for m!${reg}!\n"; + return -1; + } + return 0; +} + +sub check_reference { + my ($col, $line, $linenr, $file) = @_; + my $opts = substr($line, $col); + my $def = "default=((\"(\\\"|[^\"])*\")|null)"; + + unless ($opts eq "" || $opts =~m!${def}!) { + print STDERR "$file:$linenr:$col: error: reference can only " + . "take the 'default' option\n"; + return -1; + } + return 0; +} + + +sub check_enum { + my ($col, $line, $linenr, $file) = @_; + my $ret = ®_words_check($col, $line, $linenr, $file, + "^(" + . "[{},]" + . "|[A-Z][A-Z0-9_]+,?" + . "|default=[A-Z][A-Z0-9_]+" + . "|restart" + . ")\$"); + return -1 if $ret; + $col = index($line, '}', $col) + 1; #move $col to end of enum --> } + while (substr($line, $col, 1) =~ m=[\s\{]=) { + ++$col; + } + return 0 if $col >= length($line); + + + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=[A-Z][A-Z0-9_]+" + . "|restart" + . ")\$"); +} + +sub check_bool { + my ($col, $line, $linenr, $file) = @_; + return ®_words_check($col, $line, $linenr, $file, + "^(" + . "default=(true|false)" + . "|restart" + . ")\$"); +} + +sub check_properties { + my ($col, $line, $linenr, $file) = @_; + return ®_words_check($col, $line, $linenr, $file, "^restart\$"); +} + +sub check_import { + my ($col, $line, $linenr, $file) = @_; + my $word = "[a-zA-Z][_a-zA-Z0-9]*"; + my $fnam = "${word}(\\.${word})*"; + my $var = "${word}((\\[\\])?\.${word})*(\\[\\])?"; + return ®_words_check($col, $line, $linenr, $file, + "^${fnam}\\.def:(${var})?\$"); + return 0; +} + + +my $lines = &do_file($defname, "", ""); + +if ($lines == -1) { + die "There were irrecoverable errors in \"$defname\"!\n"; +} +if ($lines == 0) { + die "$defname:1: error: Resulting definition is empty!\n"; +} + +exit 0; diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala new file mode 100644 index 00000000000..e4b9879f7f7 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala @@ -0,0 +1,350 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + +import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} +import JavaClassBuilder.{Indentation, createUniqueSymbol} +import ConfigGenerator.{indentCode, nodeClass, userDataType, boxedDataType} +import com.yahoo.config.codegen.LeafCNode._ + +/** + * @author gjoranv + */ + +object BuilderGenerator { + + def getBuilder(node: InnerCNode): String = { + getDeclaration(node) + "\n" + + indentCode(Indentation, + getUninitializedScalars(node) + "\n\n" + + node.getChildren.map(getBuilderFieldDefinition).mkString("\n") + "\n\n" + + getBuilderConstructors(node, nodeClass(node)) + "\n\n" + + getOverrideMethod(node) + "\n\n" + + getBuilderSetters(node) + "\n" + + getSpecialRootBuilderCode(node) + ) + + "}" + } + + private def getDeclaration(node: InnerCNode) = { + def getInterfaces = + if (node.getParent == null) "implements ConfigInstance.Builder" + else "implements ConfigBuilder" + + "public static class Builder " + getInterfaces + " {" + } + + private def getSpecialRootBuilderCode(node: InnerCNode) = { + if (node.getParent == null) "\n" + getDispatchCode(node) + "\n" + else "" + } + + private def getDispatchCode(node: InnerCNode) = { + // Use full path to @Override, as users are free to define an inner node called 'override'. (summarymap.def does) + // The generated inner 'Override' class would otherwise be mistaken for the annotation. + """ + |@java.lang.Override + |public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { + | if (producer instanceof Producer) { + | ((Producer)producer).getConfig(this); + | return true; + | } + | return false; + |} + | + |@java.lang.Override + |public final String getDefMd5() { return CONFIG_DEF_MD5; } + |@java.lang.Override + |public final String getDefName() { return CONFIG_DEF_NAME; } + |@java.lang.Override + |public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + """.stripMargin.trim + } + + private def getUninitializedScalars(node: InnerCNode): String = { + val scalarsWithoutDefault = { + node.getChildren.collect { + case leaf: LeafCNode if (!leaf.isArray && !leaf.isMap && leaf.getDefaultValue == null) => + "\"" + leaf.getName + "\"" + } + } + + val uninitializedList = + if (scalarsWithoutDefault.size > 0) + "Arrays.asList(\n" + indentCode(Indentation, scalarsWithoutDefault.mkString("",",\n","\n)")) + else + "" + + "private Set<String> " + InternalPrefix + "uninitialized = new HashSet<String>(" + uninitializedList + ");" + } + + private def getBuilderFieldDefinition(node: CNode): String = { + + (node match { + case array if node.isArray => + "public List<%s> %s = new ArrayList<>()".format(builderType(array), array.getName) + case map if node.isMap => + "public Map<String, %s> %s = new LinkedHashMap<>()".format(builderType(map), map.getName) + case struct: InnerCNode => + "private %s %s = new %s()".format(builderType(struct), struct.getName, builderType(struct)) + case scalar : LeafCNode => + "private " + boxedBuilderType(scalar) + " " + scalar.getName + " = null" + }) + ";" + } + + private def getBuilderSetters(node: CNode): String = { + val children: Array[CNode] = node.getChildren + + def structSetter(node: InnerCNode) = { + <code> + |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ + | {node.getName} = {InternalPrefix}builder; + | return this; + |}} + </code>.text.stripMargin.trim + } + + def innerArraySetters(node: InnerCNode) = { + <code> + |/** + | * Add the given builder to this builder's list of {nodeClass(node)} builders + | * @param {InternalPrefix}builder a builder + | * @return this builder + | */ + |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ + | {node.getName}.add({InternalPrefix}builder); + | return this; + |}} + | + |/** + | * Set the given list as this builder's list of {nodeClass(node)} builders + | * @param __builders a list of builders + | * @return this builder + | */ + |public Builder {node.getName}(List<{builderType(node)}> __builders) {{ + | {node.getName} = __builders; + | return this; + |}} + </code>.text.stripMargin.trim + } + + def leafArraySetters(node: LeafCNode) = { + val setters = + <code> + |public Builder {node.getName}({builderType(node)} {InternalPrefix}value) {{ + | {node.getName}.add({InternalPrefix}value); + | return this; + |}} + | + |public Builder {node.getName}(Collection<{builderType(node)}> {InternalPrefix}values) {{ + | {node.getName}.addAll({InternalPrefix}values); + | return this; + |}} + </code>.text.stripMargin.trim + + val privateSetter = + if (builderType(node) == "String" || builderType(node) == "FileReference") + "" + else + "\n\n" + + <code> + | + | + |private Builder {node.getName}(String {InternalPrefix}value) {{ + | return {node.getName}({builderType(node)}.valueOf({InternalPrefix}value)); + |}} + </code>.text.stripMargin.trim + + setters + privateSetter + } + + def innerMapSetters(node: CNode) = { + <code> + |public Builder {node.getName}(String {InternalPrefix}key, {builderType(node)} {InternalPrefix}value) {{ + | {node.getName}.put({InternalPrefix}key, {InternalPrefix}value); + | return this; + |}} + | + |public Builder {node.getName}(Map<String, {builderType(node)}> {InternalPrefix}values) {{ + | {node.getName}.putAll({InternalPrefix}values); + | return this; + |}} + </code>.text.stripMargin.trim + } + + def leafMapSetters(node: LeafCNode) = { + val privateSetter = + if (builderType(node) == "String" || builderType(node) == "FileReference") + "" + else + "\n\n" + + <code> + | + | + |private Builder {node.getName}(String {InternalPrefix}key, String {InternalPrefix}value) {{ + | return {node.getName}({InternalPrefix}key, {builderType(node)}.valueOf({InternalPrefix}value)); + |}} + </code>.text.stripMargin.trim + + innerMapSetters(node) + privateSetter + } + + def scalarSetters(node: LeafCNode): String = { + val name = node.getName + + val signalInitialized = + if (node.getDefaultValue == null) InternalPrefix + "uninitialized.remove(\"" + name + "\");\n" + else "" + + val stringSetter = + builderType(node) match { + case "String" => "" + case "FileReference" => "" + case _ => + """| + |private Builder %s(String %svalue) { + | return %s(%s.valueOf(%svalue)); + |}""".stripMargin.format(name, InternalPrefix, + name, boxedDataType(node), InternalPrefix) + } + + def getNullGuard = { + if (builderType(node) != boxedBuilderType(node)) + "" + else + "\n" + "if (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");" + .format(InternalPrefix) + } + + // TODO: check if 2.9.2 allows string to start with a newline + """|public Builder %s(%s %svalue) {%s + | %s = %svalue; + | %s + """.stripMargin.format(name, builderType(node), InternalPrefix, getNullGuard, + name, InternalPrefix, + signalInitialized).trim + + "\n return this;" + "\n}\n" + + stringSetter + } + + (children collect { + case innerArray: InnerCNode if innerArray.isArray => innerArraySetters(innerArray) + case innerMap: InnerCNode if innerMap.isMap => innerMapSetters(innerMap) + case leafArray: LeafCNode if leafArray.isArray => leafArraySetters(leafArray) + case leafMap: LeafCNode if leafMap.isMap => leafMapSetters(leafMap) + case struct: InnerCNode => structSetter(struct) + case scalar: LeafCNode => scalarSetters(scalar) + } ).mkString("\n\n") + } + + private def getBuilderConstructors(node: CNode, className: String): String = { + def setBuilderValueFromConfig(child: CNode) = { + val name = child.getName + val isArray = child.isArray + val isMap = child.isMap + + child match { + case fileArray: FileLeaf if isArray => name + "(" + userDataType(fileArray) + ".toValues(config." + name + "()));" + case fileMap: FileLeaf if isMap => name + "(" + userDataType(fileMap) + ".toValueMap(config." + name + "()));" + case file: FileLeaf => name + "(config." + name + "().value());" + case pathArray: PathLeaf if isArray => name + "(" + nodeClass(pathArray) + ".toFileReferences(config." + name + "));" + case pathMap: PathLeaf if isMap => name + "(" + nodeClass(pathMap) + ".toFileReferenceMap(config." + name + "));" + case path: PathLeaf => name + "(config." + name + ".getFileReference());" + case leaf: LeafCNode => name + "(config." + name + "());" + case innerArray: InnerCNode if isArray => setInnerArrayBuildersFromConfig(innerArray) + case innerMap: InnerCNode if isMap => setInnerMapBuildersFromConfig(innerMap) + case struct => name + "(new " + builderType(struct) + "(config." + name + "()));" + } + } + + def setInnerArrayBuildersFromConfig(innerArr: InnerCNode) = { + val elemName = createUniqueSymbol(node, innerArr.getName) + <code> + |for ({userDataType(innerArr)} {elemName} : config.{innerArr.getName}()) {{ + | {innerArr.getName}(new {builderType(innerArr)}({elemName})); + |}} + </code>.text.stripMargin.trim + } + + def setInnerMapBuildersFromConfig(innerMap: InnerCNode) = { + val entryName = InternalPrefix + "entry" + <code> + |for (Map.Entry<String, {userDataType(innerMap)}> {entryName} : config.{innerMap.getName}().entrySet()) {{ + | {innerMap.getName}({entryName}.getKey(), new {userDataType(innerMap)}.Builder({entryName}.getValue())); + |}} + </code>.text.stripMargin.trim + } + + <code> + |public Builder() {{ }} + | + |public Builder({className} config) {{ + |{indentCode(Indentation, node.getChildren.map(setBuilderValueFromConfig).mkString("\n"))} + |}} + </code>.text.stripMargin.trim + } + + def arrayOverride(name: String, superior: String): String = { + Indentation + name + ".addAll(" + superior + "." + name + ");" + } + + private def getOverrideMethod(node:CNode): String = { + val method = "override" + val superior = InternalPrefix + "superior" + + def callSetter(name: String): String = { + name + "(" + superior + "." + name + ");" + } + def overrideBuilderValue(child: CNode) = { + val name = child.getName + child match { + case leafArray: CNode if (child.isArray) => + conditionStatement(child) + "\n" + arrayOverride(name, superior) + case struct: InnerCNode if !(child.isArray || child.isMap) => + name + "(" + name + "." + method + "(" + superior + "." + name + "));" + case map: CNode if child.isMap => + callSetter(name) + case _ => + conditionStatement(child) + "\n" + + Indentation + callSetter(name) + } + } + + def conditionStatement(child: CNode) = { + val name = child.getName + val isArray = child.isArray + val isMap = child.isMap + child match { + case _ if isArray => "if (!" + superior + "." + name + ".isEmpty())" + case _ if isMap => "" + case scalar: LeafCNode => "if (" + superior + "." + name + " != null)" + case struct => "" + } + } + + <code> + |private Builder {method}(Builder {superior}) {{ + |{indentCode(Indentation, node.getChildren.map(overrideBuilderValue).mkString("\n"))} + | return this; + |}} + </code>.text.stripMargin.trim + } + + private def builderType(node: CNode): String = { + node match { + case inner: InnerCNode => boxedDataType(node) + ".Builder" + case file: FileLeaf => "String" + case path: PathLeaf => "FileReference" + case leafArray: LeafCNode if (node.isArray || node.isMap) => boxedDataType(node) + case _ => userDataType(node) + } + } + + private def boxedBuilderType(node: LeafCNode): String = { + node match { + case file: FileLeaf => "String" + case path: PathLeaf => "FileReference" + case _ => boxedDataType(node) + } + } + +}
\ No newline at end of file diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala new file mode 100644 index 00000000000..716f2a60c33 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala @@ -0,0 +1,459 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + + +import com.yahoo.config.codegen.LeafCNode._ +import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} +import JavaClassBuilder.Indentation +import BuilderGenerator.getBuilder +import util.parsing.combinator.JavaTokenParsers + +/** + * @author gjoranv + * @author tonytv + */ +// TODO: don't take indent as method param - the caller should indent +object ConfigGenerator { + + def generateContent(indent: String, node: InnerCNode, isOuter: Boolean = true): String = { + val children: Array[CNode] = node.getChildren + + def generateCodeForChildren: String = { + (children collect { + case enum: EnumLeaf => getEnumCode(enum, "") + "\n" + case inner: InnerCNode => getInnerDefinition(inner, indent) + "\n" + } ).mkString("\n") + } + + def getInnerDefinition(inner: InnerCNode, indent: String) = { + <code> + |{getClassDoc(inner, indent)} + |{getClassDeclaration(inner)} + |{generateContent(indent, inner, false)} + </code>.text.stripMargin.trim + "\n}" + } + + def getClassDeclaration(node: CNode): String = { + "public final static class " + nodeClass(node)+ " extends InnerNode { " + "\n" + } + + def getFieldDefinition(node: CNode): String = { + node.getCommentBlock("//") + "private final " + + (node match { + case _: LeafCNode if node.isArray => + "LeafNodeVector<%s, %s> %s;".format(boxedDataType(node), nodeClass(node), node.getName) + case _: InnerCNode if node.isArray => + "InnerNodeVector<%s> %s;".format(nodeClass(node), node.getName) + case _ if node.isMap => + "Map<String, %s> %s;".format(nodeClass(node), node.getName) + case _ => + "%s %s;".format(nodeClass(node), node.getName) + }) + } + + def getStaticMethods = { + if (node.isArray) getStaticMethodsForInnerArray(node) + "\n\n" + else if (node.isMap) getStaticMethodsForInnerMap(node) + "\n\n" + else "" + } + + def getContainsFieldsFlaggedWithRestart(node: CNode): String = { + if (isOuter) { + """ + |private static boolean containsFieldsFlaggedWithRestart() { + | return %b; + |} + """.stripMargin.trim.format(node.needRestart) + "\n\n" + } else "" + } + + indentCode(indent, + getBuilder(node) + "\n\n" + + children.map(getFieldDefinition).mkString("\n") + "\n\n" + + getConstructors(node) + "\n\n" + + getAccessors(children) + "\n\n" + + getGetChangesRequiringRestart(node) + "\n\n" + + getContainsFieldsFlaggedWithRestart(node) + + getStaticMethods + + generateCodeForChildren + ) + } + + private def getGetChangesRequiringRestart(node: InnerCNode): String = { + def quotedComment(node: CNode): String = { + node.getComment.replace("\n", "\\n").replace("\"", "\\\"") + } + + def getComparison(node: CNode): String = node match { + case inner: InnerCNode if inner.isArray => + <code> + | changes.compareArray(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", + | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); + </code>.text.stripMargin.trim + case inner: InnerCNode if inner.isMap => + <code> + | changes.compareMap(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", + | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); + </code>.text.stripMargin.trim + case inner: InnerCNode => + <code> + | changes.mergeChanges("{inner.getName}", this.{inner.getName}.getChangesRequiringRestart(newConfig.{inner.getName})); + </code>.text.stripMargin.trim + case node: CNode if node.isArray => + <code> + | changes.compareArray(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", + | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); + </code>.text.stripMargin.trim + case node: CNode if node.isMap => + <code> + | changes.compareMap(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", + | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); + </code>.text.stripMargin.trim + case node: CNode => + <code> + | changes.compare(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}"); + </code>.text.stripMargin.trim + } + + val comparisons = + for { + c <- node.getChildren if c.needRestart + } yield "\n " + getComparison(c) + + <code> + |private ChangesRequiringRestart getChangesRequiringRestart({nodeClass(node)} newConfig) {{ + | ChangesRequiringRestart changes = new ChangesRequiringRestart("{node.getName}");{comparisons.mkString("")} + | return changes; + |}} + </code>.text.stripMargin.trim + } + + + private def scalarDefault(scalar: LeafCNode): String = { + scalar match { + case _ if scalar.getDefaultValue == null => "" + case enumWithNullDefault: EnumLeaf if enumWithNullDefault.getDefaultValue.getValue == null => "" + case enum: EnumLeaf => nodeClass(enum) + "." + enum.getDefaultValue.getStringRepresentation + case long: LongLeaf => long.getDefaultValue.getStringRepresentation + "L" + case double: DoubleLeaf => double.getDefaultValue.getStringRepresentation + "D" + case _ => scalar.getDefaultValue.getStringRepresentation + } + } + + private def getConstructors(inner: InnerCNode) = { + + def assignFromBuilder(child: CNode) = { + val name = child.getName + val className = nodeClass(child) + val dataType = boxedDataType(child) + val isArray = child.isArray + val isMap = child.isMap + + def assignIfInitialized(leaf: LeafCNode) = { + <code> + |{name} = (builder.{name} == null) ? + | new {className}({scalarDefault(leaf)}) : new {className}(builder.{name}); + </code>.text.stripMargin.trim + } + + child match { + case fileArray: FileLeaf if isArray => + name + " = LeafNodeVector.createFileNodeVector(builder."+ name +");" + case pathArray: PathLeaf if isArray => + name + " = LeafNodeVector.createPathNodeVector(builder."+ name +");" + case leafArray: LeafCNode if isArray => + name + " = new LeafNodeVector<>(builder."+ name +", new " + className + "());" + case fileMap: LeafCNode if isMap && child.isInstanceOf[FileLeaf] => + name + " = LeafNodeMaps.asFileNodeMap(builder."+ name +");" + case pathMap: LeafCNode if isMap && child.isInstanceOf[PathLeaf] => + name + " = LeafNodeMaps.asPathNodeMap(builder."+ name +");" + case leafMap: LeafCNode if isMap => + name + " = LeafNodeMaps.asNodeMap(builder."+ name +", new " + className + "());" + case innerArray: InnerCNode if isArray => + name + " = " + className + ".createVector(builder." + name + ");" + case innerMap: InnerCNode if isMap => + name + " = " + className + ".createMap(builder." + name + ");" + case struct: InnerCNode => + name + " = new " + className + "(builder." + name + ", throwIfUninitialized);" + case leaf: LeafCNode => + assignIfInitialized(leaf) + } + } + + // TODO: The default ctor can be removed if the config library uses builders to set values from payload, but ... + // a default ctor is also needed for all innerArrays, because of InnerNodeVector.createNew() + def defaultConstructor = { + // TODO @link gives javadoc warnings, although the syntax seems to be valid + //def link = "{@link " + {nodeClass(inner)} + "#" + {nodeClass(inner)} + "(Builder)}" + def link = {nodeClass(inner)} + "(Builder)" + + <code> + |/** + | * @deprecated Not for public use. + | * Does not check for uninitialized fields. + | * Replaced by {link} + | */ + |@Deprecated + |public {nodeClass(inner)}() {{ + | this(new Builder(), false); + |}} + </code>.text.stripMargin.trim + } + + // TODO: merge these two constructors into one when the config library uses builders to set values from payload. + <code> + |{defaultConstructor} + | + |public {nodeClass(inner)}(Builder builder) {{ + | this(builder, true); + |}} + | + |private {nodeClass(inner)}(Builder builder, boolean throwIfUninitialized) {{ + | if (throwIfUninitialized && ! builder.{InternalPrefix}uninitialized.isEmpty()) + | throw new IllegalArgumentException("The following builder parameters for " + + | "{inner.getFullName} must be initialized: " + builder.{InternalPrefix}uninitialized); + | + |{indentCode(Indentation, inner.getChildren.map(assignFromBuilder).mkString("\n"))} + |}} + </code>.text.stripMargin.trim + } + + private def getAccessors(children: Array[CNode]): String = { + + def getAccessorCode(indent: String, node: CNode): String = { + indentCode(indent, + if (node.isArray) + accessorsForArray(node) + else if (node.isMap) + accessorsForMap(node) + else + accessorForStructOrScalar(node)) + } + + def valueAccessor(node: CNode) = node match { + case leaf: LeafCNode => ".value()" + case inner => "" + } + + def listAccessor(node: CNode) = node match { + case leaf: LeafCNode => "%s.asList()".format(leaf.getName) + case inner => inner.getName + } + + def mapAccessor(node: CNode) = node match { + case leaf: LeafCNode => "LeafNodeMaps.asValueMap(%s)".format(leaf.getName) + case inner => "Collections.unmodifiableMap(%s)".format(inner.getName) + } + + def accessorsForArray(node: CNode): String = { + val name = node.getName + val fullName = node.getFullName + <code> + |/** + | * @return {fullName} + | */ + |public List<{boxedDataType(node)}> {name}() {{ + | return {listAccessor(node)}; + |}} + | + |/** + | * @param i the index of the value to return + | * @return {fullName} + | */ + |public {userDataType(node)} {name}(int i) {{ + | return {name}.get(i){valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + def accessorsForMap(node: CNode): String = { + val name = node.getName + val fullName = node.getFullName + <code> + |/** + | * @return {fullName} + | */ + |public Map<String, {boxedDataType(node)}> {name}() {{ + | return {mapAccessor(node)}; + |}} + | + |/** + | * @param key the key of the value to return + | * @return {fullName} + | */ + |public {userDataType(node)} {name}(String key) {{ + | return {name}.get(key){valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + def accessorForStructOrScalar(node: CNode): String = { + <code> + |/** + | * @return {node.getFullName} + | */ + |public {userDataType(node)} {node.getName}() {{ + | return {node.getName}{valueAccessor(node)}; + |}} + </code>.text.stripMargin.trim + } + + val accessors = + for { + c <- children + accessor = getAccessorCode("", c) + if (accessor.length > 0) + } yield (accessor + "\n") + accessors.mkString("\n").trim + } + + private def getStaticMethodsForInnerArray(inner: InnerCNode) = { + """ + |private static InnerNodeVector<%s> createVector(List<Builder> builders) { + | List<%s> elems = new ArrayList<>(); + | for (Builder b : builders) { + | elems.add(new %s(b)); + | } + | return new InnerNodeVector<%s>(elems, new %s()); + |} + """.stripMargin.format(List.fill(5)(nodeClass(inner)): _*).trim + } + + private def getStaticMethodsForInnerMap(inner: InnerCNode) = { + """ + |private static Map<String, %s> createMap(Map<String, Builder> builders) { + | Map<String, %s> ret = new LinkedHashMap<>(); + | for(String key : builders.keySet()) { + | ret.put(key, new %s(builders.get(key))); + | } + | return Collections.unmodifiableMap(ret); + |} + """.stripMargin.format(List.fill(3)(nodeClass(inner)): _*).trim + } + + private def getEnumCode(enum: EnumLeaf, indent: String): String = { + + def getEnumValues(enum: EnumLeaf): String = { + val enumValues = + for (value <- enum.getLegalValues) yield + """ public final static Enum %s = Enum.%s;""".format(value, value) + enumValues.mkString("\n") + } + + // TODO: try to rewrite to xml + val code = + """ + |%s + |public final static class %s extends EnumNode<%s> { + + | public %s(){ + | this.value = null; + | } + + | public %s(Enum enumValue) { + | super(enumValue != null); + | this.value = enumValue; + | } + + | public enum Enum {%s} + |%s + + | @Override + | protected boolean doSetValue(@NonNull String name) { + | try { + | value = Enum.valueOf(name); + | return true; + | } catch (IllegalArgumentException e) { + | } + | return false; + | } + |} + |""" + .stripMargin.format(getClassDoc(enum, indent), + nodeClass(enum), + nodeClass(enum)+".Enum", + nodeClass(enum), + nodeClass(enum), + enum.getLegalValues.mkString(", "), + getEnumValues(enum)) + + indentCode(indent, code).trim + } + + def getClassDoc(node: CNode, indent: String): String = { + val header = "/**\n" + " * This class represents " + node.getFullName + val nodeComment = node.getCommentBlock(" *") match { + case "" => "" + case s => "\n *\n" + s.stripLineEnd // TODO: strip trailing \n in CNode.getCommentBlock + } + header + nodeComment + "\n */" + } + + def indentCode(indent: String, code: String): String = { + val indentedLines = + for (s <- code.split("\n", -1)) yield + if (s.length() > 0) (indent + s) else s + indentedLines.mkString("\n") + } + + /** + * @return the name of the class that is generated by this node. + */ + def nodeClass(node: CNode): String = { + node match { + case emptyName: CNode if node.getName.length == 0 => + throw new CodegenRuntimeException("Node with empty name, under parent " + emptyName.getParent.getName) + case root: InnerCNode if root.getParent == null => createClassName(root.getName) + case b: BooleanLeaf => "BooleanNode" + case d: DoubleLeaf => "DoubleNode" + case f: FileLeaf => "FileNode" + case p: PathLeaf => "PathNode" + case i: IntegerLeaf => "IntegerNode" + case l: LongLeaf => "LongNode" + case r: ReferenceLeaf => "ReferenceNode" + case s: StringLeaf => "StringNode" + case _ => node.getName.capitalize + } + } + + def userDataType(node: CNode): String = { + node match { + case inner: InnerCNode => nodeClass(node) + case enum: EnumLeaf => nodeClass(enum) + ".Enum" + case b: BooleanLeaf => "boolean" + case d: DoubleLeaf => "double" + case f: FileLeaf => "FileReference" + case p: PathLeaf => "Path" + case i: IntegerLeaf => "int" + case l: LongLeaf => "long" + case s: StringLeaf => "String" + } + } + + /** + * @return the boxed java data type, e.g. Integer for int + */ + def boxedDataType(node: CNode): String = { + val rawType = userDataType(node) + + rawType match { + case "int" => "Integer" + case _ if rawType == rawType.toLowerCase => rawType.capitalize + case _ => rawType + } + } + + /** + * Create class name from def name + * @param defName The file name without the '.def' suffix + */ + def createClassName(defName: String): String = { + val className = defName.split("-").map (_.capitalize).mkString + "Config" + val parser = new JavaTokenParsers {} + parser.parseAll(parser.ident, className) match { + case parser.NoSuccess(msg, _) => + throw new CodegenRuntimeException("Illegal config definition file name '" + defName + "': " + msg) + case success => success.get + } + } +} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala new file mode 100644 index 00000000000..c346338e543 --- /dev/null +++ b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + +import java.io.{FileNotFoundException, FileOutputStream, PrintStream, File} +import ConfigGenerator.{indentCode, createClassName} +import util.Random +import scala.collection.JavaConversions._ + +/** + * Builds one Java class based on the given CNode tree. + * + * @author gjoranv + * @author tonytv + */ +class JavaClassBuilder( + root: InnerCNode, + nd: NormalizedDefinition, + destDir: File) + extends ClassBuilder +{ + import JavaClassBuilder._ + + val javaPackage = PackagePrefix + root.getNamespace + val className = createClassName(root.getName) + + override def createConfigClasses() { + try { + val outFile = new File(getDestPath(destDir, root.getNamespace), className + ".java") + val out = new PrintStream(new FileOutputStream(outFile)) + out.print(getConfigClass(className)) + System.err.println(outFile.getPath + " successfully written.") + } + catch { + case e: FileNotFoundException => { + throw new CodegenRuntimeException(e) + } + } + } + + def getConfigClass(className:String): String = { + val ret = new StringBuilder + + ret.append(getHeader).append("\n\n") + ret.append(getRootClassDeclaration(root, className)).append("\n\n") + ret.append(indentCode(Indentation, getFrameworkCode(className))).append("\n\n") + ret.append(ConfigGenerator.generateContent(Indentation, root)).append("\n") + ret.append("}\n") + + ret.toString() + } + + private def getHeader: String = { + <code> + |/** + | * This file is generated from a config definition file. + | * ------------ D O N O T E D I T ! ------------ + | */ + | + |package {javaPackage}; + | + |import java.util.*; + |import java.nio.file.Path; + |import edu.umd.cs.findbugs.annotations.NonNull; + |{getImportFrameworkClasses(root.getNamespace)} + </code>.text.stripMargin.trim + } + + private def getImportFrameworkClasses(namespace: String): String = { + if (namespace != CNode.DEFAULT_NAMESPACE) + "import " + PackagePrefix + CNode.DEFAULT_NAMESPACE + ".*;\n" + else + "" + } + + // TODO: remove the extra comment line " *" if root.getCommentBlock is empty + private def getRootClassDeclaration(root:InnerCNode, className: String): String = { + <code> + |/** + | * This class represents the root node of {root.getFullName} + | * + |{root.getCommentBlock(" *")} */ + |public final class {className} extends ConfigInstance {{ + | + | public final static String CONFIG_DEF_MD5 = "{root.getMd5}"; + | public final static String CONFIG_DEF_NAME = "{root.getName}"; + | public final static String CONFIG_DEF_NAMESPACE = "{root.getNamespace}"; + | public final static String CONFIG_DEF_VERSION = "{root.getVersion}"; + | public final static String[] CONFIG_DEF_SCHEMA = {{ + |{indentCode(Indentation * 2, getDefSchema)} + | }}; + | + | public static String getDefMd5() {{ return CONFIG_DEF_MD5; }} + | public static String getDefName() {{ return CONFIG_DEF_NAME; }} + | public static String getDefNamespace() {{ return CONFIG_DEF_NAMESPACE; }} + | public static String getDefVersion() {{ return CONFIG_DEF_VERSION; }} + </code>.text.stripMargin.trim + } + + private def getDefSchema: String = { + nd.getNormalizedContent.map { line => + "\"" + + line.replace("\"", "\\\"") + + "\"" + }.mkString(",\n") + } + + private def getFrameworkCode(className: String): String = { + getProducerBase + } + + private def getProducerBase = { + """ + |public interface Producer extends ConfigInstance.Producer { + | void getConfig(Builder builder); + |} + """.stripMargin.trim + } +} + + +object JavaClassBuilder { + private val PackagePrefix: String = System.getProperty("config.packagePrefix", "com.yahoo.") + + val Indentation = " " + + /** + * Returns a name that can be safely used as a local variable in the generated config class + * for the given node. The name will be based on the given basis string, but the basis itself is + * not a possible return value. + * + * @param node The node to find a unused symbol name for. + * @param basis The basis for the generated symbol name. + * @return A name that is not used in the given config node. + */ + def createUniqueSymbol(node: CNode, basis: String) = { + + def getCandidate(cnt: Int) = { + if (cnt < basis.length()) + basis.substring(0, cnt) + else + ReservedWords.INTERNAL_PREFIX + basis + Random.nextInt().abs + } + + def getUsedSymbols: Set[String] = { + (node.getChildren map (child => child.getName)).toSet + } + + // TODO: refactoring potential + val usedSymbols = getUsedSymbols + var count = 1 + var candidate = getCandidate(count) + while (usedSymbols contains(candidate)) { + count += 1 + candidate = getCandidate(count) + } + candidate + } + + /** + * @param rootDir The root directory for the destination path. + * @param namespace The namespace from the def file + * @return the destination path for the generated config file, including the given rootDir. + */ + private def getDestPath(rootDir: File, namespace: String): File = { + var dir: File = rootDir + val subDirs: Array[String] = (PackagePrefix + namespace).split("""\.""") + for (subDir <- subDirs) { + dir = new File(dir, subDir) + if (!dir.isDirectory && !dir.mkdir) throw new CodegenRuntimeException("Could not create " + dir.getPath) + } + dir + } +} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java new file mode 100644 index 00000000000..515f7838436 --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java @@ -0,0 +1,221 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Tests parsing of a single line of a .def file + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + * @author gjoranv + */ +public class DefLineParsingTest { + + @Test(expected = IllegalArgumentException.class) + public void require_that_null_default_is_not_allowed() { + new DefLine("s string default=null"); + } + + @Test + public void testParseIntArray() { + DefLine l = new DefLine("foo[] int"); + + assertEquals("foo[]", l.getName()); + assertNull(l.getDefault()); + assertEquals("int", l.getType().getName()); + } + + @Test + public void testParseIntMap() { + DefLine l = new DefLine("foo{} int"); + + assertEquals("foo{}", l.getName()); + assertNull(l.getDefault()); + assertEquals("int", l.getType().getName()); + } + + @Test + public void testParseInnerMap() { + DefLine l = new DefLine("foo{}.i int"); + + assertEquals("foo{}.i", l.getName()); + assertNull(l.getDefault()); + assertEquals("int", l.getType().getName()); + } + + @Test + public void testParseEnum() { + DefLine l = new DefLine("idtype enum { us_east, US_WEST, EMEA } default=EMEA"); + + assertEquals("idtype", l.getName()); + assertEquals("EMEA", l.getDefault().getValue()); + assertEquals("EMEA", l.getDefault().getStringRepresentation()); + assertEquals("enum", l.getType().getName()); + assertEquals("us_east", l.getType().getEnumArray()[0]); + assertEquals("US_WEST", l.getType().getEnumArray()[1]); + assertEquals("EMEA", l.getType().getEnumArray()[2]); + } + + @Test + public void testParseDefaultReference() { + DefLine l = new DefLine("foo.bar reference default=\"value\""); + + assertEquals("foo.bar", l.getName()); + assertEquals("value", l.getDefault().getValue()); + assertEquals("\"value\"", l.getDefault().getStringRepresentation()); + assertEquals("reference", l.getType().getName()); + } + + @Test + public void testParseNoDefaultReference() { + DefLine l = new DefLine("foo.bar reference"); + + assertEquals("foo.bar", l.getName()); + assertNull(l.getDefault()); + assertEquals("reference", l.getType().getName()); + } + + /** + * 'file' parameters with default value is currently (2010-01-05) not allowed, but that might change in + * the future, so the test is included to verify that value and name can be retrieved. + */ + @Test + public void testParseDefaultFile() { + DefLine l = new DefLine("fileWithDef file default=\"value\""); + + assertEquals("fileWithDef", l.getName()); + assertEquals("value", l.getDefault().getValue()); + assertEquals("\"value\"", l.getDefault().getStringRepresentation()); + assertEquals("file", l.getType().getName()); + } + + @Test + public void testParseNoDefaultFile() { + DefLine l = new DefLine("fileVal file"); + + assertEquals("fileVal", l.getName()); + assertNull(l.getDefault()); + assertEquals("file", l.getType().getName()); + } + + @Test + public void testParseDefaultInt() { + DefLine l = new DefLine("foo int default=1000"); + + assertEquals("foo", l.getName()); + assertEquals("1000", l.getDefault().getValue()); + assertEquals("1000", l.getDefault().getStringRepresentation()); + assertEquals("int", l.getType().getName()); + } + + @Test + public void testParseDefaultLong() { + DefLine l = new DefLine("foo long default=9223372036854775807"); + + assertEquals("foo", l.getName()); + assertEquals("9223372036854775807", l.getDefault().getValue()); + assertEquals("9223372036854775807", l.getDefault().getStringRepresentation()); + assertEquals("long", l.getType().getName()); + } + + @Test + public void testParseDefaultDouble() { + DefLine l = new DefLine("foo double default=5.37"); + + assertEquals("foo", l.getName()); + assertEquals("5.37", l.getDefault().getValue()); + assertEquals("5.37", l.getDefault().getStringRepresentation()); + assertEquals("double", l.getType().getName()); + } + + @Test + public void testParseDefaultFalseBoolean() { + DefLine l = new DefLine("foo bool default=false"); + + assertEquals("foo", l.getName()); + assertEquals("false", l.getDefault().getValue()); + assertEquals("false", l.getDefault().getStringRepresentation()); + assertEquals("bool", l.getType().getName()); + } + + @Test + public void testParseDefaultTrueBoolean() { + DefLine l = new DefLine("foo bool default=true"); + + assertEquals("foo", l.getName()); + assertEquals("true", l.getDefault().getValue()); + assertEquals("true", l.getDefault().getStringRepresentation()); + assertEquals("bool", l.getType().getName()); + } + + @Test + public void testParseNoDefaultString() { + DefLine l = new DefLine("foo.bar string"); + + assertEquals("foo.bar", l.getName()); + assertNull(l.getDefault()); + assertEquals("string", l.getType().getName()); + } + + @Test + public void testParseDefaultString() { + DefLine l = new DefLine("foo.bar string default=\"value\""); + + assertEquals("foo.bar", l.getName()); + assertEquals("value", l.getDefault().getValue()); + assertEquals("\"value\"", l.getDefault().getStringRepresentation()); + assertEquals("string", l.getType().getName()); + } + + @Test + public void testParseDefaultEmptyString() { + DefLine l = new DefLine("foo.bar string default=\"\""); + + assertEquals("foo.bar", l.getName()); + assertEquals("", l.getDefault().getValue()); + assertEquals("\"\"", l.getDefault().getStringRepresentation()); + assertEquals("string", l.getType().getName()); + } + + @Test + public void testParseDefaultStringUnquoted() { + DefLine l = new DefLine("foo.bar string default=value"); + + assertEquals("foo.bar", l.getName()); + assertEquals("value", l.getDefault().getValue()); + assertEquals("\"value\"", l.getDefault().getStringRepresentation()); + assertEquals("string", l.getType().getName()); + } + + @Test + public void testParseStringnullDefaultString() { + DefLine l = new DefLine("foo.bar string default=\"null\""); + + assertEquals("foo.bar", l.getName()); + assertEquals("null", l.getDefault().getValue()); + assertEquals("\"null\"", l.getDefault().getStringRepresentation()); + assertEquals("string", l.getType().getName()); + } + + @Test + public void testRanges() { + DefLine i = new DefLine("i int range=[0, 100]"); + DefLine l = new DefLine("l long range=[-1e10, 0]"); + DefLine d = new DefLine("d double range=[0, 1.0]"); + assertThat(i.getRange(), is("[0, 100]")); + assertThat(l.getRange(), is("[-1e10, 0]")); + assertThat(d.getRange(), is("[0, 1.0]")); + } + + @Test + public void testRestartSpecification() { + DefLine r0 = new DefLine("i int"); + DefLine r1 = new DefLine("i int restart"); + assertFalse(r0.getRestart()); + assertTrue(r1.getRestart()); + } + +} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java new file mode 100644 index 00000000000..567d7419778 --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java @@ -0,0 +1,514 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.Ignore; + +import static org.hamcrest.CoreMatchers.is; + +import java.io.*; + +/** + * Unit tests for DefParser. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class DefParserTest { + + private static final String TEST_DIR = "target/test-classes/"; + private static final String DEF_NAME = TEST_DIR + "allfeatures.def"; + + @Test + public void testTraverseTree() throws IOException { + File defFile = new File(DEF_NAME); + CNode root = new DefParser("test", new FileReader(defFile)).getTree(); + assertNotNull(root); + CNode[] children = root.getChildren(); + assertThat(children.length, is(31)); + + int numGrandChildren = 0; + int numGreatGrandChildren = 0; + for (CNode child : children) { + CNode[] childsChildren = child.getChildren(); + numGrandChildren += childsChildren.length; + for (CNode grandChild : childsChildren) { + numGreatGrandChildren += grandChild.getChildren().length; + } + } + assertThat(numGrandChildren, is(14)); + assertThat(numGreatGrandChildren, is(6)); + + // Verify that each array creates a sub-tree, and that defaults for leafs are handled correctly. + CNode myArray = root.getChild("myArray"); + assertThat(myArray.getChildren().length, is(5)); + // int within array + LeafCNode myArrayInt = (LeafCNode) myArray.getChild("intVal"); + assertThat(myArrayInt.getDefaultValue().getValue(), is("14")); + // enum within array + LeafCNode myArrayEnum = (LeafCNode) myArray.getChild("enumVal"); + assertThat(myArrayEnum.getDefaultValue().getValue(), is("TYPE")); + + // Verify array within array and a default value for a leaf in the inner array. + CNode anotherArray = myArray.getChild("anotherArray"); + assertThat(anotherArray.getChildren().length, is(1)); + LeafCNode foo = (LeafCNode) anotherArray.getChild("foo"); + assertThat(foo.getDefaultValue().getValue(), is("-4")); + } + + @Test + public void testFileWithNamespaceInFilename() throws IOException { + File defFile = new File(TEST_DIR + "bar.foo.def"); + CNode root = new DefParser("test", new FileReader(defFile)).getTree(); + assertThat(root.defMd5, is("31a0f9bda0e5ff929762a29569575a7e")); + } + + @Test + public void testMd5Sum() throws IOException { + File defFile = new File(DEF_NAME); + CNode root = new DefParser("test", new FileReader(defFile)).getTree(); + assertThat(root.defMd5, is("eb2d24dbbcf054b21be729e2cfaafd93")); + } + + @Test + public void testMd5Sum2() { + String def = "version=1\na string\n"; + CNode root = new DefParser("testMd5Sum2", new StringReader(def)).getTree(); + assertThat(root.defMd5, is("a5e5fdbb2b27e56ba7d5e60e335c598b")); + } + + @Test + public void testExplicitNamespace() { + DefParser parser = createParser("version=1\nnamespace=myproject.config\na string\n"); + CNode root = parser.getTree(); + assertThat(root.getNamespace(), is("myproject.config")); + + // with spaces + parser = createParser("version=1\nnamespace = myproject.config\na string\n"); + root = parser.getTree(); + assertThat(root.getNamespace(), is("myproject.config")); + + // invalid + parser = createParser("version=1\nnamespace \na string\n"); + try { + parser.getTree(); + fail(); + } catch (Exception e) { + //e.printStackTrace(); + assertExceptionAndMessage(e, CodegenRuntimeException.class, + "Error parsing or reading config definition.Error when parsing line 2: namespace \n" + + "namespace"); + } + + // invalid + parser = createParser("version=1\nnamespace=a..b\na string\n"); + try { + parser.getTree(); + fail(); + } catch (Exception e) { + //e.printStackTrace(); + assertExceptionAndMessage(e, CodegenRuntimeException.class, + "Error parsing or reading config definition.Error when parsing line 2: namespace=a..b\n" + + "namespace=a..b"); + } + } + + @Test + public void verifyThatExplicitNamespaceAltersDefMd5() { + DefParser parser = createParser("version=1\na string\n"); + CNode root = parser.getTree(); + + parser = createParser("version=1\nnamespace=myproject.config\na string\n"); + CNode namespaceRoot = parser.getTree(); + + assertThat(root.defMd5, not(namespaceRoot.defMd5)); + } + + + @Test(expected = CodegenRuntimeException.class) + public void verify_fail_on_illegal_char_in_namespace() { + createParser("version=1\nnamespace=Foo\na string\n").getTree(); + } + + @Test(expected = CodegenRuntimeException.class) + public void verify_fail_on_com_yahoo_in_explicit_namespace() { + createParser("version=1\n" + + "namespace=com.yahoo.myproject.config\n" + + "a string\n").getTree(); + } + + @Test + public void testInvalidType() { + Class<?> exceptionClass = DefParser.DefParserException.class; + try { + createParser("version=1\n" + + "# comment\n" + + "a sting").getTree(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage((Exception) e.getCause(), exceptionClass, + "Error when parsing line 3: a sting", false); + } + } + + @Test + public void testValidVersions() { + try { + testExpectedVersion("version=8", "8"); + testExpectedVersion("version=8-1", "8-1"); + testExpectedVersion("version =8", "8"); + testExpectedVersion("version = 8", "8"); + testExpectedVersion("version = 8 ", "8"); + testExpectedVersion("version =\t8", "8"); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + private void testExpectedVersion(String versionLine, String expectedVersion) { + InnerCNode root = createParser(versionLine).getTree(); + assertThat(root.defVersion, is(expectedVersion)); + } + + @Test + public void testMissingVersion() { + try { + createParser("a string\n").parse(); + } catch (Exception e) { + fail("Should not get an exception here"); + } + } + + private DefParser createParser(String def) { + return new DefParser("test", new StringReader(def)); + } + + @Test + public void testInvalidVersion() { + Class<?> exceptionClass = DefParser.DefParserException.class; + testInvalidVersion("version=a\n", exceptionClass, + "Error when parsing line 1: version=a\nversion=a"); + testInvalidVersion("version = a\n", exceptionClass, + "Error when parsing line 1: version = a\n a"); + } + + private void testInvalidVersion(String versionLine, Class<?> exceptionClass, String exceptionMessage) { + try { + createParser(versionLine).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, exceptionMessage); + } + } + + @Test + public void verify_fail_on_default_for_file() { + Class<?> exceptionClass = DefParser.DefParserException.class; + DefParser parser = createParser("version=1\nf file default=\"file1.txt\"\n"); + try { + parser.getTree(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage((Exception) e.getCause(), exceptionClass, + "Error when parsing line 2: f file default=\"file1.txt\"\n" + + "Invalid default value", false); + } + } + + // Helper method for checking correct exception class and message + void assertExceptionAndMessage(Exception e, Class<?> exceptionClass, String message) { + assertExceptionAndMessage(e, exceptionClass, message, true); + } + + // Helper method for checking correct exception class and message + void assertExceptionAndMessage(Exception e, Class<?> exceptionClass, String message, boolean exact) { + if (exact) { + assertEquals(message, e.getMessage()); + } else { + assertTrue(e.getMessage().startsWith(message)); + } + assertEquals(exceptionClass.getName(), e.getClass().getName()); + } + + @Test(expected = CodegenRuntimeException.class) + @Ignore("Not implemented yet") + public void testInvalidEnum() throws DefParser.DefParserException { + DefParser parser = createParser("version=1\nanEnum enum {A, B, A}\n"); + //parser.validateDef(def); + } + + @Test + public void testEnum() { + StringBuilder sb = createDefTemplate(); + sb.append("enum1 enum {A,B} default=A\n"); + sb.append("enum2 enum {A, B} default=A\n"); + sb.append("enum3 enum { A, B} default=A\n"); + sb.append("enum4 enum { A, B } default=A\n"); + sb.append("enum5 enum { A , B } default=A\n"); + sb.append("enum6 enum {A , B } default=A\n"); + sb.append("enumVal enum { FOO, BAR, FOOBAR }\n"); + + DefParser parser = createParser(sb.toString()); + try { + parser.getTree(); + } catch (Exception e) { + assertNotNull(null); + } + CNode root = parser.getTree(); + LeafCNode node = (LeafCNode) root.getChild("enum1"); + assertNotNull(node); + assertThat(node.getDefaultValue().getStringRepresentation(), is("A")); + } + + @Test(expected = DefParser.DefParserException.class) + public void testInvalidCommaInEnum() throws DefParser.DefParserException, IOException { + String invalidEnum = "anEnum enum {A, B, } default=A\n"; + String validEnum = "anotherEnum enum {A, B} default=A\n"; + StringBuilder sb = createDefTemplate(); + sb.append(invalidEnum); + sb.append(validEnum); + DefParser parser = createParser(sb.toString()); + parser.parse(); + } + + @Ignore //TODO: finish this! The numeric leaf nodes must contain their range. + @Test + public void testRanges() { + StringBuilder sb = new StringBuilder("version=1\n"); + sb.append("i int range=[0,10]"); + sb.append("l long range=[-1e20,0]"); + sb.append("d double range=[0,1]"); + + DefParser parser = createParser(sb.toString()); + CNode root = parser.getTree(); + LeafCNode.IntegerLeaf intNode = (LeafCNode.IntegerLeaf) root.getChild("i"); + } + + @Test + public void testInvalidLine() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "a inta\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "Could not create inta a"); + } + } + + @Test + public void testDuplicateDefinition() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "b int\n"; + sb.append(invalidLine); + // Add a duplicate line, which should be illegal + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 4: " + invalidLine + "b is already defined"); + } + } + + @Test + public void testIllegalCharacterInName() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "a-b int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "a-b contains unexpected character"); + } + } + + @Test + public void require_that_parameter_name_starting_with_digit_is_illegal() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "1a int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "1a must start with a non-digit character"); + } + } + + @Test + public void require_that_parameter_name_starting_with_uppercase_is_illegal() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "SomeInt int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "'SomeInt' cannot start with an uppercase letter"); + } + } + + @Test + public void require_that_parameter_name_starting_with_the_internal_prefix_is_illegal() { + String internalPrefix = ReservedWords.INTERNAL_PREFIX; + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = internalPrefix + "i int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + + "'" + internalPrefix + "i' cannot start with '" + internalPrefix + "'"); + } + } + + @Test + public void testIllegalArray() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "intArr[ int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "intArr[ Expected ] to terminate array definition"); + } + } + + @Test + public void testIllegalDefault() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "a int deflt 10\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + " deflt 10"); + } + } + + @Test + public void testReservedWordInC() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "auto int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "auto is a reserved word in C"); + } + } + + @Test + public void testReservedWordInJava() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "abstract int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "abstract is a reserved word in Java"); + } + } + + @Test + public void testReservedWordInCAndJava() { + Class<?> exceptionClass = DefParser.DefParserException.class; + StringBuilder sb = createDefTemplate(); + String invalidLine = "continue int\n"; + sb.append(invalidLine); + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + invalidLine + "continue is a reserved word in C and Java"); + } + } + + @Test + public void testNumberInNamespace() throws IOException, DefParser.DefParserException { + StringBuilder sb = createDefTemplate(); + String line = "namespace=a.b.c2\nfoo int\n"; + sb.append(line); + createParser(sb.toString()).parse(); + + sb = createDefTemplate(); + line = "namespace=2.a.b\n"; + sb.append(line); + Class<?> exceptionClass = DefParser.DefParserException.class; + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + line + "namespace=2.a.b"); + } + + sb = createDefTemplate(); + line = "namespace=a.b.2c\n"; + sb.append(line); + exceptionClass = DefParser.DefParserException.class; + try { + createParser(sb.toString()).parse(); + fail("Didn't find expected exception of type " + exceptionClass); + } catch (Exception e) { + assertExceptionAndMessage(e, exceptionClass, + "Error when parsing line 3: " + line + "namespace=a.b.2c"); + } + } + + @Test + public void testUnderscoreInNamespace() throws IOException, DefParser.DefParserException { + StringBuilder sb = createDefTemplate(); + String line = "namespace=a_b.c\nfoo int\n"; + sb.append(line); + createParser(sb.toString()).parse(); + + sb = createDefTemplate(); + line = "namespace=a_b.c_d\nfoo int\n"; + sb.append(line); + createParser(sb.toString()).parse(); + } + + private StringBuilder createDefTemplate() { + StringBuilder sb = new StringBuilder(); + sb.append("version=8\n"); + // Add a comment line to check that we get correct line number with comments + sb.append("# comment\n"); + + return sb; + } +} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java b/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java new file mode 100644 index 00000000000..a9f29fd23dd --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MakeConfigTest { + + File dest; + + @Before + public void setUp() { + dest = new File("/tmp/"+System.currentTimeMillis()+File.separator); + dest.mkdir(); + } + + @After + public void tearDown() { + recursiveDeleteDir(dest); + } + + private boolean recursiveDeleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + + for (int i = 0; i < children.length; i++) { + boolean success = recursiveDeleteDir(new File(dir, children[i])); + + if (!success) return false; + } + } + + // The directory is now empty so delete it + return dir.delete(); + } + + @Test + public void testProps() throws PropertyException { + long ts = System.currentTimeMillis(); + System.setProperty("config.dumpTree", "true"); + System.setProperty("config.useFramework", "true"); + System.setProperty("config.requireNamespace", "true"); + System.setProperty("config.dest", dest.getAbsolutePath()); + System.setProperty("config.spec", "src/test/resources/allfeatures.def"); + MakeConfigProperties p = new MakeConfigProperties(); + assertEquals(p.destDir.getAbsolutePath(), dest.getAbsolutePath()); + assertTrue(p.dumpTree); + assertTrue(p.generateFrameworkCode); + assertEquals(p.specFiles.length, 1); + assertEquals(p.specFiles[0].getAbsolutePath(), new File("src/test/resources/allfeatures.def").getAbsolutePath()); + + System.setProperty("config.dumpTree", "false"); + System.setProperty("config.useFramework", "false"); + System.setProperty("config.requireNamespace", "false"); + System.setProperty("config.dest", dest.getAbsolutePath()); + System.setProperty("config.spec", "src/test/resources/allfeatures.def,src/test/resources/bar.foo.def"); + p = new MakeConfigProperties(); + assertEquals(p.destDir.getAbsolutePath(), dest.getAbsolutePath()); + assertFalse(p.dumpTree); + assertFalse(p.generateFrameworkCode); + assertEquals(p.specFiles.length, 2); + } + + @Test + public void testMake() throws IOException, InterruptedException { + System.setProperty("config.dumpTree", "true"); + System.setProperty("config.useFramework", "true"); + System.setProperty("config.requireNamespace", "true"); + System.setProperty("config.dest", dest.getAbsolutePath()); + System.setProperty("config.spec", "src/test/resources/allfeatures.def"); + MakeConfig.main(new String[]{}); + } + +} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java new file mode 100644 index 00000000000..f0713a090ec --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import static org.junit.Assert.*; + +import java.io.*; +import java.util.List; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; + + +/** + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class NormalizedDefinitionTest { + + @Test + public void testNormalizingFromReader() { + String def = + "version=1\n" + + "aString string \n" + + "anInt int #comment \n" + + "aStringCommentCharacterAfter string default=\"ab\" #foo\n" + + "aStringWithCommentCharacter string default=\"a#b\"\n" + + "aStringWithEscapedQuote string default=\"a\"b\"\n"; + + StringReader reader = new StringReader(def); + + NormalizedDefinition nd = new NormalizedDefinition(); + List<String> out = null; + try { + nd.normalize(new BufferedReader(reader)); + out = nd.getNormalizedContent(); + } catch (IOException e) { + e.printStackTrace(); + } + + assertNotNull(out); + assertThat(out.size(), is(6)); + assertThat(out.get(0), is ("version=1\n")); + assertThat(out.get(1), is ("aString string\n")); + assertThat(out.get(2), is ("anInt int\n")); + assertThat(out.get(3), is ("aStringCommentCharacterAfter string default=\"ab\"\n")); + assertThat(out.get(4), is ("aStringWithCommentCharacter string default=\"a#b\"\n")); + assertThat(out.get(5), is ("aStringWithEscapedQuote string default=\"a\"b\"\n")); + + reader.close(); + } + + @Test + public void testNormalizingFromFile() throws IOException { + FileReader fileReader = null; + try { + fileReader = new FileReader("src/test/resources/allfeatures.def"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + NormalizedDefinition nd = new NormalizedDefinition(); + List<String> out = null; + try { + nd.normalize(new BufferedReader(fileReader)); + out = nd.getNormalizedContent(); + } catch (IOException e) { + e.printStackTrace(); + } + + assertNotNull(out); + assertThat(out.size(), is(70)); + assertThat(out.get(19), is ("version=11\n")); + + assertNotNull(fileReader); + fileReader.close(); + } +} diff --git a/configgen/src/test/resources/allfeatures.def b/configgen/src/test/resources/allfeatures.def new file mode 100644 index 00000000000..9a4d732b880 --- /dev/null +++ b/configgen/src/test/resources/allfeatures.def @@ -0,0 +1,77 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This def file should test most aspects of def files that makes a difference +# for the generated config classes. The goal is to trigger all blocks of +# code in the code generators. This includes: +# +# - Use all legal special characters in the def file name, to ensure that those +# that needs to be replaced in type names are actually replaced. +# - Use the same enum type twice to verify that we dont declare or define it +# twice. +# - Use the same struct type twice for the same reason. +# - Include arrays of primitives and structs. +# - Include enum primitives and array of enums. Arrays of enums must be handled +# specially by the C++ code. +# - Include enums both with and without default values. +# - Include primitive string, numbers & doubles both with and without default +# values. +# - Have an array within a struct, to verify that we correctly recurse. +# - Reuse type name further within to ensure that this works. + +version=11 +namespace=configgen +# Some random bool without a default value. These comments exist to check + # that comment parsing works.e +boolVal bool + ## A bool with a default value set. +bool_with_def bool default=false +intVal int +intWithDef int default=-545 +longVal long +longWithDef long default=1234567890123 +doubleVal double +double_with_def double default=-6.43 +# Another comment +stringVal string +stringwithdef string default="foobar#notacomment" +enumVal enum { FOO, BAR, FOOBAR } +enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2 +refVal reference +refwithdef reference default=":parent:" +fileVal file +pathVal path + +boolarr[] bool +intarr[] int +longarr[] long +doublearr[] double +stringarr[] string +enumarr[] enum { ARRAY, VALUES } +refarr[] reference +filearr[] file +pathArr[] path + +intMap{} int +pathMap{} file + +# A basic struct +basic_struct.foo string default="foo" +basic_struct.bar int default=0 + +# A struct of struct +struct_of_struct.inner0.name string default="inner0" +struct_of_struct.inner0.index int default=0 +struct_of_struct.inner1.name string default="inner1" +struct_of_struct.inner1.index int default=1 + +myArray[].intVal int default=14 +myArray[].stringVal[] string +myArray[].enumVal enum { INNER, ENUM, TYPE } default=TYPE +myArray[].refVal reference # Value in array without default +myArray[].anotherArray[].foo int default=-4 + +myMap{}.intVal int default=15 +myMap{}.stringVal[] string +myMap{}.enumVal enum { INNER, ENUM, TYPE } default=ENUM +myMap{}.refVal reference # Value in map without default +myMap{}.anotherArray[].foo int default=-5 diff --git a/configgen/src/test/resources/bar.foo.def b/configgen/src/test/resources/bar.foo.def new file mode 100644 index 00000000000..6373a55a864 --- /dev/null +++ b/configgen/src/test/resources/bar.foo.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=baz +xyzzy int default=10 diff --git a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala b/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala new file mode 100644 index 00000000000..b9704ecd284 --- /dev/null +++ b/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen + +import org.junit.Assert.assertThat +import org.junit.Assert.assertTrue +import org.hamcrest.CoreMatchers.is +import java.io.StringReader +import ConfigGenerator.createClassName +import JavaClassBuilder.createUniqueSymbol +import org.junit.{Ignore, Test} + +/** + * @author gjoranv + */ +class JavaClassBuilderTest { + + @Ignore + @Test + def visual_inspection_of_generated_class() { + val testDefinition = + """version=1 + |namespace=test + |p path + |pathArr[] path + |f file + |fileArr[] file + |i int default=0 + |# A long value + |l long default=0 + |s string default="" + |b bool + |# An enum value + |e enum {A, B, C} + |intArr[] int + |boolArr[] bool + |enumArr[] enum {FOO, BAR} + |intMap{} int + |# A struct + |# with multi-line + |# comment and "quotes". + |myStruct.i int + |myStruct.s string + |# An inner array + |myArr[].i int + |myArr[].newStruct.s string + |myArr[].newStruct.b bool + |myArr[].intArr[] int + |# An inner map + |myMap{}.i int + |myMap{}.newStruct.s string + |myMap{}.newStruct.b bool + |myMap{}.intArr[] int + |intMap{} int + |""".stripMargin + + val parser = new DefParser("test", new StringReader(testDefinition)) + val root = parser.getTree + val builder = new JavaClassBuilder(root, parser.getNormalizedDefinition, null) + val configClass = builder.getConfigClass("TestConfig") + print(configClass) + } + + @Test + def testCreateUniqueSymbol() { + val testDefinition = + """version=1 + |namespace=test + |m int + |n int + """.stripMargin + val root = new DefParser("test", new StringReader(testDefinition)).getTree + + assertThat(createUniqueSymbol(root, "foo"), is("f")) + assertThat(createUniqueSymbol(root, "name"), is("na")) + assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")) + + // The basis string is not a legal return value, even if unique, to avoid multiple symbols + // with the same name if the same basis string is given twice. + assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my")) + } + + @Test + def testCreateClassName() { + assertThat(createClassName("simple"), is("SimpleConfig")) + assertThat(createClassName("a"), is("AConfig")) + assertThat(createClassName("a-b-c"), is("ABCConfig")) + assertThat(createClassName("a-1-2b"), is("A12bConfig")) + assertThat(createClassName("my-app"), is("MyAppConfig")) + assertThat(createClassName("MyApp"), is("MyAppConfig")) + } + + @Test(expected=classOf[CodegenRuntimeException]) + def testIllegalClassName() { + createClassName("+illegal") + } + +} |