diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespa-documentgen-plugin |
Publish
Diffstat (limited to 'vespa-documentgen-plugin')
17 files changed, 1754 insertions, 0 deletions
diff --git a/vespa-documentgen-plugin/.gitignore b/vespa-documentgen-plugin/.gitignore new file mode 100644 index 00000000000..4e7693f37ce --- /dev/null +++ b/vespa-documentgen-plugin/.gitignore @@ -0,0 +1,3 @@ +/target +/deploy +/pom.xml.build diff --git a/vespa-documentgen-plugin/OWNERS b/vespa-documentgen-plugin/OWNERS new file mode 100644 index 00000000000..0e39145d8c3 --- /dev/null +++ b/vespa-documentgen-plugin/OWNERS @@ -0,0 +1 @@ +dybdahl diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd new file mode 100644 index 00000000000..4b3c85a96de --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/book.sd @@ -0,0 +1,90 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search book { + document book inherits common { + struct ss0 { + field s0 type string{} + field d0 type double{} + } + struct ss1 { + field s1 type string { } + field l1 type long { } + field i1 type int { } + field d1 type double { } + field as1 type array<string> { } + field al1 type array<long> { } + field ss01 type ss0 {} + } + + struct ss2 { + field s21 type string {} + field ss02 type ss0 {} + } + + field mystruct type ss1 { + } + + field mystruct2 type ss2 {} + + field mywsfloat type weightedset<float> { + indexing: attribute + attribute: prefetch + } + + field mynestedwsfloat type weightedset<weightedset<float>> {} + + field myarrayint type array<int> { + indexing: attribute + attribute: prefetch + } + + field stringmap type map<string, string> { + } + + field structmap type map<int, ss1> { + } + + field mysinglestructarray type array<ss1> {} + + #field mydoublestructarray type array<array<ss1>> {} + + field author type string { + bolding: on + # index-to: default, author + indexing: summary | index + } + field isbn type string { + # index-to: default, isbn + indexing: summary | index + } + field year type int { + indexing: summary | index + } + field description type string { + # index-to: default, description + indexing: summary | index + summary: dynamic + } + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | for_each { to_int } | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } +} diff --git a/vespa-documentgen-plugin/etc/complex/common.sd b/vespa-documentgen-plugin/etc/complex/common.sd new file mode 100644 index 00000000000..6441884b506 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/common.sd @@ -0,0 +1,46 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search common { + document common { + field uri type string { + indexing: summary + } + field title type string { + bolding: on + # index-to: default, title + indexing: index|summary + summary-to: smallsum + } + field mid type int { + indexing: attribute|index|summary + } + field scorekey type string { + indexing: summary + } + field weight type float { + indexing { + input weight * 10 | attribute | summary; + } + } + field w1 type float { + indexing { + input weight * 6 + input w1 | summary; + } + } + field w2 type float { + indexing { + input w2 + input weight | summary; + } + } + field did type string { + indexing: attribute|index|summary + } + field ew type string { + indexing: index|summary + } + } + rank-profile default { + first-phase { + expression: nativeRank + } + } +} diff --git a/vespa-documentgen-plugin/etc/complex/music2.sd b/vespa-documentgen-plugin/etc/complex/music2.sd new file mode 100644 index 00000000000..4428e0d1c43 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/music2.sd @@ -0,0 +1,74 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music2 { + document music2 inherits common { + field artist type string { + bolding: on + # index-to: default, artist + indexing: index|summary + } + field disp_song type string { + indexing: summary + } + field song type string { + # index-to: default, song + indexing: index + } + field isbn type string { + bolding: on + # index-to: default, isbn + indexing: index|summary + } + field year type int { + indexing: attribute|index|summary + } + annotation recordlabel {} + + annotation person { + field name type string { } + } + + annotation artist inherits person { + field instrument type int { } + } + + annotation date { + field exacttime type long { } + } + + annotation place { + field lat type long { } + field lon type long { } + } + + annotation event { + field description type string { } + field person type annotationreference<person> { } + field date type annotationreference<date> { } + field place type annotationreference<place> { } + } + + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } + +} diff --git a/vespa-documentgen-plugin/etc/complex/sombrerowebdoc.sd.FOO b/vespa-documentgen-plugin/etc/complex/sombrerowebdoc.sd.FOO new file mode 100644 index 00000000000..1524e7480a0 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/sombrerowebdoc.sd.FOO @@ -0,0 +1,73 @@ +search sombrerowebdoc { + + document sombrerowebdoc { + version : 0 + + struct sombreroattribute { + field attributetype type int {} + field intvalue type int {} + field huffmanindex type int {} + field lexicon type string {} + field name type string {} + field strvalue type string {} + } + + annotation sombreroTOKEN { } + + annotation sombreroPHRASE { + field prefixused type int {} + field suffixused type int {} + field attributes type array<sombreroattribute> {} + field annotationreferences type array<annotationreference<sombreroTOKEN>> {} + } + + struct sombrerocontentfilter { + field id type string {} + field value type string {} + } + + struct sombrerorule { + field name type string {} + field location type string {} + field contentfilters type array<sombrerocontentfilter> {} + } + + struct sombrerometrics { + field executiontime type double {} + } + + struct sombreroextraction { + field adjuncts type string {} + field metrics type sombrerometrics {} + field status type string {} + } + + + field url type string {} + + field raw_html type string {} + + field decoded_html type string {} + + field rules type array<sombrerorule> {} + + field extraction type sombreroextraction {} + + field content_language type string {} + + field dupdetectshinglesketch type string {} + + field mlprisma type string {} + + field raw_bytes type raw {} + + field charset_encoding type string {} + + field meta_language type string {} + + field language type string {} + + field region type string {} + + } +} diff --git a/vespa-documentgen-plugin/etc/complex/video.sd b/vespa-documentgen-plugin/etc/complex/video.sd new file mode 100644 index 00000000000..3714cd95bc2 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/video.sd @@ -0,0 +1,49 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search video { + document video inherits common { + field director type string { + bolding: on + indexing: index|summary + } + field disp_actor type string { + bolding: on + indexing: index|summary + } + field actor type string { + bolding: on + indexing: index|summary + } + field fmt type string { + indexing: index|summary + } + field isbn type string { + bolding: on + indexing: index|summary + } + field year type int { + indexing: attribute|index|summary + } + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | for_each { to_int } | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } +} diff --git a/vespa-documentgen-plugin/etc/localapp/book.sd b/vespa-documentgen-plugin/etc/localapp/book.sd new file mode 100644 index 00000000000..fe9f6b6dcef --- /dev/null +++ b/vespa-documentgen-plugin/etc/localapp/book.sd @@ -0,0 +1,91 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search book { + document book inherits common { + struct ss0 { + field s0 type string{} + field d0 type double{} + } + struct ss1 { + field s1 type string { } + field l1 type long { } + field i1 type int { } + field d1 type double { } + field as1 type array<string> { } + field al1 type array<long> { } + field ss01 type ss0 {} + } + + struct ss2 { + field s21 type string {} + field ss02 type ss0 {} + } + + field mystruct type ss1 { + } + + field mystruct2 type ss2 {} + + field mywsfloat type weightedset<float> { + indexing: attribute + attribute: prefetch + } + + field myarrayint type array<int> { + indexing: attribute + attribute: prefetch + } + + field mytriplearray type array<array<array<int>>> {} + + field stringmap type map<string, string> { + } + + field structmap type map<int, ss1> { + } + + field mysinglestructarray type array<ss1> {} + #field mydoublestructarray type array<array<ss1>> {} + + field author type string { + bolding: on + # index-to: default, author + indexing: summary | index + } + field isbn type string { + # index-to: default, isbn + indexing: summary | index + } + field year type int { + indexing: summary | index + } + field description type string { + # index-to: default, description + indexing: summary | index + summary: dynamic + } + + field myraw type raw {} + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | for_each { to_int } | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } +} diff --git a/vespa-documentgen-plugin/etc/localapp/common.sd b/vespa-documentgen-plugin/etc/localapp/common.sd new file mode 100644 index 00000000000..32dd440fe3c --- /dev/null +++ b/vespa-documentgen-plugin/etc/localapp/common.sd @@ -0,0 +1,51 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search common { + document common { + field uri type string { + indexing: summary + } + field title type string { + bolding: on + # index-to: default, title + indexing: index|summary + summary-to: smallsum + } + field mid type int { + indexing: attribute|index|summary + } + field scorekey type string { + indexing: summary + } + field weight type float { + indexing { + input weight * 10 | attribute | summary; + } + } + field w1 type float { + indexing { + input weight * 6 + input w1 | summary; + } + } + field w2 type float { + indexing { + input w2 + input weight | summary; + } + } + field did type string { + indexing: attribute|index|summary + } + field ew type string { + indexing: index|summary + } + annotation date { + field exacttime type long { } + } + + annotation emptyannotation {} + } + rank-profile default { + first-phase { + expression: nativeRank + } + } +} diff --git a/vespa-documentgen-plugin/etc/localapp/music.sd b/vespa-documentgen-plugin/etc/localapp/music.sd new file mode 100644 index 00000000000..dbbd70d1054 --- /dev/null +++ b/vespa-documentgen-plugin/etc/localapp/music.sd @@ -0,0 +1,70 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music inherits common { + field artist type string { + bolding: on + # index-to: default, artist + indexing: index|summary + } + field disp_song type string { + indexing: summary + } + field song type string { + # index-to: default, song + indexing: index + } + field isbn type string { + bolding: on + # index-to: default, isbn + indexing: index|summary + } + field year type int { + indexing: attribute|index|summary + } + annotation recordlabel {} + + annotation person { + field name type string { } + } + + annotation artist inherits person { + field instrument type int { } + } + + annotation place { + field lat type long { } + field lon type long { } + } + + annotation event { + field description type string { } + field person type annotationreference<person> { } + field date type annotationreference<date> { } + field place type annotationreference<place> { } + } + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } + + +} diff --git a/vespa-documentgen-plugin/etc/localapp/video.sd b/vespa-documentgen-plugin/etc/localapp/video.sd new file mode 100644 index 00000000000..3106477b244 --- /dev/null +++ b/vespa-documentgen-plugin/etc/localapp/video.sd @@ -0,0 +1,54 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search video { + document video inherits common { + field director type string { + bolding: on + # index-to: default, director + indexing: index|summary + } + field disp_actor type string { + bolding: on + # index-to: default, disp_actor + indexing: index|summary + } + field actor type string { + bolding: on + # index-to: default, actor + indexing: index|summary + } + field fmt type string { + # index-to: default, fmt + indexing: index|summary + } + field isbn type string { + bolding: on + # index-to: default, isbn + indexing: index|summary + } + field year type int { + indexing: attribute|index|summary + } + } + + field sw1 type float { + indexing { + input weight * 6 + input w1 + input w2 | summary; + } + } + + field didinteger type array<int> { + indexing: input did | split " " | for_each { to_int } | attribute + } + + rank-profile default { + first-phase { + expression: nativeRank + } + } + rank-profile rp1 inherits default { + #static-rank: weight1 + } + rank-profile rp2 inherits default { + #static-rank: weight2 + } +} diff --git a/vespa-documentgen-plugin/etc/music/music.sd b/vespa-documentgen-plugin/etc/music/music.sd new file mode 100644 index 00000000000..668bce263bb --- /dev/null +++ b/vespa-documentgen-plugin/etc/music/music.sd @@ -0,0 +1,57 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# A basic search definition - called music, should be saved to music.sd +search music { + + # It contains one document type only - called music as well + document music { + + field title type string { + indexing: summary | index # How this field should be indexed + # index-to: title, default # Create two indexes + weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature + header + } + + field artist type string { + indexing: summary | attribute | index + # index-to: artist, default + + weight: 25 + header + } + + field year type int { + indexing: summary | attribute + header + } + + # Increase query + field popularity type int { + indexing: summary | attribute + body + } + + field url type uri { + indexing: summary | index + header + } + + } + + rank-profile default inherits default { + first-phase { + expression: nativeRank(title,artist) + attribute(popularity) + } + + } + + rank-profile textmatch inherits default { + first-phase { + expression: nativeRank(title,artist) + } + + } + + + +} diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml new file mode 100644 index 00000000000..f75637e1978 --- /dev/null +++ b/vespa-documentgen-plugin/pom.xml @@ -0,0 +1,74 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-documentgen-plugin</artifactId> + <packaging>maven-plugin</packaging> + <version>6-SNAPSHOT</version> + <name>Vespa DocumentGen Plugin</name> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.1.1</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-project</artifactId> + <version>2.2.1</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.2</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java new file mode 100644 index 00000000000..995148ab19b --- /dev/null +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa; + +/** + * Represents one configured provided annotation type + * + * @author vegardh + */ +public class Annotation { + + String type; + String clazz; +} diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java new file mode 100644 index 00000000000..1307b25920a --- /dev/null +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -0,0 +1,927 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa; + +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +import java.io.*; +import java.util.*; + +/** + * Goal which generates Vespa document classes from SD files. + * @author vegardh + */ +@Mojo(name = "document-gen", defaultPhase = LifecyclePhase.GENERATE_SOURCES) +public class DocumentGenMojo extends AbstractMojo { + + private long newestModifiedTime = 0; + + private static final int STD_INDENT = 4; + + @Component + private MavenProject project; + + /** + * Directory containing the SD files + */ + @Parameter(defaultValue = ".", required = true) + private File sdDirectory; + + /** + * Java package for generated classes + */ + @Parameter(defaultValue = "com.yahoo.vespa.document", required = true) + private String packageName; + + /** + * User provided annotation types that the plugin should not generate types for. They should however be included in + * ConcreteDocumentFactory. + */ + @Parameter + private List<Annotation> provided = new ArrayList<Annotation>(); + + /** + * Annotation types for which the generated class should be abstract + */ + @Parameter + private List<Annotation> abztract = new ArrayList<Annotation>(); + + /** + * Generate source to here. + */ + @Parameter( + property = "plugin.configuration.outputDirectory", + defaultValue="${project.build.directory}/generated-sources/vespa-documentgen-plugin/", + required = true) + private File outputDirectory; + + private Map<String, Search> searches; + private Map<String, String> docTypes; + private Map<String, String> structTypes; + private Map<String, String> annotationTypes; + + void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException { + if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); + searches = new HashMap<String, Search>(); + docTypes = new HashMap<String, String>(); + structTypes = new HashMap<String, String>(); + annotationTypes = new HashMap<String, String>(); + + outputDir.mkdirs(); + SearchBuilder builder = buildSearches(sdDir); + + boolean annotationsExported=false; + for (NewDocumentType docType : builder.getModel().getDocumentManager().getTypes()) { + if ( docType != VespaDocumentType.INSTANCE) { + exportDocumentSources(outputDir, docType, packageName); + + for (AnnotationType annotationType : docType.getAllAnnotations()) { + if (provided(annotationType.getName())!=null) continue; + annotationsExported=true; + exportAnnotationSources(outputDir, annotationType, docType, packageName); + } + } + } + exportPackageInfo(outputDir, packageName); + if (annotationsExported) exportPackageInfo(outputDir, packageName+".annotation"); + exportDocFactory(outputDir, builder.getSearchList(), packageName); + if (project!=null) project.addCompileSourceRoot(outputDirectory.toString()); + } + + private SearchBuilder buildSearches(File sdDir) { + File[] sdFiles = sdDir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.endsWith(".sd"); + }}); + SearchBuilder builder = new UnprocessingSearchBuilder(); + for (File f : sdFiles) { + try { + long modTime = f.lastModified(); + if (modTime > newestModifiedTime) { + newestModifiedTime = modTime; + } + builder.importFile(f.getAbsolutePath()); + } catch (ParseException | IOException e) { + throw new IllegalArgumentException(e); + } + } + builder.build(); + for (Search search : builder.getSearchList() ) { + this.searches.put(search.getName(), search); + } + return builder; + } + + /** + * Creates package-info.java, so that the package of the concrete doc types get exported from user's bundle. + */ + private void exportPackageInfo(File outputDir, String packageName) { + File dirForSources = new File(outputDir, packageName.replaceAll("\\.", "/")); + dirForSources.mkdirs(); + File target = new File(dirForSources, "package-info.java"); + if (target.lastModified() > newestModifiedTime) { + getLog().debug("No changes, not updating "+target); + return; + } + try (Writer out = new FileWriter(target)) { + out.write("@ExportPackage\n" + + "package "+packageName+";\n\n" + + "import com.yahoo.osgi.annotation.ExportPackage;\n"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * The class for this type if provided, otherwise null + */ + private String provided(String annotationType) { + if (provided==null) return null; + for (Annotation a : provided) { + if (a.type.equals(annotationType)) return a.clazz; + } + return null; + } + + private boolean isAbstract(String annotationType) { + if (abztract==null) return false; + for (Annotation a : abztract) { + if (a.type.equals(annotationType)) return true; + } + return false; + } + + private void exportDocFactory(File outputDir, List<Search> searches, String packageName) { + File dirForSources = new File(outputDir, packageName.replaceAll("\\.", "/")); + dirForSources.mkdirs(); + File target = new File(dirForSources, "ConcreteDocumentFactory.java"); + if (target.lastModified() > newestModifiedTime) { + getLog().debug("No changes, not updating "+target); + return; + } + try (Writer out = new FileWriter(target)) { + out.write("package "+packageName+";\n\n" + + "/**\n" + + " * Registry of generated concrete document, struct and annotation types.\n" + + " *\n" + + " * Generated by vespa-documentgen-plugin, do not edit.\n" + + " * Date: "+new Date()+"\n" + + " */\n"); + out.write("@com.yahoo.document.Generated public class ConcreteDocumentFactory extends com.yahoo.docproc.AbstractConcreteDocumentFactory {\n"); + out.write(ind()+"private static java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.Document>> dTypes = new java.util.HashMap<java.lang.String, java.lang.Class<? extends com.yahoo.document.Document>>();\n"); + out.write(ind()+"private static java.util.Map<java.lang.String, com.yahoo.document.DocumentType> docTypes = new java.util.HashMap<>();\n"); + out.write(ind()+"private static java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.datatypes.Struct>> sTypes = new java.util.HashMap<java.lang.String, java.lang.Class<? extends com.yahoo.document.datatypes.Struct>>();\n"); + out.write(ind()+"private static java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.annotation.Annotation>> aTypes = new java.util.HashMap<java.lang.String, java.lang.Class<? extends com.yahoo.document.annotation.Annotation>>();\n"); + out.write(ind()+"static {\n"); + for (Map.Entry<String, String> e : docTypes.entrySet()) { + String docTypeName = e.getKey(); + String fullClassName = e.getValue(); + out.write(ind(2)+"dTypes.put(\""+docTypeName+"\","+fullClassName+".class);\n"); + } + for (Map.Entry<String, String> e : docTypes.entrySet()) { + String docTypeName = e.getKey(); + String fullClassName = e.getValue(); + out.write(ind(2)+"docTypes.put(\""+docTypeName+"\","+fullClassName+".type);\n"); + } + for (Map.Entry<String, String> e : structTypes.entrySet()) { + String structTypeName = e.getKey(); + String fullClassName = e.getValue(); + out.write(ind(2)+"sTypes.put(\""+structTypeName+"\","+fullClassName+".class);\n"); + } + for (Map.Entry<String, String> e : annotationTypes.entrySet()) { + String annotationTypeName = e.getKey(); + String fullClassName = e.getValue(); + out.write(ind(2)+"aTypes.put(\""+annotationTypeName+"\","+fullClassName+".class);\n"); + } + for (Annotation a : provided) { + out.write(ind(2)+"aTypes.put(\""+a.type+"\","+a.clazz+".class);\n"); + } + out.write(ind()+"}\n\n"); + + out.write( + ind()+"public static final java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.Document>> documentTypes = java.util.Collections.unmodifiableMap(dTypes);\n" + + ind()+"public static final java.util.Map<java.lang.String, com.yahoo.document.DocumentType> documentTypeObjects = java.util.Collections.unmodifiableMap(docTypes);\n" + + ind()+"public static final java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.datatypes.Struct>> structTypes = java.util.Collections.unmodifiableMap(sTypes);\n" + + ind()+"public static final java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.annotation.Annotation>> annotationTypes = java.util.Collections.unmodifiableMap(aTypes);\n\n"); + out.write( + ind()+"@Override public java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.Document>> documentTypes() { return ConcreteDocumentFactory.documentTypes; }\n" + + ind()+"@Override public java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.datatypes.Struct>> structTypes() { return ConcreteDocumentFactory.structTypes; }\n" + + ind()+"@Override public java.util.Map<java.lang.String, java.lang.Class<? extends com.yahoo.document.annotation.Annotation>> annotationTypes() { return ConcreteDocumentFactory.annotationTypes; }\n\n"); + + out.write( + ind()+"/**\n" + + ind()+" * Returns a new Document which will be of the correct concrete type and a copy of the given StructFieldValue.\n" + + ind()+" */\n" + + ind()+"@Override public com.yahoo.document.Document getDocumentCopy(java.lang.String type, com.yahoo.document.datatypes.StructuredFieldValue src, com.yahoo.document.DocumentId id) {\n"); + for (Map.Entry<String, String> e : docTypes.entrySet()) { + out.write(ind(2)+"if (\""+e.getKey()+"\".equals(type)) return new "+e.getValue()+"(src, id);\n"); + } + out.write(ind(2)+"throw new java.lang.IllegalArgumentException(\"No such concrete document type: \"+type);\n"); + out.write(ind()+"}\n\n"); + + // delete, bad to use reflection? + out.write( + ind()+"/**\n" + + ind()+" * Returns a new Document which will be of the correct concrete type.\n" + + ind()+" */\n" + + ind()+"public static com.yahoo.document.Document getDocument(java.lang.String type, com.yahoo.document.DocumentId id) {\n" + + ind(2)+"if (!ConcreteDocumentFactory.documentTypes.containsKey(type)) throw new java.lang.IllegalArgumentException(\"No such concrete document type: \"+type);\n" + + ind(2)+"try {\n" + + ind(3)+"return ConcreteDocumentFactory.documentTypes.get(type).getConstructor(com.yahoo.document.DocumentId.class).newInstance(id);\n" + + ind(2)+"} catch (java.lang.Exception ex) { throw new java.lang.RuntimeException(ex); }\n" + + ind()+"}\n\n"); + + // delete, bad to use reflection? + out.write( + ind()+"/**\n" + + ind()+" * Returns a new Struct which will be of the correct concrete type.\n" + + ind()+" */\n" + + ind()+"public static com.yahoo.document.datatypes.Struct getStruct(java.lang.String type) {\n" + + ind(2)+"if (!ConcreteDocumentFactory.structTypes.containsKey(type)) throw new java.lang.IllegalArgumentException(\"No such concrete struct type: \"+type);\n" + + ind(2)+"try {\n" + + ind(3)+"return ConcreteDocumentFactory.structTypes.get(type).getConstructor().newInstance();\n" + + ind(2)+"} catch (java.lang.Exception ex) { throw new java.lang.RuntimeException(ex); }\n" + + ind()+"}\n\n"); + + // delete, bad to use reflection? + out.write( + ind()+"/**\n" + + ind()+" * Returns a new Annotation which will be of the correct concrete type.\n" + + ind()+" */\n" + + ind()+"public static com.yahoo.document.annotation.Annotation getAnnotation(java.lang.String type) {\n" + + ind(2)+"if (!ConcreteDocumentFactory.annotationTypes.containsKey(type)) throw new java.lang.IllegalArgumentException(\"No such concrete annotation type: \"+type);\n" + + ind(2)+"try {\n" + + ind(3)+"return ConcreteDocumentFactory.annotationTypes.get(type).getConstructor().newInstance();\n" + + ind(2)+"} catch (java.lang.Exception ex) { throw new java.lang.RuntimeException(ex); }\n" + + ind()+"}\n\n"); + + out.write("}\n"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void exportAnnotationSources(File outputDir, AnnotationType annType, NewDocumentType docType, String packageName) { + File dirForSources = new File(outputDir, packageName.replaceAll("\\.", "/")+"/annotation/"); + dirForSources.mkdirs(); + String className = className(annType.getName()); + File target = new File(dirForSources, className+".java"); + if (target.lastModified() > newestModifiedTime) { + getLog().debug("No changes, not updating "+target); + return; + } + try (Writer out = new FileWriter(target)) { + out.write("package "+packageName+".annotation;\n\n" + + "import "+packageName+".ConcreteDocumentFactory;\n" + + exportInnerImportsFromDocAndSuperTypes(docType, packageName) + + exportImportProvidedAnnotationRefs(annType) + + "/**\n" + + " * Generated by vespa-documentgen-plugin, do not edit.\n" + + " * Input annotation type: "+annType.getName()+"\n" + + " * Date: "+new Date()+"\n" + + " */\n" + + "@com.yahoo.document.Generated public "+annTypeModifier(annType)+"class "+className+" extends "+getParentAnnotationType(annType)+" {\n\n"); + if (annType.getDataType() instanceof StructDataType) { + out.write(ind() + "public "+className+"() {\n" + + ind(2) + "setType(new com.yahoo.document.annotation.AnnotationType(\""+annType.getName()+"\", Fields.type));\n" + + ind()+"}\n\n"); + StructDataType annStruct = (StructDataType)annType.getDataType(); + StructDataType annStructTmp = new StructDataType("fields"); // Change the type name + annStructTmp.assign(annStruct); + Collection<DataType> tmpList = new ArrayList<DataType>(); + tmpList.add(annStructTmp); + exportStructTypes(tmpList, out, 1, null); + exportFieldsAndAccessors(className, annStruct.getFieldsThisTypeOnly(), out, 1, false); + out.write(ind()+"@Override public boolean hasFieldValue() { return true; }\n\n"); + out.write(ind()+"@Override public com.yahoo.document.datatypes.FieldValue getFieldValue() {\n"); + out.write(ind(2)+"com.yahoo.document.datatypes.Struct ret = new Fields();\n"); + out.write(ind(2)+"com.yahoo.document.datatypes.FieldValue fv; com.yahoo.document.Field f;\n"); + + for (Field f : annStruct.getFields()) { + out.write( + ind(2)+"if ("+getter(f.getName()) +"()!=null) {\n" + + ind(3)+"f = new com.yahoo.document.Field(\""+f.getName()+"\", "+toJavaReference(f.getDataType())+");\n" + + ind(3)+"fv = f.getDataType().createFieldValue("+getter(f.getName())+"());\n" + + ind(3)+"ret.setFieldValue(f, fv);\n" + + ind(2)+"}\n"); + } + out.write(ind(2)+"return ret;\n"); + out.write(ind()+"}\n\n"); + + } else { + out.write(ind() + "public "+className+"() { setType(new com.yahoo.document.annotation.AnnotationType(\""+annType.getName()+"\")); } \n\n"); + out.write(ind()+"@Override public boolean hasFieldValue() { return false; }\n\n"); + out.write(ind()+"@Override public com.yahoo.document.datatypes.FieldValue getFieldValue() { return null; }\n\n"); + } + out.write("}\n"); + annotationTypes.put(annType.getName(), packageName+".annotation."+className); + } catch (IOException e) { + throw new RuntimeException("Could not export sources for annotation type '"+annType.getName()+"'", e); } + } + + /** + * Handle the case of an annotation reference with a type that is user provided, we need to know the class name then + */ + private String exportImportProvidedAnnotationRefs(AnnotationType annType) { + String ret = ""; + if (annType.getDataType()==null) return ret; + for (Field f : ((StructDataType)annType.getDataType()).getFields()) { + if (f.getDataType() instanceof AnnotationReferenceDataType) { + AnnotationReferenceDataType refType = (AnnotationReferenceDataType) f.getDataType(); + AnnotationType referenced = refType.getAnnotationType(); + String providedClass = provided(referenced.getName()); + if (providedClass!=null) { + // Annotationreference is to a type that is user-provided + ret = ret + + "import "+providedClass+";\n"; + } + } + } + return ret; + } + + private String annTypeModifier(AnnotationType annType) { + if (isAbstract(annType.getName())) return "abstract "; + return ""; + } + + private String exportInnerImportsFromDocAndSuperTypes(NewDocumentType docType, String packageName) { + String ret = ""; + ret = ret + "import "+packageName+"."+className(docType.getName())+".*;\n"; + ret = ret + exportInnerImportsFromSuperTypes(docType, packageName); + return ret; + } + + private String exportInnerImportsFromSuperTypes(NewDocumentType docType, String packageName) { + String ret = ""; + for (NewDocumentType inherited : docType.getInherited()) { + if (inherited.getName().equals("document")) continue; + ret = ret + "import "+packageName+"."+className(inherited.getName())+".*;\n"; + } + return ret; + } + + private String getParentAnnotationType(AnnotationType annType) { + if (annType.getInheritedTypes().isEmpty()) return "com.yahoo.document.annotation.Annotation"; + String superType = annType.getInheritedTypes().iterator().next().getName(); + String providedSuperType = provided(superType); + if (providedSuperType!=null) return providedSuperType; + return className(annType.getInheritedTypes().iterator().next().getName()); + } + + private void exportDocumentSources(File outputDir, NewDocumentType docType, String packageName) + throws MojoFailureException { + File dirForSources = new File(outputDir, packageName.replaceAll("\\.", "/")); + dirForSources.mkdirs(); + File target = new File(dirForSources, className(docType.getName())+".java"); + if (target.lastModified() > newestModifiedTime) { + getLog().debug("No changes, not updating "+target); + return; + } + try (Writer doc = new FileWriter(target)) { + exportDocumentClass(docType, doc, packageName); + } catch (IOException e) { + throw new RuntimeException("Could not export sources for document type '"+docType.getName()+"'", e); + } + } + + /** + * Generates the .java file for the Document subclass for the given document type + */ + private void exportDocumentClass(NewDocumentType docType, Writer out, String packageName) throws IOException { + String className = className(docType.getName()); + String superType = javaSuperType(docType); + out.write( + "package "+packageName+";\n\n" + + exportInnerImportsFromSuperTypes(docType, packageName) + + "/**\n" + + " * Generated by vespa-documentgen-plugin, do not edit.\n" + + " * Input document type: "+docType.getName()+"\n" + + " * Date: "+new Date()+"\n" + + " */\n" + + "@com.yahoo.document.Generated public class "+className+" extends "+superType+" {\n\n"+ + ind(1)+"/** The doc type of this.*/\n" + + ind(1)+"public static final com.yahoo.document.DocumentType type = getDocumentType();\n\n"+ + ind(1)+"/** Struct type view of the type of the body of this.*/\n" + + ind(1)+"private static final com.yahoo.document.StructDataType bodyStructType = getBodyStructType();\n\n" + + ind(1)+"/** Struct type view of the type of the header of this.*/\n" + + ind(1)+"private static final com.yahoo.document.StructDataType headerStructType = getHeaderStructType();\n\n"); + + // Constructor + out.write( + ind(1)+"public "+className+"(com.yahoo.document.DocumentId docId) {\n" + + ind(2)+"super("+className+".type, docId);\n" + + ind(1)+"}\n\n"); + out.write( + ind(1)+"protected "+className+"(com.yahoo.document.DocumentType type, com.yahoo.document.DocumentId docId) {\n" + + ind(2)+"super(type, docId);\n" + + ind(1)+"}\n\n"); + // isGenerated() + out.write(ind(1)+"@Override protected boolean isGenerated() { return true; }\n\n"); + + // Mimic header and body to make serialization work. + // This can be improved by generating a method to serialize the document _here_, and use that in serialization. + exportOverriddenStructGetter(((StructDataType) docType.allHeader()).getFields(), out, 1, "getHeader", className+".headerStructType"); + exportOverriddenStructGetter(((StructDataType) docType.allBody()).getFields(), out, 1, "getBody", className+".bodyStructType"); + exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType"); + exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); + + exportStructTypeGetter(docType.getName(), docType.getAllFields(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); + exportCopyConstructor(className, docType.getAllFields(), out, 1, true); + exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? docType.getAllFields() : docType.getFields(), out, 1, true); + exportDocumentMethods(docType.getAllFields(), out, 1); + exportHashCode(docType.getAllFields(), out, 1, "(getDataType() != null ? getDataType().hashCode() : 0) + getId().hashCode()"); + exportEquals(className, docType.getAllFields(), out, 1); + Set<DataType> exportedStructs = exportStructTypes(docType.getTypes(), out, 1, null); + docTypes.put(docType.getName(), packageName+"."+className); + for (DataType exportedStruct : exportedStructs) { + structTypes.put(exportedStruct.getName(), packageName+"."+className+"."+className(exportedStruct.getName())); + } + out.write("}\n"); + } + + /** + * The Java class the class of the given type should inherit from. If the input type inherits from _one_ + * other type, use that, otherwise Document. + */ + private String javaSuperType(NewDocumentType docType) { + String ret = "com.yahoo.document.Document"; + Collection<NewDocumentType> specInheriteds = specificInheriteds(docType); + if (!specInheriteds.isEmpty() && singleInheritance(specInheriteds)) ret = className(specInheriteds.iterator().next().getName()); + return ret; + } + + private boolean singleInheritance(Collection<NewDocumentType> specInheriteds) { + if (specInheriteds.isEmpty()) return true; + if (specInheriteds.size()>1) return false; + return singleInheritance(specificInheriteds(specInheriteds.iterator().next())); + } + + /** + * The inherited types that are not Document + * @return collection of specific inherited types + */ + private Collection<NewDocumentType> specificInheriteds(NewDocumentType type) { + List<NewDocumentType> ret = new ArrayList<NewDocumentType>(); + for (NewDocumentType t : type.getInherited()) { + if (!"document".equals(t.getName())) ret.add(t); + } + return ret; + } + + /** + * Exports the copy constructor. + * + * NOTE: This is important, the docproc framework uses that constructor. + */ + private void exportCopyConstructor(String className, Collection<Field> fieldSet, Writer out, int ind, boolean docId) throws IOException { + out.write( + ind(ind)+"/**\n"+ + ind(ind)+" * Constructs a "+className+" by taking a deep copy of the provided StructuredFieldValue.\n" + + ind(ind)+" */\n"); + if (docId) { + out.write(ind(ind)+"public "+className+"(com.yahoo.document.datatypes.StructuredFieldValue src, com.yahoo.document.DocumentId docId) {\n"+ + ind(ind+1)+"super("+className+".type,docId);\n"); + } else { + out.write(ind(ind)+"public "+className+"(com.yahoo.document.datatypes.StructuredFieldValue src) {\n"+ + ind(ind+1)+"super("+className+".type);\n"); + } + out.write( + ind(ind+1)+"for (java.util.Iterator<java.util.Map.Entry<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue>>i=src.iterator() ; i.hasNext() ; ) {\n" + + ind(ind+2)+"java.util.Map.Entry<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue> e = i.next();\n" + + ind(ind+2)+"com.yahoo.document.Field f = e.getKey();\n" + + ind(ind+2)+"com.yahoo.document.datatypes.FieldValue fv = e.getValue();\n" + + ind(ind+2)+"if (fv instanceof com.yahoo.document.datatypes.StructuredFieldValue) {\n" + + ind(ind+3)+"try {\n" + + ind(ind+4)+"com.yahoo.document.datatypes.StructuredFieldValue newVal = ConcreteDocumentFactory.structTypes.get(f.getDataType().getName()).getConstructor(com.yahoo.document.datatypes.StructuredFieldValue.class).newInstance(fv);\n" + + ind(ind+4)+"setFieldValue(f, newVal);\n" + + ind(ind+3)+"} catch (java.lang.Exception ex) { throw new java.lang.RuntimeException(ex); }\n" + + ind(ind+2)+"} else {\n" + + ind(ind+3)+"setFieldValue(f, fv);\n" + + ind(ind+2)+"}\n" + + ind(ind+1)+"}\n"+ + ind(ind)+"}\n\n"); + } + + private void exportStructTypeGetter(String name, Collection<Field> fields, Writer out, int ind, String methodName, String retType) throws IOException { + out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n" + + ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + for (Field f : fields) { + out.write(ind(ind+1)+"ret.addField(new com.yahoo.document.Field(\""+f.getName()+"\", "+toJavaReference(f.getDataType())+"));\n"); + + } + out.write(ind(ind+1)+"return ret;\n"); + out.write(ind(ind)+"}\n\n"); + } + + private void exportOverriddenStructGetter(Collection<Field> fields, Writer out, int ind, String methodName, String structType) throws IOException { + out.write(ind(ind)+"@Override public com.yahoo.document.datatypes.Struct "+methodName+"() {\n" + + ind(ind+1)+"com.yahoo.document.datatypes.Struct ret = new com.yahoo.document.datatypes.Struct("+structType+");\n"); + for (Field f : fields) { + out.write(ind(ind+1)+"ret.setFieldValue(\""+f.getName()+"\", getFieldValue(getField(\""+f.getName()+"\")));\n"); + + } + out.write(ind(ind+1)+"return ret;\n"); + out.write(ind(ind)+"}\n\n"); + } + + /** + * Exports the necessary overridden methods from Document/StructuredFieldValue + */ + private void exportDocumentMethods(Collection<Field> fieldSet, Writer out, + int ind) throws IOException { + exportGetFieldCount(fieldSet, out, ind); + exportGetField(fieldSet, out, ind); + exportGetFieldValue(fieldSet, out, ind); + exportToNamedMap(out, ind); + exportSetFieldValue(fieldSet, out, ind); + exportRemoveFieldValue(fieldSet, out, ind); + exportIterator(fieldSet, out, ind); + exportClear(fieldSet, out, ind); + + } + + private void exportEquals(String className, Collection<Field> fieldSet, Writer out, int ind) throws IOException { + out.write(ind(ind)+"@Override public boolean equals(Object o) {\n"); + out.write(ind(ind+1)+"if (!(o instanceof "+className+")) return false;\n"); + out.write(ind(ind+1)+className+" other = ("+className+")o;\n"); + for (Field field: fieldSet) { + out.write(ind(ind+1)+"if ("+getter(field.getName())+"()==null) { if (other."+getter(field.getName())+"()!=null) return false; }\n"); + out.write(ind(ind+2)+"else if (!("+getter(field.getName())+"().equals(other."+getter(field.getName())+"()))) return false;\n"); + } + out.write(ind(ind+1)+"return true;\n"); + out.write(ind(ind)+"}\n\n"); + } + + private void exportToNamedMap(Writer out, int ind) throws IOException { + // A helper to convert from SpanTree collection to Map. Can be removed if StringFieldValue is fixed to expose the map. + out.write( + ind()+"private java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> toNamedMap(java.util.Collection<com.yahoo.document.annotation.SpanTree> coll) {\n" + + ind(ind+1)+"if (coll==null) return null;\n" + + ind(ind+1)+"java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> ret = new java.util.HashMap<java.lang.String,com.yahoo.document.annotation.SpanTree>();\n" + + ind(ind+1)+"for (com.yahoo.document.annotation.SpanTree st : coll) ret.put(st.getName(), st);\n" + + ind(ind+1)+"return ret;\n" + + ind()+"}\n\n" + ); + } + + private void exportHashCode(Collection<Field> fieldSet, Writer out, int ind, String hcBase) + throws IOException { + out.write(ind(ind)+"@Override public int hashCode() {\n"); + out.write(ind(ind+1)+"int hc = "+hcBase+";\n"); + for (Field field: fieldSet) { + out.write(ind(ind+1)+"hc += "+getter(field.getName())+"()!=null ? "+getter(field.getName())+"().hashCode() : 0;\n"); + } + out.write(ind(ind+1)+"return hc;\n"); + out.write(ind(ind)+"}\n\n"); + } + + private void exportClear(Collection<Field> fieldSet, Writer out, int ind) + throws IOException { + out.write(ind(ind)+"@Override public void clear() {\n"); + for (Field field: fieldSet) { + out.write(ind(ind+1)+setter(field.getName())+"(null);\n"); + } + out.write(ind(ind)+"}\n\n"); + } + + private void exportIterator(Collection<Field> fieldSet, Writer out, int ind) + throws IOException { + out.write(ind(ind)+"@Override public java.util.Iterator<java.util.Map.Entry<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue>> iterator() {\n"); + out.write( + ind(ind+1)+"java.util.Map<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue> ret = new java.util.HashMap<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue>();\n" + + ind(ind+1)+"com.yahoo.document.Field f;\n"); + for (Field field: fieldSet) { + String name = field.getName(); + out.write( + ind(ind+1)+"if ("+getter(name)+"()!=null) {\n" + + ind(ind+2)+"f = new com.yahoo.document.Field(\""+name+"\", "+toJavaReference(field.getDataType())+");\n" + + ind(ind+2)+"com.yahoo.document.datatypes.FieldValue fv = f.getDataType().createFieldValue("+getter(name)+"());\n"); + if (field.getDataType().equals(DataType.STRING)) { + out.write( + ind(ind+2)+"if (fv instanceof com.yahoo.document.datatypes.StringFieldValue) if ("+spanTreeGetter(name)+"()!=null) {\n" + + ind(ind+3)+"for (com.yahoo.document.annotation.SpanTree tree : "+spanTreeGetter(name)+"().values()) ((com.yahoo.document.datatypes.StringFieldValue)fv).setSpanTree(tree);\n"+ + ind(ind+2)+"}\n"); + } + out.write( + ind(ind+2)+"ret.put(f, fv);\n" + + ind(ind+1)+"}\n"); + + } + out.write( + ind(ind+1)+"return ret.entrySet().iterator();\n" + + ind(ind)+"}\n\n"); + } + + private void exportRemoveFieldValue(Collection<Field> fieldSet, Writer out, + int ind) throws IOException { + out.write(ind(ind)+"@Override public com.yahoo.document.datatypes.FieldValue removeFieldValue(com.yahoo.document.Field field) {\n"); + out.write(ind(ind+1)+"if (field==null) return null;\n"); + for (Field field: fieldSet) { + String name = field.getName(); + out.write( + ind(ind+1)+"if (\""+name+"\".equals(field.getName())) {\n"); + out.write( + ind(ind+2)+"com.yahoo.document.datatypes.FieldValue ret = field.getDataType().createFieldValue("+getter(name)+"());\n"); + if (field.getDataType().equals(DataType.STRING)) { + out.write( + ind(ind+2)+"if (ret instanceof com.yahoo.document.datatypes.StringFieldValue) if ("+spanTreeGetter(name)+"()!=null) {\n" + + ind(ind+3)+"for (com.yahoo.document.annotation.SpanTree tree : "+spanTreeGetter(name)+"().values()) ((com.yahoo.document.datatypes.StringFieldValue)ret).setSpanTree(tree);\n"+ + ind(ind+3)+spanTreeSetter(name)+"(null);\n"+ + ind(ind+2)+"}\n"); + } + out.write( + ind(ind+2)+ setter(name)+"(null);\n" + + ind(ind+2)+"return ret;\n" + + ind(ind+1)+"}\n"); + } + out.write( + ind(ind+1)+"return super.removeFieldValue(field);\n" + + ind(ind)+"}\n\n"); + } + + private void exportSetFieldValue(Collection<Field> fieldSet, Writer out, + int ind) throws IOException { + out.write(ind(ind)+"@Override public com.yahoo.document.datatypes.FieldValue setFieldValue(com.yahoo.document.Field field, com.yahoo.document.datatypes.FieldValue value) {\n"); + for (Field field: fieldSet) { + String name = field.getName(); + out.write( + ind(ind+1)+"if (\""+name+"\".equals(field.getName())) {\n"); + if (field.getDataType().equals(DataType.STRING)) { + out.write( + ind(ind+2)+"if (value instanceof com.yahoo.document.datatypes.StringFieldValue) "+spanTreeSetter(name)+"(toNamedMap(((com.yahoo.document.datatypes.StringFieldValue)value).getSpanTrees()));\n"); + } + out.write( + ind(ind+2)+"com.yahoo.document.datatypes.FieldValue old=getFieldValue(field);\n"+ + ind(ind+2)+setter(name)+"(("+toJavaType(field.getDataType())+")value.getWrappedValue());\n" + + ind(ind+2)+"return old;\n" + + ind(ind+1)+"}\n"); + } + out.write( + ind(ind+1)+"return super.setFieldValue(field, value);\n" + + ind(ind)+"}\n\n"); + } + + private void exportGetFieldValue(Collection<Field> fieldSet, Writer out, + int ind) throws IOException { + out.write(ind(ind)+"@Override public com.yahoo.document.datatypes.FieldValue getFieldValue(com.yahoo.document.Field field) {\n"); + out.write(ind(ind+1)+"if (field==null) return null;\n"); + for (Field field: fieldSet) { + String name = field.getName(); + if (field.getDataType().equals(DataType.STRING)) { + out.write( + ind(ind+1)+"if (\""+name+"\".equals(field.getName())) {\n"+ + ind(ind+2)+"if ("+getter(name)+"()==null) return null;\n" + + ind(ind+2)+"com.yahoo.document.datatypes.FieldValue fv = field.getDataType().createFieldValue("+getter(name)+"());\n" + + ind(ind+2)+"if (!(fv instanceof com.yahoo.document.datatypes.StringFieldValue)) return fv;\n"+ // Should never happen + ind(ind+2)+"com.yahoo.document.datatypes.StringFieldValue ret = (com.yahoo.document.datatypes.StringFieldValue)fv;\n" + + ind(ind+2)+"if ("+spanTreeGetter(name)+"()!=null) for (java.util.Map.Entry<java.lang.String,com.yahoo.document.annotation.SpanTree> tree : "+spanTreeGetter(name)+"().entrySet()) ret.setSpanTree(tree.getValue());\n" + + ind(ind+2)+"return ret;\n" + + ind(ind+1)+"}\n"); + } else if (field.getDataType() instanceof StructDataType) { + out.write(ind(ind+1)+"if (\""+name+"\".equals(field.getName())) return "+name+";\n"); + } else { + out.write( + ind(ind+1)+"if (\""+name+"\".equals(field.getName())) { if ("+getter(name)+"()==null) return null; return field.getDataType().createFieldValue("+getter(name)+"()); }\n"); + } + } + out.write( + ind(ind+1)+"return super.getFieldValue(field);\n" + + ind(ind)+"}\n\n"); + } + + private void exportGetField(Collection<Field> fieldSet, Writer out, int ind) + throws IOException { + out.write(ind(ind)+"@Override public com.yahoo.document.Field getField(String fieldName) {\n"); + out.write(ind(ind+1)+"if (fieldName==null) return null;\n"); + for (Field field: fieldSet) { + String name = field.getName(); + out.write( + ind(ind+1)+"if (\""+name+"\".equals(fieldName) ) { return new com.yahoo.document.Field(fieldName, "+toJavaReference(field.getDataType())+"); }\n"); // TODO what now... + } + out.write( + ind(ind+1)+"return null;\n" + + ind(ind)+"}\n\n"); + } + + /** + * Exports the struct types found in this collection of fields as separate Java classes + */ + private Set<DataType> exportStructTypes(Collection<DataType> fields, Writer out, int ind, Set<DataType> exportedStructs) throws IOException { + if (exportedStructs==null) exportedStructs=new HashSet<DataType>(); + for (DataType f : fields) { + if ((f instanceof StructDataType) && ! f.getName().contains(".")) { + if (exportedStructs.contains(f)) continue; + exportedStructs.add(f); + StructDataType structType = (StructDataType) f; + String structClassName = className(structType.getName()); + out.write(ind(ind)+"/**\n" + + ind(ind)+" * Generated by vespa-documentgen-plugin, do not edit.\n" + + ind(ind)+" * Input struct type: "+structType.getName()+"\n" + + ind(ind)+" * Date: "+new Date()+"\n" + + ind(ind)+" */\n" + + ind(ind)+"@com.yahoo.document.Generated public static class "+structClassName+" extends com.yahoo.document.datatypes.Struct {\n\n" + + ind(ind+1)+"/** The type of this.*/\n" + + ind(ind+1)+"public static final com.yahoo.document.StructDataType type = getStructType();\n\n"); + out.write(ind(ind+1)+"public "+structClassName+"() {\n" + + ind(ind+2)+"super("+structClassName+".type);\n" + + ind(ind+1)+"}\n\n"); + exportCopyConstructor(structClassName, structType.getFields(), out, ind+1, false); + exportStructTypeGetter(structType.getName(), structType.getFields(), out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportAssign(structType, structClassName, out, ind+1); + exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); + + // Need these methods for serialization. + // This can be improved by generating a method to serialize the struct _here_ (and one in the document), and use that in serialization. + exportGetFields(structType.getFields(), out, ind+1); + exportDocumentMethods(structType.getFields(), out, ind+1); + exportHashCode(structType.getFields(), out, ind+1, "(getDataType() != null ? getDataType().hashCode() : 0)"); + exportEquals(structClassName, structType.getFields(), out, ind+1); + out.write(ind(ind)+"}\n\n"); + } + } + return exportedStructs; + } + + /** + * Override this, serialization of structs relies on it + */ + private void exportGetFieldCount(Collection<Field> fields, Writer out, int ind) throws IOException { + out.write(ind(ind)+"@Override public int getFieldCount() {\n"); + out.write(ind(ind+1)+"int ret=0;\n"); + for (Field f : fields) { + out.write(ind(ind+1)+"if ("+getter(f.getName())+"()!=null) ret++;\n"); + } + out.write(ind(ind+1)+"return ret;\n"); + out.write(ind(ind)+"}\n\n"); + } + + /** + * Override the getFields() method of Struct, since serialization of Struct relies on it. + */ + private void exportGetFields(Collection<Field> fields, Writer out, int ind) throws IOException { + out.write(ind(ind)+"@Override public java.util.Set<java.util.Map.Entry<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue>> getFields() {\n" + + ind(ind+1)+"java.util.Map<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue> ret = new java.util.LinkedHashMap<com.yahoo.document.Field, com.yahoo.document.datatypes.FieldValue>();\n" + + ind(ind+1)+"com.yahoo.document.Field f;\n"); + for (Field f : fields) { + out.write(ind(ind+1)+"if ("+getter(f.getName())+"()!=null) {\n"); + out.write(ind(ind+2)+"f = new com.yahoo.document.Field(\""+f.getName()+"\", "+toJavaReference(f.getDataType())+");\n"); + out.write(ind(ind+2)+"ret.put(f, f.getDataType().createFieldValue("+getter(f.getName())+"()));\n"); + out.write(ind(ind+1)+"}\n"); + } + out.write(ind(ind+1)+"return ret.entrySet();\n"); + out.write(ind(ind)+"}\n\n"); + } + + private void exportAssign(StructDataType structType, String structClassName, Writer out, int ind) throws IOException { + out.write(ind(ind)+"@Override public void assign(Object o) {\n"+ + ind(ind+1)+"if (!(o instanceof "+structClassName+")) { super.assign(o); return; }\n"+ + ind(ind+1)+structClassName+" other = ("+structClassName+")o;\n"); + for (Field f: structType.getFields()) { + out.write(ind(ind+1)+setter(f.getName())+"(other."+getter(f.getName())+"());\n"); + } + + out.write(ind(ind)+"}\n\n"); + } + + /** + * Exports this set of fields with getters and setters + * @param spanTrees If true, include a reference to the list of span trees for the string fields + */ + private void exportFieldsAndAccessors(String className, Collection<Field> fields, Writer out, int ind, boolean spanTrees) throws IOException { + // List the fields as Java fields + for (Field field: fields) { + DataType dt = field.getDataType(); + out.write( + ind(ind)+toJavaType(dt)+" "+field.getName()+";\n"); + if (spanTrees && dt.equals(DataType.STRING)) { + out.write(ind(ind)+"java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> "+spanTreeGetter(field.getName())+";\n"); // same name on field as get method + } + } + out.write(ind(ind)+"\n"); + // Getters, setters and annotation spantree lists for string fields + for (Field field: fields) { + DataType dt = field.getDataType(); + out.write( + ind(ind)+"public "+toJavaType(dt)+" "+getter(field.getName())+"() { return "+field.getName()+"; }\n"+ + ind(ind)+"public "+className+" "+setter(field.getName())+"("+toJavaType(dt)+" "+field.getName()+") { this."+field.getName()+"="+field.getName()+"; return this; }\n"); + if (spanTrees && dt.equals(DataType.STRING)) { + out.write(ind(ind)+"public java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> "+spanTreeGetter(field.getName())+"() { return "+field.getName()+"SpanTrees; }\n" + + ind(ind)+"public void "+spanTreeSetter(field.getName())+"(java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> spanTrees) { this."+field.getName()+"SpanTrees=spanTrees; }\n"); + } + } + out.write("\n"); + } + + private String spanTreeSetter(String field) { + return setter(field)+"SpanTrees"; + } + + private String spanTreeGetter(String field) { + return field+"SpanTrees"; + } + + /** + * Returns spaces corresponding to the given levels of indentations + */ + private String ind(int levels) { + int indent = levels*STD_INDENT; + StringBuilder sb = new StringBuilder(""); + for (int i = 0 ; i<indent ; i++) { + sb.append(" "); + } + return sb.toString(); + } + + /** + * Returns spaces corresponding to 1 level of indentation + */ + private String ind() { + return ind(1); + } + + private String getter(String field) { + return "get"+upperCaseFirstChar(field); + } + + private String setter(String field) { + return "set"+upperCaseFirstChar(field); + } + + private String className(String field) { + return upperCaseFirstChar(field); + } + + private String toJavaType(DataType dt) { + if (DataType.NONE.equals(dt)) return "void"; + if (DataType.INT.equals(dt)) return "java.lang.Integer"; + if (DataType.FLOAT.equals(dt)) return "java.lang.Float"; + if (DataType.STRING.equals(dt)) return "java.lang.String"; + if (DataType.RAW.equals(dt)) return "java.nio.ByteBuffer"; + if (DataType.LONG.equals(dt)) return "java.lang.Long"; + if (DataType.DOUBLE.equals(dt)) return "java.lang.Double"; + if (DataType.DOCUMENT.equals(dt)) return "com.yahoo.document.Document"; + if (DataType.URI.equals(dt)) return "java.net.URI"; + if (DataType.BYTE.equals(dt)) return "java.lang.Byte"; + if (DataType.TAG.equals(dt)) return "java.lang.String"; + if (dt instanceof StructDataType) return className(dt.getName()); + if (dt instanceof WeightedSetDataType) return "java.util.Map<"+toJavaType(((WeightedSetDataType)dt).getNestedType())+",java.lang.Integer>"; + if (dt instanceof ArrayDataType) return "java.util.List<"+toJavaType(((ArrayDataType)dt).getNestedType())+">"; + if (dt instanceof MapDataType) return "java.util.Map<"+toJavaType(((MapDataType)dt).getKeyType())+","+toJavaType(((MapDataType)dt).getValueType())+">"; + if (dt instanceof AnnotationReferenceDataType) return className(((AnnotationReferenceDataType) dt).getAnnotationType().getName()); + return "byte[]"; + } + + // bit stupid... + private String toJavaReference(DataType dt) { + if (DataType.NONE.equals(dt)) return "com.yahoo.document.DataType.NONE"; + if (DataType.INT.equals(dt)) return "com.yahoo.document.DataType.INT"; + if (DataType.FLOAT.equals(dt)) return "com.yahoo.document.DataType.FLOAT"; + if (DataType.STRING.equals(dt)) return "com.yahoo.document.DataType.STRING"; + if (DataType.RAW.equals(dt)) return "com.yahoo.document.DataType.RAW"; + if (DataType.LONG.equals(dt)) return "com.yahoo.document.DataType.LONG"; + if (DataType.DOUBLE.equals(dt)) return "com.yahoo.document.DataType.DOUBLE"; + if (DataType.DOCUMENT.equals(dt)) return "com.yahoo.document.DataType.DOCUMENT"; + if (DataType.URI.equals(dt)) return "com.yahoo.document.DataType.URI"; + if (DataType.BYTE.equals(dt)) return "com.yahoo.document.DataType.BYTE"; + if (DataType.TAG.equals(dt)) return "com.yahoo.document.DataType.TAG"; + if (dt instanceof StructDataType) return "new com.yahoo.document.StructDataType(\""+dt.getName()+"\")"; + if (dt instanceof WeightedSetDataType) return "new com.yahoo.document.WeightedSetDataType("+toJavaReference(((WeightedSetDataType)dt).getNestedType())+", "+ + ((WeightedSetDataType)dt).createIfNonExistent()+", "+ ((WeightedSetDataType)dt).removeIfZero()+","+dt.getId()+")"; + if (dt instanceof ArrayDataType) return "new com.yahoo.document.ArrayDataType("+toJavaReference(((ArrayDataType)dt).getNestedType())+")"; + if (dt instanceof MapDataType) return "new com.yahoo.document.MapDataType("+toJavaReference(((MapDataType)dt).getKeyType())+", "+ + toJavaReference(((MapDataType)dt).getValueType())+", "+dt.getId()+")"; + // For annotation references and generated types, the references are to the actual objects of the correct types, so most likely this is never needed, + // but there might be scenarios where we want to look up the AnnotationType in the AnnotationTypeRegistry here instead. + if (dt instanceof AnnotationReferenceDataType) return "new com.yahoo.document.annotation.AnnotationReferenceDataType(new com.yahoo.document.annotation.AnnotationType(\""+((AnnotationReferenceDataType)dt).getAnnotationType().getName()+"\"))"; + return "DataType.RAW"; + } + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + execute(this.sdDirectory, this.outputDirectory, this.packageName); + } + + public Map<String, Search> getSearches() { + return searches; + } + + private String upperCaseFirstChar(String s) { + return s.substring(0, 1).toUpperCase()+s.substring(1, s.length()); + } +} diff --git a/vespa-documentgen-plugin/src/main/resources/META-INF/m2e/lifecycle-mapping-metadata.xml b/vespa-documentgen-plugin/src/main/resources/META-INF/m2e/lifecycle-mapping-metadata.xml new file mode 100644 index 00000000000..855f73da7b9 --- /dev/null +++ b/vespa-documentgen-plugin/src/main/resources/META-INF/m2e/lifecycle-mapping-metadata.xml @@ -0,0 +1,18 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <goals> + <goal>document-gen</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + </pluginExecutions> +</lifecycleMappingMetadata> diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java new file mode 100644 index 00000000000..1220521558f --- /dev/null +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa; + +import com.yahoo.document.DataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.Search; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.junit.Test; + +import java.io.File; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class DocumentGenTest { + + @Test + public void testMusic() throws MojoExecutionException, MojoFailureException { + DocumentGenMojo mojo = new DocumentGenMojo(); + mojo.execute(new File("etc/music/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); + Map<String, Search> searches = mojo.getSearches(); + assertEquals(searches.size(),1); + assertEquals(searches.get("music").getDocument("music").getField("title").getDataType(), DataType.STRING); + } + + @Test + public void testComplex() throws MojoFailureException { + DocumentGenMojo mojo = new DocumentGenMojo(); + mojo.execute(new File("etc/complex/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); + Map<String, Search> searches = mojo.getSearches(); + assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT); + assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); + assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType); + assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT); + } + + @Test + public void testLocalApp() throws MojoFailureException { + DocumentGenMojo mojo = new DocumentGenMojo(); + mojo.execute(new File("etc/localapp/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); + Map<String, Search> searches = mojo.getSearches(); + assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT); + assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); + assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType); + assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT); + } + + @Test + public void testEmptyPkgNameForbidden() throws MojoFailureException { + DocumentGenMojo mojo = new DocumentGenMojo(); + try { + mojo.execute(new File("etc/localapp/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), ""); + fail("Didn't throw in empty pkg"); + } catch (IllegalArgumentException e) { + + } + } + +} |