diff options
author | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-01-10 17:35:00 +0100 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@yahooinc.com> | 2023-01-11 16:50:51 +0100 |
commit | 43a5ab35b4dfd0f4c5dd2bdbbc241345938d3142 (patch) | |
tree | b6e8244e36b6cd9621b925ff946a6fe64de749c3 | |
parent | c6191deb7c5fd1501c1d07b3cae0c8e8b9486434 (diff) |
Don't embed JARs installed in lib/jars
Define installed JARs in vespa-3party-jars. Add bundle-plugin goal
wrapping maven-shade-plugin's DefaultShader that excludes installed JARs
and lists them in manifest's Class-Path instead.
-rw-r--r-- | CMakeLists.txt | 1 | ||||
-rw-r--r-- | bundle-plugin/pom.xml | 6 | ||||
-rw-r--r-- | bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java | 194 | ||||
-rw-r--r-- | config-proxy/pom.xml | 25 | ||||
-rw-r--r-- | logserver/pom.xml | 26 | ||||
-rw-r--r-- | maven-plugins/allowed-maven-dependencies.txt | 12 | ||||
-rw-r--r-- | pom.xml | 1 | ||||
-rw-r--r-- | predicate-search/pom.xml | 13 | ||||
-rw-r--r-- | vespa-3party-jars/CMakeLists.txt | 2 | ||||
-rw-r--r-- | vespa-3party-jars/OWNERS | 1 | ||||
-rw-r--r-- | vespa-3party-jars/README | 1 | ||||
-rw-r--r-- | vespa-3party-jars/pom.xml | 61 | ||||
-rw-r--r-- | vespaclient-java/pom.xml | 16 | ||||
-rw-r--r-- | zookeeper-command-line-client/pom.xml | 35 |
14 files changed, 322 insertions, 72 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d432994d1b..5463c4215a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,7 @@ add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) add_subdirectory(vespa-3party-bundles) +add_subdirectory(vespa-3party-jars) add_subdirectory(vespa-athenz) add_subdirectory(vespabase) add_subdirectory(vespaclient) diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 4911f88b61a..25382f317bf 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -61,6 +61,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>${maven-shade-plugin.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </dependency> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java new file mode 100644 index 00000000000..b71912fc7b0 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java @@ -0,0 +1,194 @@ +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.plugins.shade.DefaultShader; +import org.apache.maven.plugins.shade.ShadeRequest; +import org.apache.maven.plugins.shade.filter.SimpleFilter; +import org.apache.maven.plugins.shade.mojo.ArchiveFilter; +import org.apache.maven.plugins.shade.relocation.Relocator; +import org.apache.maven.plugins.shade.resource.ResourceTransformer; +import org.apache.maven.project.DefaultProjectBuildingRequest; +import org.apache.maven.project.MavenProject; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException; +import org.apache.maven.shared.dependency.graph.DependencyNode; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Collectors; + +/** + * Wrapper around maven-shade-plugin's {@link DefaultShader} for packaging Vespa fat jars for `$VESPA_HOME/lib/jars`. + * The produced fat jar will add dependencies which are already installed in lib/jars to manifest's "Class-Path" instead of embedding. + * + * @author bjorncs + */ +@Mojo(name = "assemble-fat-jar", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true) +public class AssembleFatJarMojo extends AbstractMojo { + + @Parameter(defaultValue = "${session}") + public MavenSession session; + + @Parameter(defaultValue = "${project}") + public MavenProject project; + + @Component + public DependencyGraphBuilder dependencyGraphBuilder; + + @Parameter(defaultValue = "${project.artifactId}-jar-with-dependencies") + public String finalName; + + @Parameter(defaultValue = "com.yahoo.vespa:vespa-3party-jars") + public String projectDefiningInstalledDependencies; + + @Parameter(defaultValue = "${project.build.directory}") + public File outputDirectory; + + @Parameter + public String mainClass; + + @Parameter + public String[] excludes = new String[] { + "META-INF/DEPENDENCIES", + "META-INF/LICENSE*", + "META-INF/NOTICE*", + "META-INF/MANIFEST.MF", + "META-INF/*.SF", + "META-INF/*.DSA", + "META-INF/*.RSA", + "about.html", + "module-info.class", + "license/*", + "**/package-info.class", + "**/module-info.class", + }; + + @Override + public void execute() throws MojoExecutionException { + var installedDependencies = resolveThirdPartyArtifactsInstalledInVespaHomeLibJars(); + var projectDependencies = new TreeSet<>(project.getArtifacts()); + File outputFile = outputFile(); + var archiveFilter = new ArchiveFilter() { + @Override public String getArtifact() { return null; } + @Override public Set<String> getIncludes() { return Set.of(); } + @Override public Set<String> getExcludes() { return Set.of(excludes); } + @Override public boolean getExcludeDefaults() { return true; } + }; + var jarsToShade = projectDependencies.stream() + .filter(d -> !installedDependencies.contains(d) && !d.getType().equals("pom") && d.getScope().equals("compile")) + .map(Artifact::getFile) + .collect(Collectors.toCollection(TreeSet::new)); + jarsToShade.add(project.getArtifact().getFile()); + try { + var classpath = generateClasspath(installedDependencies, projectDependencies); + var req = new ShadeRequest(); + req.setJars(jarsToShade); + req.setUberJar(outputFile); + req.setFilters(List.of(new SimpleFilter(jarsToShade, archiveFilter))); + req.setRelocators(List.of()); + req.setResourceTransformers(List.of(new ManifestWriter(classpath, mainClass))); + req.setShadeSourcesContent(false); + new DefaultShader().shade(req); + Files.copy(outputFile.toPath(), finalFile().toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new MojoExecutionException(e); + } + } + + private String generateClasspath(Set<Artifact> installedDependencies, SortedSet<Artifact> projectDependencies) { + return projectDependencies.stream() + .filter(installedDependencies::contains) + .map(d -> d.getFile().getName()) + .collect(Collectors.joining(" ")); + } + + private File outputFile() { + var a = project.getArtifact(); + var name = project.getArtifactId() + "-" + a.getVersion() + "-shaded." + a.getArtifactHandler().getExtension(); + return new File(outputDirectory, name); + } + + private File finalFile() { + var name = finalName + "." + project.getArtifact().getArtifactHandler().getExtension(); + return new File(outputDirectory, name); + } + + private SortedSet<Artifact> resolveThirdPartyArtifactsInstalledInVespaHomeLibJars() throws MojoExecutionException { + try { + var installedDepsProject = projectDefiningInstalledDependencies.split(":"); + var project = session.getAllProjects().stream() + .filter(p -> p.getGroupId().equals(installedDepsProject[0]) && p.getArtifactId().equals(installedDepsProject[1])) + .findAny().orElseThrow(() -> new IllegalStateException("Cannot find " + projectDefiningInstalledDependencies)); + var req = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); + req.setProject(project); + var root = dependencyGraphBuilder.buildDependencyGraph(req, null); + return getAllRecursive(root); + } catch (DependencyGraphBuilderException e) { + throw new MojoExecutionException(e); + } + } + + private static SortedSet<Artifact> getAllRecursive(DependencyNode node) { + SortedSet<Artifact> children = new TreeSet<>(); + if (node.getChildren() != null) { + for (DependencyNode dep : node.getChildren()) { + children.add(dep.getArtifact()); + children.addAll(getAllRecursive(dep)); + } + } + return children; + } + + private static class ManifestWriter implements ResourceTransformer { + + private final String classpath; + private final String mainclass; + + ManifestWriter(String classpath, String mainclass) { + this.classpath = classpath; + this.mainclass = mainclass; + } + + @Override public boolean canTransformResource(String resource) { return false; } + @SuppressWarnings("deprecation") @Override public void processResource(String resource, InputStream is, List<Relocator> relocators) {} + @Override public boolean hasTransformedResource() { return true; } + + @Override + public void modifyOutputStream(JarOutputStream os) throws IOException { + var manifest = new Manifest(); + var attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + attributes.putValue("Class-Path", classpath); + attributes.putValue("Created-By", "vespa container maven plugin"); + if (mainclass != null) attributes.putValue("Main-Class", mainclass); + var entry = new JarEntry(JarFile.MANIFEST_NAME); + entry.setTime(System.currentTimeMillis()); + os.putNextEntry(entry); + var baos = new ByteArrayOutputStream(); + manifest.write(baos); + os.write(baos.toByteArray()); + os.flush(); + } + } + +} diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index aeccead46ce..51fba61563f 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -81,20 +81,20 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> - </plugin> <plugin> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> @@ -102,13 +102,6 @@ <forkMode>once</forkMode> </configuration> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/logserver/pom.xml b/logserver/pom.xml index 68ed459f315..e3ef730bb36 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -60,30 +60,20 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.yahoo.logserver.Server</mainClass> - </transformer> - </transformers> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> <configuration> - <updateReleaseInfo>true</updateReleaseInfo> + <mainClass>com.yahoo.logserver.Server</mainClass> </configuration> </plugin> <plugin> diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index cab4de7817d..bc93ff58719 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -14,6 +14,7 @@ com.google.j2objc:j2objc-annotations:1.1 commons-io:commons-io:2.11.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 +org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 @@ -29,11 +30,15 @@ org.apache.maven:maven-settings:3.8.6 org.apache.maven:maven-settings-builder:3.8.6 org.apache.maven.enforcer:enforcer-api:3.0.0 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 +org.apache.maven.plugins:maven-shade-plugin:3.4.1 org.apache.maven.resolver:maven-resolver-api:1.6.3 org.apache.maven.resolver:maven-resolver-impl:1.6.3 org.apache.maven.resolver:maven-resolver-spi:1.6.3 org.apache.maven.resolver:maven-resolver-util:1.6.3 +org.apache.maven.shared:maven-artifact-transfer:0.13.1 +org.apache.maven.shared:maven-common-artifact-filters:3.1.0 org.apache.maven.shared:maven-dependency-tree:3.1.1 +org.apache.maven.shared:maven-dependency-tree:3.2.0 org.apache.maven.shared:maven-shared-utils:3.3.4 org.codehaus.plexus:plexus-archiver:4.4.0 org.codehaus.plexus:plexus-cipher:2.0 @@ -43,15 +48,22 @@ org.codehaus.plexus:plexus-interpolation:1.26 org.codehaus.plexus:plexus-io:3.4.0 org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:3.3.1 +org.eclipse.aether:aether-api:1.0.0.v20140518 +org.eclipse.aether:aether-util:1.0.0.v20140518 org.eclipse.aether:aether-util:1.1.0 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 org.iq80.snappy:snappy:0.4 +org.jdom:jdom2:2.0.6.1 org.ow2.asm:asm:9.3 +org.ow2.asm:asm-analysis:9.3 +org.ow2.asm:asm-commons:9.3 +org.ow2.asm:asm-tree:9.3 org.slf4j:slf4j-api:1.7.32 org.slf4j:slf4j-simple:1.7.22 org.tukaani:xz:1.9 org.twdata.maven:mojo-executor:2.3.0 +org.vafer:jdependency:2.8.0 #[test-only] # Contains dependencies that are used exclusively in 'test' scope @@ -137,6 +137,7 @@ <module>vespa-osgi-testrunner</module> <module>vespa-testrunner-components</module> <module>vespa-3party-bundles</module> + <module>vespa-3party-jars</module> <module>vespajlib</module> <module>vespalog</module> <module>zkfacade</module> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index a41c72b0211..ad31c35bb8c 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -72,19 +72,12 @@ <artifactId>abi-check-plugin</artifactId> </plugin> <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> - <id>make-assembly</id> <phase>package</phase> - <goals> - <goal>single</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> </plugin> diff --git a/vespa-3party-jars/CMakeLists.txt b/vespa-3party-jars/CMakeLists.txt new file mode 100644 index 00000000000..436d9aec39a --- /dev/null +++ b/vespa-3party-jars/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_jar_dependencies(vespa-3party-jars) diff --git a/vespa-3party-jars/OWNERS b/vespa-3party-jars/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/vespa-3party-jars/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/vespa-3party-jars/README b/vespa-3party-jars/README new file mode 100644 index 00000000000..e24bd86e20e --- /dev/null +++ b/vespa-3party-jars/README @@ -0,0 +1 @@ +3rd party Java libraries that are installed to VESPA_HOME/lib/jars/ diff --git a/vespa-3party-jars/pom.xml b/vespa-3party-jars/pom.xml new file mode 100644 index 00000000000..4c269e723b6 --- /dev/null +++ b/vespa-3party-jars/pom.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-3party-jars</artifactId> + <version>8-SNAPSHOT</version> + <packaging>pom</packaging> + <description> + List 3rd party Java libraries that should be installed to lib/jars. + These libraries are not embedded in fat jars produced by bundle-plugin's 'assemble-fat-jar' goal. + </description> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-3party-bundles</artifactId> + <version>${project.version}</version> + <type>pom</type> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk18on</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <!-- Explicit for IntelliJ to detect correct language level from parent --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + </execution> + </executions> + <configuration> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <excludeTransitive>false</excludeTransitive> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index 99ca84ccce0..a31d3c834dd 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -11,6 +11,7 @@ </parent> <artifactId>vespaclient-java</artifactId> <version>8-SNAPSHOT</version> + <packaging>jar</packaging> <dependencies> <dependency> <!-- Due to com.yahoo.search.query.profile.DumpTool ... --> @@ -94,17 +95,16 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> </plugin> diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index ae3f5b3bfa1..7bbbf33a62f 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -63,27 +63,22 @@ <build> <plugins> <plugin> - <artifactId>maven-assembly-plugin</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals><goal>assemble-fat-jar</goal></goals> + </execution> + </executions> <configuration> - <archive> - <manifest> - <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <!-- append to the packaging phase. --> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> + <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass> + </configuration> </plugin> </plugins> </build> |