Merge pull request #25518 from vespa-engine/bjorncs/slimmer-fat-jars

Don't embed JARs installed in lib/jars

14 files changed, 322 insertions, 72 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6d432994d1b..5463c4215a0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -144,6 +144,7 @@ add_subdirectory(vbench)
 add_subdirectory(vdslib)
 add_subdirectory(vdstestlib)
 add_subdirectory(vespa-3party-bundles)
+add_subdirectory(vespa-3party-jars)
 add_subdirectory(vespa-athenz)
 add_subdirectory(vespabase)
 add_subdirectory(vespaclient)
diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml
index 4911f88b61a..25382f317bf 100644
--- a/bundle-plugin/pom.xml
+++ b/bundle-plugin/pom.xml
@@ -61,6 +61,12 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>${maven-shade-plugin.version}</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
             <groupId>org.ow2.asm</groupId>
             <artifactId>asm</artifactId>
         </dependency>
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java
new file mode 100644
index 00000000000..b71912fc7b0
--- /dev/null
+++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java
@@ -0,0 +1,194 @@
+package com.yahoo.container.plugin.mojo;
+
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.execution.MavenSession;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.Component;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.plugins.shade.DefaultShader;
+import org.apache.maven.plugins.shade.ShadeRequest;
+import org.apache.maven.plugins.shade.filter.SimpleFilter;
+import org.apache.maven.plugins.shade.mojo.ArchiveFilter;
+import org.apache.maven.plugins.shade.relocation.Relocator;
+import org.apache.maven.plugins.shade.resource.ResourceTransformer;
+import org.apache.maven.project.DefaultProjectBuildingRequest;
+import org.apache.maven.project.MavenProject;
+import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder;
+import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException;
+import org.apache.maven.shared.dependency.graph.DependencyNode;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+
+/**
+ * Wrapper around maven-shade-plugin's {@link DefaultShader} for packaging Vespa fat jars for `$VESPA_HOME/lib/jars`.
+ * The produced fat jar will add dependencies which are already installed in lib/jars to manifest's "Class-Path" instead of embedding.
+ *
+ * @author bjorncs
+ */
+@Mojo(name = "assemble-fat-jar", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true)
+public class AssembleFatJarMojo extends AbstractMojo {
+
+    @Parameter(defaultValue = "${session}")
+    public MavenSession session;
+
+    @Parameter(defaultValue = "${project}")
+    public MavenProject project;
+
+    @Component
+    public DependencyGraphBuilder dependencyGraphBuilder;
+
+    @Parameter(defaultValue = "${project.artifactId}-jar-with-dependencies")
+    public String finalName;
+
+    @Parameter(defaultValue = "com.yahoo.vespa:vespa-3party-jars")
+    public String projectDefiningInstalledDependencies;
+
+    @Parameter(defaultValue = "${project.build.directory}")
+    public File outputDirectory;
+
+    @Parameter
+    public String mainClass;
+
+    @Parameter
+    public String[] excludes = new String[] {
+            "META-INF/DEPENDENCIES",
+            "META-INF/LICENSE*",
+            "META-INF/NOTICE*",
+            "META-INF/MANIFEST.MF",
+            "META-INF/*.SF",
+            "META-INF/*.DSA",
+            "META-INF/*.RSA",
+            "about.html",
+            "module-info.class",
+            "license/*",
+            "**/package-info.class",
+            "**/module-info.class",
+    };
+
+    @Override
+    public void execute() throws MojoExecutionException {
+        var installedDependencies = resolveThirdPartyArtifactsInstalledInVespaHomeLibJars();
+        var projectDependencies = new TreeSet<>(project.getArtifacts());
+        File outputFile = outputFile();
+        var archiveFilter = new ArchiveFilter() {
+            @Override public String getArtifact() { return null; }
+            @Override public Set<String> getIncludes() { return Set.of(); }
+            @Override public Set<String> getExcludes() { return Set.of(excludes); }
+            @Override public boolean getExcludeDefaults() { return true; }
+        };
+        var jarsToShade = projectDependencies.stream()
+                .filter(d -> !installedDependencies.contains(d) && !d.getType().equals("pom") && d.getScope().equals("compile"))
+                .map(Artifact::getFile)
+                .collect(Collectors.toCollection(TreeSet::new));
+        jarsToShade.add(project.getArtifact().getFile());
+        try {
+            var classpath = generateClasspath(installedDependencies, projectDependencies);
+            var req = new ShadeRequest();
+            req.setJars(jarsToShade);
+            req.setUberJar(outputFile);
+            req.setFilters(List.of(new SimpleFilter(jarsToShade, archiveFilter)));
+            req.setRelocators(List.of());
+            req.setResourceTransformers(List.of(new ManifestWriter(classpath, mainClass)));
+            req.setShadeSourcesContent(false);
+            new DefaultShader().shade(req);
+            Files.copy(outputFile.toPath(), finalFile().toPath(), StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new MojoExecutionException(e);
+        }
+    }
+
+    private String generateClasspath(Set<Artifact> installedDependencies, SortedSet<Artifact> projectDependencies) {
+        return projectDependencies.stream()
+                .filter(installedDependencies::contains)
+                .map(d -> d.getFile().getName())
+                .collect(Collectors.joining(" "));
+    }
+
+    private File outputFile() {
+        var a = project.getArtifact();
+        var name = project.getArtifactId() + "-" + a.getVersion() + "-shaded." + a.getArtifactHandler().getExtension();
+        return new File(outputDirectory, name);
+    }
+
+    private File finalFile() {
+        var name = finalName + "." + project.getArtifact().getArtifactHandler().getExtension();
+        return new File(outputDirectory, name);
+    }
+
+    private SortedSet<Artifact> resolveThirdPartyArtifactsInstalledInVespaHomeLibJars() throws MojoExecutionException {
+        try {
+            var installedDepsProject = projectDefiningInstalledDependencies.split(":");
+            var project = session.getAllProjects().stream()
+                    .filter(p -> p.getGroupId().equals(installedDepsProject[0]) && p.getArtifactId().equals(installedDepsProject[1]))
+                    .findAny().orElseThrow(() -> new IllegalStateException("Cannot find " + projectDefiningInstalledDependencies));
+            var req = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest());
+            req.setProject(project);
+            var root = dependencyGraphBuilder.buildDependencyGraph(req, null);
+            return getAllRecursive(root);
+        } catch (DependencyGraphBuilderException e) {
+            throw new MojoExecutionException(e);
+        }
+    }
+
+    private static SortedSet<Artifact> getAllRecursive(DependencyNode node) {
+        SortedSet<Artifact> children = new TreeSet<>();
+        if (node.getChildren() != null) {
+            for (DependencyNode dep : node.getChildren()) {
+                children.add(dep.getArtifact());
+                children.addAll(getAllRecursive(dep));
+            }
+        }
+        return children;
+    }
+
+    private static class ManifestWriter implements ResourceTransformer {
+
+        private final String classpath;
+        private final String mainclass;
+
+        ManifestWriter(String classpath, String mainclass) {
+            this.classpath = classpath;
+            this.mainclass = mainclass;
+        }
+
+        @Override public boolean canTransformResource(String resource) { return false; }
+        @SuppressWarnings("deprecation") @Override public void processResource(String resource, InputStream is, List<Relocator> relocators) {}
+        @Override public boolean hasTransformedResource() { return true; }
+
+        @Override
+        public void modifyOutputStream(JarOutputStream os) throws IOException {
+            var manifest = new Manifest();
+            var attributes = manifest.getMainAttributes();
+            attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");
+            attributes.putValue("Class-Path", classpath);
+            attributes.putValue("Created-By", "vespa container maven plugin");
+            if (mainclass != null) attributes.putValue("Main-Class", mainclass);
+            var entry = new JarEntry(JarFile.MANIFEST_NAME);
+            entry.setTime(System.currentTimeMillis());
+            os.putNextEntry(entry);
+            var baos = new ByteArrayOutputStream();
+            manifest.write(baos);
+            os.write(baos.toByteArray());
+            os.flush();
+        }
+    }
+
+}
diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml
index aeccead46ce..51fba61563f 100644
--- a/config-proxy/pom.xml
+++ b/config-proxy/pom.xml
@@ -81,20 +81,20 @@
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <configuration>
-          <finalName>${project.artifactId}-jar-with-dependencies</finalName>
-          <createDependencyReducedPom>false</createDependencyReducedPom>
-        </configuration>
+        <artifactId>maven-install-plugin</artifactId>
+        <configuration><skip>true</skip></configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
         <executions>
           <execution>
             <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
+            <goals><goal>assemble-fat-jar</goal></goals>
           </execution>
         </executions>
-      </plugin>      <plugin>
+      </plugin>
+      <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
@@ -102,13 +102,6 @@
           <forkMode>once</forkMode>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-install-plugin</artifactId>
-        <configuration>
-          <updateReleaseInfo>true</updateReleaseInfo>
-        </configuration>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/logserver/pom.xml b/logserver/pom.xml
index 68ed459f315..e3ef730bb36 100644
--- a/logserver/pom.xml
+++ b/logserver/pom.xml
@@ -60,30 +60,20 @@
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <configuration>
-          <finalName>${project.artifactId}-jar-with-dependencies</finalName>
-          <createDependencyReducedPom>false</createDependencyReducedPom>
-          <transformers>
-            <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-              <mainClass>com.yahoo.logserver.Server</mainClass>
-            </transformer>
-          </transformers>
-        </configuration>
+        <artifactId>maven-install-plugin</artifactId>
+        <configuration><skip>true</skip></configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
         <executions>
           <execution>
             <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
+            <goals><goal>assemble-fat-jar</goal></goals>
           </execution>
         </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-install-plugin</artifactId>
         <configuration>
-          <updateReleaseInfo>true</updateReleaseInfo>
+          <mainClass>com.yahoo.logserver.Server</mainClass>
         </configuration>
       </plugin>
       <plugin>
diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt
index cab4de7817d..bc93ff58719 100644
--- a/maven-plugins/allowed-maven-dependencies.txt
+++ b/maven-plugins/allowed-maven-dependencies.txt
@@ -14,6 +14,7 @@ com.google.j2objc:j2objc-annotations:1.1
 commons-io:commons-io:2.11.0
 javax.annotation:javax.annotation-api:1.2
 javax.inject:javax.inject:1
+org.apache.commons:commons-collections4:4.2
 org.apache.commons:commons-compress:1.22
 org.apache.commons:commons-lang3:3.12.0
 org.apache.maven:maven-archiver:3.6.0
@@ -29,11 +30,15 @@ org.apache.maven:maven-settings:3.8.6
 org.apache.maven:maven-settings-builder:3.8.6
 org.apache.maven.enforcer:enforcer-api:3.0.0
 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4
+org.apache.maven.plugins:maven-shade-plugin:3.4.1
 org.apache.maven.resolver:maven-resolver-api:1.6.3
 org.apache.maven.resolver:maven-resolver-impl:1.6.3
 org.apache.maven.resolver:maven-resolver-spi:1.6.3
 org.apache.maven.resolver:maven-resolver-util:1.6.3
+org.apache.maven.shared:maven-artifact-transfer:0.13.1
+org.apache.maven.shared:maven-common-artifact-filters:3.1.0
 org.apache.maven.shared:maven-dependency-tree:3.1.1
+org.apache.maven.shared:maven-dependency-tree:3.2.0
 org.apache.maven.shared:maven-shared-utils:3.3.4
 org.codehaus.plexus:plexus-archiver:4.4.0
 org.codehaus.plexus:plexus-cipher:2.0
@@ -43,15 +48,22 @@ org.codehaus.plexus:plexus-interpolation:1.26
 org.codehaus.plexus:plexus-io:3.4.0
 org.codehaus.plexus:plexus-sec-dispatcher:2.0
 org.codehaus.plexus:plexus-utils:3.3.1
+org.eclipse.aether:aether-api:1.0.0.v20140518
+org.eclipse.aether:aether-util:1.0.0.v20140518
 org.eclipse.aether:aether-util:1.1.0
 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5
 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5
 org.iq80.snappy:snappy:0.4
+org.jdom:jdom2:2.0.6.1
 org.ow2.asm:asm:9.3
+org.ow2.asm:asm-analysis:9.3
+org.ow2.asm:asm-commons:9.3
+org.ow2.asm:asm-tree:9.3
 org.slf4j:slf4j-api:1.7.32
 org.slf4j:slf4j-simple:1.7.22
 org.tukaani:xz:1.9
 org.twdata.maven:mojo-executor:2.3.0
+org.vafer:jdependency:2.8.0
 
 #[test-only]
 # Contains dependencies that are used exclusively in 'test' scope
diff --git a/pom.xml b/pom.xml
index 56dc6ad81ca..ced21ab7b77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
         <module>vespa-osgi-testrunner</module>
         <module>vespa-testrunner-components</module>
         <module>vespa-3party-bundles</module>
+        <module>vespa-3party-jars</module>
         <module>vespajlib</module>
         <module>vespalog</module>
         <module>zkfacade</module>
diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml
index a41c72b0211..ad31c35bb8c 100644
--- a/predicate-search/pom.xml
+++ b/predicate-search/pom.xml
@@ -72,19 +72,12 @@
                 <artifactId>abi-check-plugin</artifactId>
             </plugin>
             <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>make-assembly</id>
                         <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
+                        <goals><goal>assemble-fat-jar</goal></goals>
                     </execution>
                 </executions>
             </plugin>
diff --git a/vespa-3party-jars/CMakeLists.txt b/vespa-3party-jars/CMakeLists.txt
new file mode 100644
index 00000000000..436d9aec39a
--- /dev/null
+++ b/vespa-3party-jars/CMakeLists.txt
@@ -0,0 +1,2 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_jar_dependencies(vespa-3party-jars)
diff --git a/vespa-3party-jars/OWNERS b/vespa-3party-jars/OWNERS
new file mode 100644
index 00000000000..569bf1cc3a1
--- /dev/null
+++ b/vespa-3party-jars/OWNERS
@@ -0,0 +1 @@
+bjorncs
diff --git a/vespa-3party-jars/README b/vespa-3party-jars/README
new file mode 100644
index 00000000000..e24bd86e20e
--- /dev/null
+++ b/vespa-3party-jars/README
@@ -0,0 +1 @@
+3rd party Java libraries that are installed to VESPA_HOME/lib/jars/
diff --git a/vespa-3party-jars/pom.xml b/vespa-3party-jars/pom.xml
new file mode 100644
index 00000000000..4c269e723b6
--- /dev/null
+++ b/vespa-3party-jars/pom.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>8-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>vespa-3party-jars</artifactId>
+    <version>8-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <description>
+        List 3rd party Java libraries that should be installed to lib/jars.
+        These libraries are not embedded in fat jars produced by bundle-plugin's 'assemble-fat-jar' goal.
+    </description>
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespa-3party-bundles</artifactId>
+            <version>${project.version}</version>
+            <type>pom</type>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk18on</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <!-- Explicit for IntelliJ to detect correct language level from parent -->
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <overWriteReleases>false</overWriteReleases>
+                    <overWriteSnapshots>false</overWriteSnapshots>
+                    <excludeTransitive>false</excludeTransitive>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml
index 99ca84ccce0..a31d3c834dd 100644
--- a/vespaclient-java/pom.xml
+++ b/vespaclient-java/pom.xml
@@ -11,6 +11,7 @@
     </parent>
     <artifactId>vespaclient-java</artifactId>
     <version>8-SNAPSHOT</version>
+    <packaging>jar</packaging>
     <dependencies>
         <dependency>
             <!-- Due to com.yahoo.search.query.profile.DumpTool ... -->
@@ -94,17 +95,16 @@
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <configuration>
-                    <finalName>${project.artifactId}-jar-with-dependencies</finalName>
-                    <createDependencyReducedPom>false</createDependencyReducedPom>
-                </configuration>
+                <artifactId>maven-install-plugin</artifactId>
+                <configuration><skip>true</skip></configuration>
+            </plugin>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
                 <executions>
                     <execution>
                         <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
+                        <goals><goal>assemble-fat-jar</goal></goals>
                     </execution>
                 </executions>
             </plugin>
diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml
index ae3f5b3bfa1..7bbbf33a62f 100644
--- a/zookeeper-command-line-client/pom.xml
+++ b/zookeeper-command-line-client/pom.xml
@@ -63,27 +63,22 @@
   <build>
     <plugins>
       <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-install-plugin</artifactId>
+        <configuration><skip>true</skip></configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals><goal>assemble-fat-jar</goal></goals>
+          </execution>
+        </executions>
         <configuration>
-           <archive>
-             <manifest>
-               <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass>
-             </manifest>
-           </archive>
-           <descriptorRefs>
-             <descriptorRef>jar-with-dependencies</descriptorRef>
-           </descriptorRefs>
-         </configuration>
-         <executions>
-           <execution>
-             <id>make-assembly</id>
-             <phase>package</phase>
-             <!-- append to the packaging phase. -->
-             <goals>
-               <goal>single</goal>
-             </goals>
-           </execution>
-         </executions>
+          <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass>
+        </configuration>
       </plugin>
     </plugins>
   </build>