diff options
26 files changed, 365 insertions, 118 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d432994d1b..5463c4215a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,7 @@ add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) add_subdirectory(vespa-3party-bundles) +add_subdirectory(vespa-3party-jars) add_subdirectory(vespa-athenz) add_subdirectory(vespabase) add_subdirectory(vespaclient) diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 4911f88b61a..25382f317bf 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -61,6 +61,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>${maven-shade-plugin.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </dependency> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java new file mode 100644 index 00000000000..b71912fc7b0 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleFatJarMojo.java @@ -0,0 +1,194 @@ +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.plugins.shade.DefaultShader; +import org.apache.maven.plugins.shade.ShadeRequest; +import org.apache.maven.plugins.shade.filter.SimpleFilter; +import org.apache.maven.plugins.shade.mojo.ArchiveFilter; +import org.apache.maven.plugins.shade.relocation.Relocator; +import org.apache.maven.plugins.shade.resource.ResourceTransformer; +import org.apache.maven.project.DefaultProjectBuildingRequest; +import org.apache.maven.project.MavenProject; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException; +import org.apache.maven.shared.dependency.graph.DependencyNode; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Collectors; + +/** + * Wrapper around maven-shade-plugin's {@link DefaultShader} for packaging Vespa fat jars for `$VESPA_HOME/lib/jars`. + * The produced fat jar will add dependencies which are already installed in lib/jars to manifest's "Class-Path" instead of embedding. + * + * @author bjorncs + */ +@Mojo(name = "assemble-fat-jar", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true) +public class AssembleFatJarMojo extends AbstractMojo { + + @Parameter(defaultValue = "${session}") + public MavenSession session; + + @Parameter(defaultValue = "${project}") + public MavenProject project; + + @Component + public DependencyGraphBuilder dependencyGraphBuilder; + + @Parameter(defaultValue = "${project.artifactId}-jar-with-dependencies") + public String finalName; + + @Parameter(defaultValue = "com.yahoo.vespa:vespa-3party-jars") + public String projectDefiningInstalledDependencies; + + @Parameter(defaultValue = "${project.build.directory}") + public File outputDirectory; + + @Parameter + public String mainClass; + + @Parameter + public String[] excludes = new String[] { + "META-INF/DEPENDENCIES", + "META-INF/LICENSE*", + "META-INF/NOTICE*", + "META-INF/MANIFEST.MF", + "META-INF/*.SF", + "META-INF/*.DSA", + "META-INF/*.RSA", + "about.html", + "module-info.class", + "license/*", + "**/package-info.class", + "**/module-info.class", + }; + + @Override + public void execute() throws MojoExecutionException { + var installedDependencies = resolveThirdPartyArtifactsInstalledInVespaHomeLibJars(); + var projectDependencies = new TreeSet<>(project.getArtifacts()); + File outputFile = outputFile(); + var archiveFilter = new ArchiveFilter() { + @Override public String getArtifact() { return null; } + @Override public Set<String> getIncludes() { return Set.of(); } + @Override public Set<String> getExcludes() { return Set.of(excludes); } + @Override public boolean getExcludeDefaults() { return true; } + }; + var jarsToShade = projectDependencies.stream() + .filter(d -> !installedDependencies.contains(d) && !d.getType().equals("pom") && d.getScope().equals("compile")) + .map(Artifact::getFile) + .collect(Collectors.toCollection(TreeSet::new)); + jarsToShade.add(project.getArtifact().getFile()); + try { + var classpath = generateClasspath(installedDependencies, projectDependencies); + var req = new ShadeRequest(); + req.setJars(jarsToShade); + req.setUberJar(outputFile); + req.setFilters(List.of(new SimpleFilter(jarsToShade, archiveFilter))); + req.setRelocators(List.of()); + req.setResourceTransformers(List.of(new ManifestWriter(classpath, mainClass))); + req.setShadeSourcesContent(false); + new DefaultShader().shade(req); + Files.copy(outputFile.toPath(), finalFile().toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new MojoExecutionException(e); + } + } + + private String generateClasspath(Set<Artifact> installedDependencies, SortedSet<Artifact> projectDependencies) { + return projectDependencies.stream() + .filter(installedDependencies::contains) + .map(d -> d.getFile().getName()) + .collect(Collectors.joining(" ")); + } + + private File outputFile() { + var a = project.getArtifact(); + var name = project.getArtifactId() + "-" + a.getVersion() + "-shaded." + a.getArtifactHandler().getExtension(); + return new File(outputDirectory, name); + } + + private File finalFile() { + var name = finalName + "." + project.getArtifact().getArtifactHandler().getExtension(); + return new File(outputDirectory, name); + } + + private SortedSet<Artifact> resolveThirdPartyArtifactsInstalledInVespaHomeLibJars() throws MojoExecutionException { + try { + var installedDepsProject = projectDefiningInstalledDependencies.split(":"); + var project = session.getAllProjects().stream() + .filter(p -> p.getGroupId().equals(installedDepsProject[0]) && p.getArtifactId().equals(installedDepsProject[1])) + .findAny().orElseThrow(() -> new IllegalStateException("Cannot find " + projectDefiningInstalledDependencies)); + var req = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); + req.setProject(project); + var root = dependencyGraphBuilder.buildDependencyGraph(req, null); + return getAllRecursive(root); + } catch (DependencyGraphBuilderException e) { + throw new MojoExecutionException(e); + } + } + + private static SortedSet<Artifact> getAllRecursive(DependencyNode node) { + SortedSet<Artifact> children = new TreeSet<>(); + if (node.getChildren() != null) { + for (DependencyNode dep : node.getChildren()) { + children.add(dep.getArtifact()); + children.addAll(getAllRecursive(dep)); + } + } + return children; + } + + private static class ManifestWriter implements ResourceTransformer { + + private final String classpath; + private final String mainclass; + + ManifestWriter(String classpath, String mainclass) { + this.classpath = classpath; + this.mainclass = mainclass; + } + + @Override public boolean canTransformResource(String resource) { return false; } + @SuppressWarnings("deprecation") @Override public void processResource(String resource, InputStream is, List<Relocator> relocators) {} + @Override public boolean hasTransformedResource() { return true; } + + @Override + public void modifyOutputStream(JarOutputStream os) throws IOException { + var manifest = new Manifest(); + var attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + attributes.putValue("Class-Path", classpath); + attributes.putValue("Created-By", "vespa container maven plugin"); + if (mainclass != null) attributes.putValue("Main-Class", mainclass); + var entry = new JarEntry(JarFile.MANIFEST_NAME); + entry.setTime(System.currentTimeMillis()); + os.putNextEntry(entry); + var baos = new ByteArrayOutputStream(); + manifest.write(baos); + os.write(baos.toByteArray()); + os.flush(); + } + } + +} diff --git a/client/go/jvm/env.go b/client/go/jvm/env.go index dbc957e3ccb..d2f0b2e6616 100644 --- a/client/go/jvm/env.go +++ b/client/go/jvm/env.go @@ -19,7 +19,7 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { vlt := fmt.Sprintf("file:%s/vespa.log", lvd) lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) lcf := fmt.Sprintf("%s/%s.logcontrol", lcd, c.ServiceName()) - dlp := fmt.Sprintf("%s/lib64", vespaHome) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) opts.fixSpec.FixDir(lvd) opts.fixSpec.FixDir(lcd) ps.Setenv(envvars.VESPA_LOG_TARGET, vlt) diff --git a/client/go/prog/common_env.go b/client/go/prog/common_env.go index a970895bf84..150a4aaa38f 100644 --- a/client/go/prog/common_env.go +++ b/client/go/prog/common_env.go @@ -15,7 +15,7 @@ func (spec *Spec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/client/go/script-utils/startcbinary/common_env.go b/client/go/script-utils/startcbinary/common_env.go index 812f182a757..eadb0aafb1b 100644 --- a/client/go/script-utils/startcbinary/common_env.go +++ b/client/go/script-utils/startcbinary/common_env.go @@ -26,7 +26,7 @@ func (spec *ProgSpec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 3a320d5b800..db28ca0e172 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -137,9 +137,9 @@ public class VespaMetricSet { metrics.add(new Metric("serverNumOpenConnections.average")); metrics.add(new Metric("serverNumOpenConnections.max")); metrics.add(new Metric("serverNumOpenConnections.last")); - metrics.add(new Metric("serverNumConnections.average")); - metrics.add(new Metric("serverNumConnections.max")); - metrics.add(new Metric("serverNumConnections.last")); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.average())); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.max())); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.last())); metrics.add(new Metric("serverBytesReceived.sum")); metrics.add(new Metric("serverBytesReceived.count")); diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index aeccead46ce..51fba61563f 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -81,20 +81,20 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> - </plugin> <plugin> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> @@ -102,13 +102,6 @@ <forkMode>once</forkMode> </configuration> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index 8cfa1cf8b7b..ed2d044c38c 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -19,6 +19,8 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Metric; import java.util.logging.Level; + +import com.yahoo.metrics.ContainerMetrics; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; @@ -31,7 +33,7 @@ import com.yahoo.vespa.defaults.Defaults; */ public final class VipStatusHandler extends ThreadedHttpRequestHandler { - private static final String NUM_REQUESTS_METRIC = "jdisc.http.requests.status"; + private static final String NUM_REQUESTS_METRIC = ContainerMetrics.JDISC_HTTP_REQUESTS_STATUS.baseName(); private final boolean accessDisk; private final File statusFile; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index e16e4b51959..8cf5d19ee4e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -32,7 +32,7 @@ class MetricDefinitions { static final String NUM_BYTES_RECEIVED = "serverBytesReceived"; static final String NUM_BYTES_SENT = "serverBytesSent"; - static final String NUM_CONNECTIONS = "serverNumConnections"; + static final String NUM_CONNECTIONS = ContainerMetrics.SERVER_NUM_CONNECTIONS.baseName(); /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ static final String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java index ba160873c4f..50371598d08 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java @@ -10,37 +10,28 @@ public enum ContainerMetrics { HTTP_STATUS_3XX("http.status.3xx", Unit.RESPONSE, "Number of responses with a 3xx status"), HTTP_STATUS_4XX("http.status.4xx", Unit.RESPONSE, "Number of responses with a 4xx status"), HTTP_STATUS_5XX("http.status.5xx", Unit.RESPONSE, "Number of responses with a 5xx status"), + JDISC_GC_MS("jdisc.gc.ms", Unit.MILLISECOND, "Time spent in garbage collection"), - JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS("jdisc.thread_pool.unhandled_exceptions", Unit.THREAD, - "Number of exceptions thrown by tasks"), - JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY("jdisc.thread_pool.work_queue.capacity", Unit.THREAD, - "Capacity of the task queue"), - JDISC_THREAD_POOL_WORK_QUEUE_SIZE("jdisc.thread_pool.work_queue.size", Unit.THREAD, - "Size of the task queue"), - JDISC_THREAD_POOL_REJECTED_TASKS("jdisc.thread_pool.rejected_tasks", Unit.THREAD, - "Number of tasks rejected by the thread pool"), - JDISC_THREAD_POOL_SIZE("jdisc.thread_pool.size", Unit.THREAD, - "Size of the thread pool"), - JDISC_THREAD_POOL_MAX_ALLOWED_SIZE("jdisc.thread_pool.max_allowed_size", Unit.THREAD, - "The maximum allowed number of threads in the pool"), - JDISC_THREAD_POOL_ACTIVE_THREADS("jdisc.thread_pool.active_threads", Unit.THREAD, - "Number of threads that are active"), - - JETTY_THREADPOOL_MAX_THREADS("jdisc.http.jetty.threadpool.thread.max", Unit.THREAD, - "Configured maximum number of threads"), - JETTY_THREADPOOL_MIN_THREADS("jdisc.http.jetty.threadpool.thread.min", Unit.THREAD, - "Configured minimum number of threads"), - JETTY_THREADPOOL_RESERVED_THREADS("jdisc.http.jetty.threadpool.thread.reserved", Unit.THREAD, - "Configured number of reserved threads or -1 for heuristic"), - JETTY_THREADPOOL_BUSY_THREADS("jdisc.http.jetty.threadpool.thread.busy", Unit.THREAD, - "Number of threads executing internal and transient jobs"), - JETTY_THREADPOOL_IDLE_THREADS("jdisc.http.jetty.threadpool.thread.idle", Unit.THREAD, - "Number of idle threads"), - JETTY_THREADPOOL_TOTAL_THREADS("jdisc.http.jetty.threadpool.thread.total", Unit.THREAD, - "Current number of threads"), - JETTY_THREADPOOL_QUEUE_SIZE("jdisc.http.jetty.threadpool.queue.size", Unit.THREAD, - "Current size of the job queue"); + JDISC_HTTP_REQUESTS_STATUS("jdisc.http.requests.status", Unit.REQUEST, "Number of requests to the built-in status handler"), + + JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS("jdisc.thread_pool.unhandled_exceptions", Unit.THREAD, "Number of exceptions thrown by tasks"), + JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY("jdisc.thread_pool.work_queue.capacity", Unit.THREAD, "Capacity of the task queue"), + JDISC_THREAD_POOL_WORK_QUEUE_SIZE("jdisc.thread_pool.work_queue.size", Unit.THREAD, "Size of the task queue"), + JDISC_THREAD_POOL_REJECTED_TASKS("jdisc.thread_pool.rejected_tasks", Unit.THREAD, "Number of tasks rejected by the thread pool"), + JDISC_THREAD_POOL_SIZE("jdisc.thread_pool.size", Unit.THREAD, "Size of the thread pool"), + JDISC_THREAD_POOL_MAX_ALLOWED_SIZE("jdisc.thread_pool.max_allowed_size", Unit.THREAD, "The maximum allowed number of threads in the pool"), + JDISC_THREAD_POOL_ACTIVE_THREADS("jdisc.thread_pool.active_threads", Unit.THREAD, "Number of threads that are active"), + + JETTY_THREADPOOL_MAX_THREADS("jdisc.http.jetty.threadpool.thread.max", Unit.THREAD, "Configured maximum number of threads"), + JETTY_THREADPOOL_MIN_THREADS("jdisc.http.jetty.threadpool.thread.min", Unit.THREAD, "Configured minimum number of threads"), + JETTY_THREADPOOL_RESERVED_THREADS("jdisc.http.jetty.threadpool.thread.reserved", Unit.THREAD, "Configured number of reserved threads or -1 for heuristic"), + JETTY_THREADPOOL_BUSY_THREADS("jdisc.http.jetty.threadpool.thread.busy", Unit.THREAD, "Number of threads executing internal and transient jobs"), + JETTY_THREADPOOL_IDLE_THREADS("jdisc.http.jetty.threadpool.thread.idle", Unit.THREAD, "Number of idle threads"), + JETTY_THREADPOOL_TOTAL_THREADS("jdisc.http.jetty.threadpool.thread.total", Unit.THREAD, "Current number of threads"), + JETTY_THREADPOOL_QUEUE_SIZE("jdisc.http.jetty.threadpool.queue.size", Unit.THREAD, "Current size of the job queue"), + + SERVER_NUM_CONNECTIONS("serverNumConnections", Unit.CONNECTION, "The total number of connections opened"); private final String name; private final Unit unit; diff --git a/container-core/src/main/java/com/yahoo/metrics/Unit.java b/container-core/src/main/java/com/yahoo/metrics/Unit.java index d4143d2d8b9..836dab29f67 100644 --- a/container-core/src/main/java/com/yahoo/metrics/Unit.java +++ b/container-core/src/main/java/com/yahoo/metrics/Unit.java @@ -6,6 +6,7 @@ package com.yahoo.metrics; public enum Unit { BYTE(BaseUnit.BYTE), + CONNECTION(BaseUnit.CONNECTION), DOCUMENT(BaseUnit.DOCUMENT), DOCUMENT_PER_SECOND(BaseUnit.DOCUMENT, BaseUnit.SECOND), FRACTION(BaseUnit.FRACTION), @@ -15,6 +16,7 @@ public enum Unit { OPERATION_PER_SECOND(BaseUnit.OPERATION, BaseUnit.SECOND), QUERY(BaseUnit.QUERY), QUERY_PER_SECOND(BaseUnit.QUERY, BaseUnit.SECOND), + REQUEST(BaseUnit.REQUEST), RESPONSE(BaseUnit.RESPONSE), RESPONSE_PER_SECOND(BaseUnit.RESPONSE, BaseUnit.SECOND), SECOND(BaseUnit.SECOND), @@ -45,15 +47,17 @@ public enum Unit { unit.shortName + "/" + perUnit.shortName; } - public enum BaseUnit { + private enum BaseUnit { BYTE("byte"), + CONNECTION("connection"), DOCUMENT("document"), FRACTION("fraction"), HIT("hit"), MILLISECOND("millisecond", "ms"), OPERATION("operation"), QUERY("query"), + REQUEST("request"), RESPONSE("response"), SECOND("second", "s"), THREAD("thread"); diff --git a/logserver/pom.xml b/logserver/pom.xml index 68ed459f315..e3ef730bb36 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -60,30 +60,20 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.yahoo.logserver.Server</mainClass> - </transformer> - </transformers> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> <configuration> - <updateReleaseInfo>true</updateReleaseInfo> + <mainClass>com.yahoo.logserver.Server</mainClass> </configuration> </plugin> <plugin> diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index cab4de7817d..bc93ff58719 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -14,6 +14,7 @@ com.google.j2objc:j2objc-annotations:1.1 commons-io:commons-io:2.11.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 +org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 @@ -29,11 +30,15 @@ org.apache.maven:maven-settings:3.8.6 org.apache.maven:maven-settings-builder:3.8.6 org.apache.maven.enforcer:enforcer-api:3.0.0 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 +org.apache.maven.plugins:maven-shade-plugin:3.4.1 org.apache.maven.resolver:maven-resolver-api:1.6.3 org.apache.maven.resolver:maven-resolver-impl:1.6.3 org.apache.maven.resolver:maven-resolver-spi:1.6.3 org.apache.maven.resolver:maven-resolver-util:1.6.3 +org.apache.maven.shared:maven-artifact-transfer:0.13.1 +org.apache.maven.shared:maven-common-artifact-filters:3.1.0 org.apache.maven.shared:maven-dependency-tree:3.1.1 +org.apache.maven.shared:maven-dependency-tree:3.2.0 org.apache.maven.shared:maven-shared-utils:3.3.4 org.codehaus.plexus:plexus-archiver:4.4.0 org.codehaus.plexus:plexus-cipher:2.0 @@ -43,15 +48,22 @@ org.codehaus.plexus:plexus-interpolation:1.26 org.codehaus.plexus:plexus-io:3.4.0 org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:3.3.1 +org.eclipse.aether:aether-api:1.0.0.v20140518 +org.eclipse.aether:aether-util:1.0.0.v20140518 org.eclipse.aether:aether-util:1.1.0 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 org.iq80.snappy:snappy:0.4 +org.jdom:jdom2:2.0.6.1 org.ow2.asm:asm:9.3 +org.ow2.asm:asm-analysis:9.3 +org.ow2.asm:asm-commons:9.3 +org.ow2.asm:asm-tree:9.3 org.slf4j:slf4j-api:1.7.32 org.slf4j:slf4j-simple:1.7.22 org.tukaani:xz:1.9 org.twdata.maven:mojo-executor:2.3.0 +org.vafer:jdependency:2.8.0 #[test-only] # Contains dependencies that are used exclusively in 'test' scope @@ -137,6 +137,7 @@ <module>vespa-osgi-testrunner</module> <module>vespa-testrunner-components</module> <module>vespa-3party-bundles</module> + <module>vespa-3party-jars</module> <module>vespajlib</module> <module>vespalog</module> <module>zkfacade</module> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index a41c72b0211..ad31c35bb8c 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -72,19 +72,12 @@ <artifactId>abi-check-plugin</artifactId> </plugin> <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> - <id>make-assembly</id> <phase>package</phase> - <goals> - <goal>single</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> </plugin> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 584335803a8..d00342d0e51 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -976,8 +976,8 @@ TEST_F("Populates address space usage", DenseTensorAttributeHnswIndex) const auto& all = usage.get_all(); EXPECT_EQUAL(3u, all.size()); EXPECT_EQUAL(1u, all.count("tensor-store")); - EXPECT_EQUAL(1u, all.count("hnsw-node-store")); - EXPECT_EQUAL(1u, all.count("hnsw-link-store")); + EXPECT_EQUAL(1u, all.count("hnsw-levels-store")); + EXPECT_EQUAL(1u, all.count("hnsw-links-store")); } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp index c8f2f93832a..244e01a3874 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -19,7 +19,7 @@ const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; const vespalib::string AddressSpaceComponents::tensor_store = "tensor-store"; const vespalib::string AddressSpaceComponents::shared_string_repo = "shared-string-repo"; -const vespalib::string AddressSpaceComponents::hnsw_node_store = "hnsw-node-store"; -const vespalib::string AddressSpaceComponents::hnsw_link_store = "hnsw-link-store"; +const vespalib::string AddressSpaceComponents::hnsw_levels_store = "hnsw-levels-store"; +const vespalib::string AddressSpaceComponents::hnsw_links_store = "hnsw-links-store"; } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h index 3d409ece973..a8a41a024f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -18,8 +18,8 @@ public: static const vespalib::string multi_value; static const vespalib::string tensor_store; static const vespalib::string shared_string_repo; - static const vespalib::string hnsw_node_store; - static const vespalib::string hnsw_link_store; + static const vespalib::string hnsw_levels_store; + static const vespalib::string hnsw_links_store; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index aec86e95a98..abec443e13e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -726,8 +726,8 @@ template <HnswIndexType type> void HnswIndex<type>::populate_address_space_usage(search::AddressSpaceUsage& usage) const { - usage.set(AddressSpaceComponents::hnsw_node_store, _graph.levels_store.addressSpaceUsage()); - usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links_store.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_levels_store, _graph.levels_store.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_links_store, _graph.links_store.addressSpaceUsage()); } template <HnswIndexType type> diff --git a/vespa-3party-jars/CMakeLists.txt b/vespa-3party-jars/CMakeLists.txt new file mode 100644 index 00000000000..436d9aec39a --- /dev/null +++ b/vespa-3party-jars/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_jar_dependencies(vespa-3party-jars) diff --git a/vespa-3party-jars/OWNERS b/vespa-3party-jars/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/vespa-3party-jars/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/vespa-3party-jars/README b/vespa-3party-jars/README new file mode 100644 index 00000000000..e24bd86e20e --- /dev/null +++ b/vespa-3party-jars/README @@ -0,0 +1 @@ +3rd party Java libraries that are installed to VESPA_HOME/lib/jars/ diff --git a/vespa-3party-jars/pom.xml b/vespa-3party-jars/pom.xml new file mode 100644 index 00000000000..4c269e723b6 --- /dev/null +++ b/vespa-3party-jars/pom.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-3party-jars</artifactId> + <version>8-SNAPSHOT</version> + <packaging>pom</packaging> + <description> + List 3rd party Java libraries that should be installed to lib/jars. + These libraries are not embedded in fat jars produced by bundle-plugin's 'assemble-fat-jar' goal. + </description> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-3party-bundles</artifactId> + <version>${project.version}</version> + <type>pom</type> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk18on</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <!-- Explicit for IntelliJ to detect correct language level from parent --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + </execution> + </executions> + <configuration> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <excludeTransitive>false</excludeTransitive> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index 99ca84ccce0..a31d3c834dd 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -11,6 +11,7 @@ </parent> <artifactId>vespaclient-java</artifactId> <version>8-SNAPSHOT</version> + <packaging>jar</packaging> <dependencies> <dependency> <!-- Due to com.yahoo.search.query.profile.DumpTool ... --> @@ -94,17 +95,16 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <executions> <execution> <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> + <goals><goal>assemble-fat-jar</goal></goals> </execution> </executions> </plugin> diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index ae3f5b3bfa1..7bbbf33a62f 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -63,27 +63,22 @@ <build> <plugins> <plugin> - <artifactId>maven-assembly-plugin</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration><skip>true</skip></configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals><goal>assemble-fat-jar</goal></goals> + </execution> + </executions> <configuration> - <archive> - <manifest> - <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <!-- append to the packaging phase. --> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> + <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass> + </configuration> </plugin> </plugins> </build> |