diff options
6 files changed, 346 insertions, 40 deletions
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index a0215f3106d..4197b350df1 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -56,6 +56,13 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> <version>3.0.0</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-enforcer-extensions</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> <executions> <execution> <!-- To allow running 'mvn enforcer:enforce' from the command line --> @@ -65,12 +72,8 @@ </goals> <configuration> <rules> - <bannedDependencies> - <excludes> - <!-- Only allow explicitly listed dependencies --> - <exclude>*:*:*:*:*:*</exclude> - </excludes> - <includes> + <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependencies"> + <allowed> <!-- MUST BE KEPT IN SYNC WITH container-dependencies-enforcer pom --> <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> @@ -102,7 +105,6 @@ <!-- Vespa provided dependencies --> <include>com.yahoo.vespa:annotations:*:jar:provided</include> - <include>com.yahoo.vespa:chain:*:jar:provided</include> <include>com.yahoo.vespa:component:*:jar:provided</include> <include>com.yahoo.vespa:config-bundle:*:jar:provided</include> <include>com.yahoo.vespa:config-lib:*:jar:provided</include> @@ -131,7 +133,6 @@ <include>com.yahoo.vespa:messagebus:*:jar:provided</include> <include>com.yahoo.vespa:model-evaluation:*:jar:provided</include> <include>com.yahoo.vespa:predicate-search-core:*:jar:provided</include> - <include>com.yahoo.vespa:processing:*:jar:provided</include> <include>com.yahoo.vespa:provided-dependencies:*:jar:provided</include> <include>com.yahoo.vespa:searchcore:*:jar:provided</include> <include>com.yahoo.vespa:searchlib:*:jar:provided</include> @@ -164,43 +165,25 @@ <include>com.yahoo.vespa:storage:*:jar:test</include> <include>com.yahoo.vespa:tenant-cd-api:*:jar:test</include> <include>com.yahoo.vespa:tenant-cd-commons:*:jar:test</include> - <include>com.yahoo.vespa:vespa-athenz:*:jar:test</include> <include>com.yahoo.vespa:vespa-feed-client:*:jar:test</include> <include>com.yahoo.vespa:vespa-feed-client-api:*:jar:test</include> - <include>com.yahoo.vespa:vespa-3party-bundles:*:pom:test</include> <include>com.yahoo.vespa:vespaclient-core:*:jar:test</include> <include>com.yahoo.vespa:vsm:*:jar:test</include> <!-- 3rd party test dependencies --> - <include>com.amazonaws:aws-java-sdk-core:1.11.974:jar:test</include> - <include>com.auth0:java-jwt:3.10.0:jar:test</include> - <include>com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7:jar:test</include> <include>com.fasterxml.jackson.dataformat:jackson-dataformat-xml:[${jackson2.version}]:jar:test</include> <include>com.fasterxml.woodstox:woodstox-core:5.0.3:jar:test</include> <include>com.google.protobuf:protobuf-java:3.7.0:jar:test</include> <include>com.ibm.icu:icu4j:57.1:jar:test</include> - <include>com.intellij:annotations:12.0:jar:test</include> <include>com.microsoft.onnxruntime:onnxruntime:[${onnxruntime.version}]:jar:test</include> <include>com.thaiopensource:jing:20091111:jar:test</include> - <include>com.yahoo.athenz:athenz-auth-core:[${athenz.version}]:jar:test</include> - <include>com.yahoo.athenz:athenz-client-common:[${athenz.version}]:jar:test</include> - <include>com.yahoo.athenz:athenz-zms-core:[${athenz.version}]:jar:test</include> - <include>com.yahoo.athenz:athenz-zpe-java-client:[${athenz.version}]:jar:test</include> - <include>com.yahoo.athenz:athenz-zts-core:[${athenz.version}]:jar:test</include> - <include>com.yahoo.rdl:rdl-java:1.5.2:jar:test</include> - <include>commons-beanutils:commons-beanutils-core:1.8.0:jar:test</include> - <include>commons-beanutils:commons-beanutils:1.7.0:jar:test</include> <include>commons-codec:commons-codec:1.11:jar:test</include> - <include>commons-digester:commons-digester:1.8:jar:test</include> <include>io.airlift:aircompressor:0.17:jar:test</include> <include>io.airlift:airline:0.7:jar:test</include> <include>io.prometheus:simpleclient:0.6.0:jar:test</include> <include>io.prometheus:simpleclient_common:0.6.0:jar:test</include> - <include>joda-time:joda-time:2.8.1:jar:test</include> <include>junit:junit:4.13.2:jar:test</include> - <include>net.arnx:jsonic:1.2.11:jar:test</include> <include>net.java.dev.jna:jna:5.11.0:jar:test</include> - <include>org.abego.treelayout:org.abego.treelayout.core:1.0.1:jar:test</include> <include>org.antlr:antlr-runtime:3.5.2:jar:test</include> <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include> <include>org.apache.commons:commons-exec:1.3:jar:test</include> @@ -216,7 +199,8 @@ <include>org.apache.opennlp:opennlp-tools:1.9.3:jar:test</include> <include>org.apiguardian:apiguardian-api:1.1.0:jar:test</include> <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:test</include> - <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:test</include> <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> + <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:test</include> + <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> <include>org.eclipse.jetty.alpn:alpn-api:[${jetty-alpn.version}]:jar:test</include> <include>org.eclipse.jetty.http2:http2-common:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty.http2:http2-hpack:[${jetty.version}]:jar:test</include> @@ -242,13 +226,11 @@ <include>org.junit.platform:junit-platform-commons:[${junit5.platform.version}]:jar:test</include> <include>org.junit.platform:junit-platform-engine:[${junit5.platform.version}]:jar:test</include> <include>org.junit.vintage:junit-vintage-engine:[${junit5.version}]:jar:test</include> - <include>org.kohsuke:libpam4j:1.11:jar:test</include> <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:test</include> <include>org.opentest4j:opentest4j:1.2.0:jar:test</include> - <include>software.amazon.ion:ion-java:1.0.2:jar:test</include> <include>xerces:xercesImpl:2.12.1:jar:test</include> - </includes> - </bannedDependencies> + </allowed> + </enforceDependencies> </rules> <fail>true</fail> </configuration> diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index bbd816d349d..b8d661de111 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -41,6 +41,13 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-enforcer-extensions</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> <executions> <execution> <!-- To allow running 'mvn enforcer:enforce' from the command line --> @@ -50,13 +57,9 @@ </goals> <configuration> <rules> - <bannedDependencies> - <excludes> - <!-- Only allow explicitly listed deps in provided and compile scope --> - <exclude>*:*:*:jar:provided:*</exclude> - <exclude>*:*:*:jar:compile:*</exclude> - </excludes> - <includes> + <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependencies"> + <allowed> + <include>*:*:*:jar:test</include> <include>com.yahoo.vespa</include> <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> @@ -85,8 +88,8 @@ <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> - </includes> - </bannedDependencies> + </allowed> + </enforceDependencies> </rules> <fail>true</fail> </configuration> diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index 6f8b44540e2..4ad063a39cc 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -22,6 +22,7 @@ <module>../config-class-plugin</module> <module>../configgen</module> <module>../vespa-application-maven-plugin</module> + <module>../vespa-enforcer-extensions</module> </modules> <properties> diff --git a/vespa-enforcer-extensions/pom.xml b/vespa-enforcer-extensions/pom.xml new file mode 100644 index 00000000000..96df2c2ac33 --- /dev/null +++ b/vespa-enforcer-extensions/pom.xml @@ -0,0 +1,56 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>vespa-enforcer-extensions</artifactId> + <packaging>jar</packaging> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>org.apache.maven.enforcer</groupId> + <artifactId>enforcer-api</artifactId> + <version>3.0.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven.shared</groupId> + <artifactId>maven-dependency-tree</artifactId> + <version>3.1.1</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project> diff --git a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java new file mode 100644 index 00000000000..154ba9db790 --- /dev/null +++ b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependencies.java @@ -0,0 +1,151 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.maven.plugin.enforcer; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.versioning.ArtifactVersion; +import org.apache.maven.artifact.versioning.DefaultArtifactVersion; +import org.apache.maven.artifact.versioning.InvalidVersionSpecificationException; +import org.apache.maven.artifact.versioning.VersionRange; +import org.apache.maven.enforcer.rule.api.EnforcerRule; +import org.apache.maven.enforcer.rule.api.EnforcerRuleException; +import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.project.DefaultProjectBuildingRequest; +import org.apache.maven.project.MavenProject; +import org.apache.maven.project.ProjectBuildingRequest; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException; +import org.apache.maven.shared.dependency.graph.DependencyNode; +import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException; +import org.codehaus.plexus.component.repository.exception.ComponentLookupException; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Pattern; + +/** + * Enforces that all expected dependencies are present. + * Fails by default for rules that do not match any dependencies. + * Similar to the built-in 'bannedDependencies' rule in maven-enforcer-plugin. + * + * @author bjorncs + */ +public class EnforceDependencies implements EnforcerRule { + + private List<String> allowedDependencies = List.of(); + private boolean failOnUnmatched = true; + + @Override + public void execute(EnforcerRuleHelper helper) throws EnforcerRuleException { + validateDependencies(getAllDependencies(helper), Set.copyOf(allowedDependencies), failOnUnmatched); + helper.getLog().info("The 'enforceDependencies' validation completed successfully"); + } + + static void validateDependencies(Set<Artifact> dependencies, Set<String> allowedRules, boolean failOnUnmatched) + throws EnforcerRuleException { + SortedSet<Artifact> unmatchedArtifacts = new TreeSet<>(); + Set<String> matchedRules = new HashSet<>(); + for (Artifact dependency : dependencies) { + boolean matches = false; + for (String rule : allowedRules) { + if (matches(dependency, rule)){ + matchedRules.add(rule); + matches = true; + break; + } + } + if (!matches) { + unmatchedArtifacts.add(dependency); + } + } + SortedSet<String> unmatchedRules = new TreeSet<>(allowedRules); + unmatchedRules.removeAll(matchedRules); + if (!unmatchedArtifacts.isEmpty() || (failOnUnmatched && !unmatchedRules.isEmpty())) { + StringBuilder errorMessage = new StringBuilder("Vespa dependency enforcer failed:\n"); + if (!unmatchedArtifacts.isEmpty()) { + errorMessage.append("Dependencies not matching any rule:\n"); + unmatchedArtifacts.forEach(a -> errorMessage.append(" - ").append(a.toString()).append('\n')); + } + if (failOnUnmatched && !unmatchedRules.isEmpty()) { + errorMessage.append("Rules not matching any dependency:\n"); + unmatchedRules.forEach(p -> errorMessage.append(" - ").append(p).append('\n')); + } + throw new EnforcerRuleException(errorMessage.toString()); + } + } + + private static Set<Artifact> getAllDependencies(EnforcerRuleHelper helper) throws EnforcerRuleException { + try { + MavenProject project = (MavenProject) helper.evaluate("${project}"); + MavenSession session = (MavenSession) helper.evaluate("${session}"); + DependencyGraphBuilder graphBuilder = helper.getComponent(DependencyGraphBuilder.class); + ProjectBuildingRequest buildingRequest = + new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); + buildingRequest.setProject(project); + DependencyNode root = graphBuilder.buildDependencyGraph(buildingRequest, null); + return getAllRecursive(root); + } catch (ExpressionEvaluationException | DependencyGraphBuilderException | ComponentLookupException e) { + throw new EnforcerRuleException(e.getMessage(), e); + } + } + + private static Set<Artifact> getAllRecursive(DependencyNode node) { + Set<Artifact> children = new LinkedHashSet<>(); + if (node.getChildren() != null) { + for (DependencyNode dep : node.getChildren()) { + children.add(dep.getArtifact()); + children.addAll(getAllRecursive(dep)); + } + } + return children; + } + + // Similar rule matching to bannedDependencies + private static boolean matches(Artifact dependency, String rule) throws EnforcerRuleException { + String[] segments = rule.split(":"); + if (segments.length < 1 || segments.length > 6) throw new EnforcerRuleException("Invalid rule: " + rule); + if (!segmentMatches(dependency.getGroupId(), segments[0])) return false; + if (segments.length > 1 && !segmentMatches(dependency.getArtifactId(), segments[1])) return false; + if (segments.length > 2 && !versionMatches(dependency.getVersion(), segments[2])) return false; + if (segments.length > 3 && !segmentMatches(dependency.getType(), segments[3])) return false; + if (segments.length > 4 && !segmentMatches(dependency.getScope(), segments[4])) return false; + if (segments.length > 5 && dependency.hasClassifier() && !segmentMatches(dependency.getClassifier(), segments[5])) + return false; + return true; + } + + private static boolean segmentMatches(String value, String segmentPattern) { + String regex = segmentPattern + .replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.') + .replace("[", "\\[").replace("]", "\\]").replace("(", "\\(").replace(")", "\\)"); + return Pattern.matches(regex, value); + } + + private static boolean versionMatches(String rawVersion, String segmentPattern) throws EnforcerRuleException { + try { + if (segmentMatches(rawVersion, segmentPattern)) return true; + VersionRange allowedRange = VersionRange.createFromVersionSpec(segmentPattern); + ArtifactVersion version = new DefaultArtifactVersion(rawVersion); + ArtifactVersion recommended = allowedRange.getRecommendedVersion(); + if (recommended == null) return allowedRange.containsVersion(version); + return recommended.compareTo(version) <= 0; + } catch (InvalidVersionSpecificationException e) { + throw new EnforcerRuleException(e.getMessage(), e); + } + } + + public void setAllowed(List<String> allowed) { this.allowedDependencies = allowed; } + public List<String> getAllowed() { return allowedDependencies; } + public void setFailOnUnmatchedRule(boolean enabled) { this.failOnUnmatched = enabled; } + public boolean isFailOnUnmatchedRule() { return failOnUnmatched; } + + // Mark rule as not cachable + @Override public boolean isCacheable() { return false; } + @Override public boolean isResultValid(EnforcerRule enforcerRule) { return false; } + @Override public String getCacheId() { return ""; } + +} diff --git a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java new file mode 100644 index 00000000000..0dcbe595121 --- /dev/null +++ b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.maven.plugin.enforcer;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.DefaultArtifact; +import org.apache.maven.artifact.handler.DefaultArtifactHandler; +import org.apache.maven.enforcer.rule.api.EnforcerRuleException; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author bjorncs + */ +class EnforceDependenciesTest { + + @Test + void succeeds_when_all_dependencies_and_rules_match() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "container-core", "8.0.0", "provided"), + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + Set<String> rules = Set.of( + "com.yahoo.vespa:container-core:*:jar:provided", + "com.yahoo.vespa:*:*:jar:test"); + assertDoesNotThrow(() -> EnforceDependencies.validateDependencies(dependencies, rules, true)); + } + + @Test + void fails_on_unmatched_dependency() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "container-core", "8.0.0", "provided"), + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + Set<String> rules = Set.of("com.yahoo.vespa:*:*:jar:test"); + EnforcerRuleException exception = assertThrows( + EnforcerRuleException.class, + () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); + String expectedErrorMessage = + """ + Vespa dependency enforcer failed: + Dependencies not matching any rule: + - com.yahoo.vespa:container-core:jar:8.0.0:provided + """; + assertEquals(expectedErrorMessage, exception.getMessage()); + } + + @Test + void fails_on_unmatched_rule() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + Set<String> rules = Set.of( + "com.yahoo.vespa:container-core:*:jar:provided", + "com.yahoo.vespa:*:*:jar:test"); + EnforcerRuleException exception = assertThrows( + EnforcerRuleException.class, + () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); + String expectedErrorMessage = + """ + Vespa dependency enforcer failed: + Rules not matching any dependency: + - com.yahoo.vespa:container-core:*:jar:provided + """; + assertEquals(expectedErrorMessage, exception.getMessage()); + } + + @Test + void fails_on_version_mismatch() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + Set<String> rules = Set.of( + "com.yahoo.vespa:testutils:[7.0.0]:jar:test"); + EnforcerRuleException exception = assertThrows( + EnforcerRuleException.class, + () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); + String expectedErrorMessage = + """ + Vespa dependency enforcer failed: + Dependencies not matching any rule: + - com.yahoo.vespa:testutils:jar:8.0.0:test + Rules not matching any dependency: + - com.yahoo.vespa:testutils:[7.0.0]:jar:test + """; + assertEquals(expectedErrorMessage, exception.getMessage()); + } + + @Test + void fails_on_scope_mismatch() { + Set<Artifact> dependencies = Set.of( + artifact("com.yahoo.vespa", "testutils", "8.0.0", "test")); + Set<String> rules = Set.of( + "com.yahoo.vespa:testutils:8.0.0:jar:provided"); + EnforcerRuleException exception = assertThrows( + EnforcerRuleException.class, + () -> EnforceDependencies.validateDependencies(dependencies, rules, true)); + String expectedErrorMessage = + """ + Vespa dependency enforcer failed: + Dependencies not matching any rule: + - com.yahoo.vespa:testutils:jar:8.0.0:test + Rules not matching any dependency: + - com.yahoo.vespa:testutils:8.0.0:jar:provided + """; + assertEquals(expectedErrorMessage, exception.getMessage()); + } + + private static Artifact artifact(String groupId, String artifactId, String version, String scope) { + return new DefaultArtifact( + groupId, artifactId, version, scope, "jar", /*classifier*/null, new DefaultArtifactHandler("jar")); + } + +}
\ No newline at end of file |