diff options
5 files changed, 146 insertions, 227 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java new file mode 100644 index 00000000000..048d2ccde6a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import aQute.bnd.header.Parameters; +import aQute.bnd.osgi.Domain; +import aQute.bnd.version.VersionRange; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.regex.Pattern; + +/** + * Base class for OSGi bundle validator. Uses BND library for some of the validation. + * + * @author bjorncs + */ +public abstract class AbstractBundleValidator extends Validator { + + protected abstract void validateManifest(DeployState state, JarFile jar, Manifest mf); + protected abstract void validatePomXml(DeployState state, JarFile jar, Document pom); + + @Override + public final void validate(VespaModel model, DeployState state) { + ApplicationPackage app = state.getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(state.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); + try { + state.getDeployLogger() + .log(Level.FINE, String.format("Validating bundle at '%s'", path)); + JarFile jarFile = new JarFile(app.getFileReference(path)); + validateJarFile(state, jarFile); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to validate JAR file '" + path.last() + "'", e); + } + } + } + + final void validateJarFile(DeployState state, JarFile jar) throws IOException { + Manifest manifest = jar.getManifest(); + if (manifest == null) { + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename(jar)); + } + validateManifest(state, jar, manifest); + getPomXmlContent(state.getDeployLogger(), jar) + .ifPresent(pom -> validatePomXml(state, jar, pom)); + } + + protected final String filename(JarFile jarFile) { return Paths.get(jarFile.getName()).getFileName().toString(); } + + protected final void forEachPomXmlElement(Document pom, String xpath, Consumer<Element> consumer) throws XPathExpressionException { + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/" + xpath) + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element element = (Element) dependencies.item(i); + consumer.accept(element); + } + } + + protected final void forEachImportPackage(Manifest mf, BiConsumer<String, VersionRange> consumer) { + Parameters importPackage = Domain.domain(mf).getImportPackage(); + importPackage.forEach((packageName, attrs) -> { + VersionRange versionRange = attrs.getVersion() != null + ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) + : null; + consumer.accept(packageName, versionRange); + }); + } + + protected final void log(DeployState state, Level level, String fmt, Object... args) { + state.getDeployLogger().log(level, String.format(fmt, args)); + } + + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + private Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { + return jar.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + String text = new String(jar.getInputStream(f).readAllBytes()); + return DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(text))); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (SAXException e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", filename(jar))); + return null; + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jar.getName())); + return null; + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index fe5ffb4e544..bc4579b9aaa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -1,28 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import aQute.bnd.header.Parameters; -import aQute.bnd.osgi.Domain; import aQute.bnd.version.VersionRange; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.path.Path; -import com.yahoo.vespa.model.VespaModel; import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.IOException; -import java.io.StringReader; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -30,9 +12,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Consumer; import java.util.function.Predicate; import java.util.jar.Attributes; import java.util.jar.JarFile; @@ -42,44 +22,17 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * A validator for bundles. Uses BND library for some of the validation. + * A validator for bundles. * * @author hmusum * @author bjorncs */ -public class BundleValidator extends Validator { +public class BundleValidator extends AbstractBundleValidator { public BundleValidator() {} @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage app = deployState.getApplicationPackage(); - for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { - Path path = Path.fromString(info.getPathRelativeToAppDir()); - try { - DeployLogger deployLogger = deployState.getDeployLogger(); - deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); - JarFile jarFile = new JarFile(app.getFileReference(path)); - validateJarFile(deployLogger, deployState.isHosted(), jarFile); - } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to validate JAR file '" + path.last() + "'", e); - } - } - } - - void validateJarFile(DeployLogger deployLogger, boolean isHosted, JarFile jarFile) throws IOException { - Manifest manifest = jarFile.getManifest(); - String filename = Paths.get(jarFile.getName()).getFileName().toString(); - if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename); - } - validateManifest(deployLogger, filename, manifest); - getPomXmlContent(deployLogger, jarFile) - .ifPresent(pomXml -> validatePomXml(deployLogger, isHosted, filename, pomXml)); - } - - private void validateManifest(DeployLogger deployLogger, String filename, Manifest mf) { + protected void validateManifest(DeployState state, JarFile jar, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -91,30 +44,26 @@ public class BundleValidator extends Validator { for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + filename + "'"); + "' was not found in manifest in '" + filename(jar) + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + filename + - ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); + log(state, Level.WARNING, + "Deploying snapshot bundle " + filename(jar) + ".\nTo use this bundle, you must include the " + + "qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(deployLogger, filename, mf); + validateImportedPackages(state, jar, mf); } } - private static void validateImportedPackages(DeployLogger deployLogger, String filename, Manifest manifest) { - Domain osgiHeaders = Domain.domain(manifest); - Parameters importPackage = osgiHeaders.getImportPackage(); - Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); - - importPackage.forEach((packageName, attrs) -> { - VersionRange versionRange = attrs.getVersion() != null - ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) - : null; + @Override protected void validatePomXml(DeployState state, JarFile jar, Document pom) {} + private void validateImportedPackages(DeployState state, JarFile jar, Manifest manifest) { + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); + forEachImportPackage(manifest, (packageName, versionRange) -> { for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { if (matcher.test(packageName) @@ -125,107 +74,14 @@ public class BundleValidator extends Validator { } } }); - deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> { - deployLogger.logApplicationPackage(Level.WARNING, - String.format("For JAR file '%s': \n" + - "Manifest imports the following Java packages from '%s': %s. \n" + - "%s", - filename, artifact.name, packagesInUse, artifact.description)); + log(state, Level.WARNING, "For JAR file '%s': \n" + + "Manifest imports the following Java packages from '%s': %s. \n" + + "%s", + filename(jar), artifact.name, packagesInUse, artifact.description); }); } - private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); - - private Optional<String> getPomXmlContent(DeployLogger deployLogger, JarFile jarFile) { - return jarFile.stream() - .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) - .findFirst() - .map(f -> { - try { - return new String(jarFile.getInputStream(f).readAllBytes()); - } catch (IOException e) { - deployLogger.log(Level.INFO, - String.format("Unable to read '%s' from '%s'", f.getName(), jarFile.getName())); - return null; - } - }); - } - - private void validatePomXml(DeployLogger deployLogger, boolean isHosted, String jarFilename, String pomXmlContent) { - if (isHosted) { - try { - Document pom = DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() - .parse(new InputSource(new StringReader(pomXmlContent))); - validateDependencies(deployLogger, jarFilename, pom); - validateRepositories(deployLogger, jarFilename, pom); - } catch (ParserConfigurationException e) { - throw new RuntimeException(e); - } catch (Exception e) { - deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", jarFilename)); - } - } - } - - private static void validateDependencies(DeployLogger deployLogger, String jarFilename, Document pom) throws XPathExpressionException { - forEachPomXmlElement(pom, "dependencies/dependency", dependency -> { - String groupId = dependency.getElementsByTagName("groupId").item(0).getTextContent(); - String artifactId = dependency.getElementsByTagName("artifactId").item(0).getTextContent(); - for (DeprecatedMavenArtifact deprecatedArtifact : DeprecatedMavenArtifact.values()) { - if (groupId.equals(deprecatedArtifact.groupId) && artifactId.equals(deprecatedArtifact.artifactId)) { - deployLogger.logApplicationPackage(Level.WARNING, - String.format( - "The pom.xml of bundle '%s' includes a dependency to the artifact '%s:%s'. \n%s", - jarFilename, groupId, artifactId, deprecatedArtifact.description)); - } - } - }); - } - - private static void validateRepositories(DeployLogger deployLogger, String jarFilename, Document pom) throws XPathExpressionException { - forEachPomXmlElement(pom, "pluginRepositories/pluginRepository", - repository -> validateRepository(deployLogger, jarFilename, "pluginRepositories", repository)); - forEachPomXmlElement(pom, "repositories/repository", - repository -> validateRepository(deployLogger, jarFilename, "repositories", repository)); - } - - private static void validateRepository(DeployLogger deployLogger, String jarFilename, String parentElementName, - Element element) { - String url = element.getElementsByTagName("url").item(0).getTextContent(); - if (url.contains("vespa-maven-libs-release-local")) { - deployLogger.logApplicationPackage(Level.WARNING, - String.format("<%s> in pom.xml of '%s' uses deprecated Maven repository '%s'.\n See announcement.", - parentElementName, jarFilename, url)); - } - } - - private static void forEachPomXmlElement(Document pom, String xpath, Consumer<Element> consumer) throws XPathExpressionException { - NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() - .compile("/project/" + xpath) - .evaluate(pom, XPathConstants.NODESET); - for (int i = 0; i < dependencies.getLength(); i++) { - Element element = (Element) dependencies.item(i); - consumer.accept(element); - } - } - - private enum DeprecatedMavenArtifact { - VESPA_HTTP_CLIENT_EXTENSION("com.yahoo.vespa", "vespa-http-client-extensions", - "This artifact will be removed in Vespa 8. " + - "Programmatic use can be safely removed from system/staging tests. " + - "See internal Vespa 8 release notes for details."); - - final String groupId; - final String artifactId; - final String description; - - DeprecatedMavenArtifact(String groupId, String artifactId, String description) { - this.groupId = groupId; - this.artifactId = artifactId; - this.description = description; - } - } - private enum DeprecatedProvidedBundle { ORG_JSON("org.json:json", "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF deleted file mode 100644 index 1f88a5e6477..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF +++ /dev/null @@ -1,7 +0,0 @@ -Manifest-Version: 1.0 -Created-By: 1.6.0_20 (Apple Inc.) -Bundle-ManifestVersion: 2 -Bundle-Name: mybundle -Bundle-SymbolicName: mybundle -Bundle-Version: 0 - diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml deleted file mode 100644 index 167751f55c4..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.test</groupId> - <artifactId>mybundle</artifactId> - <packaging>container-plugin</packaging> - <version>1.0.0</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client-extensions</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <pluginRepositories> - <pluginRepository> - <id>my-plugin-repository</id> - <url>http://myartifactory:8000/artifactory/vespa-maven-libs-release-local</url> - </pluginRepository> - </pluginRepositories> - <repositories> - <repository> - <id>my-repository</id> - <url>http://myartifactory:8000/artifactory/vespa-maven-libs-release-local</url> - </repository> - </repositories> -</project>
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index eeae7dfe0ee..506a7bbd86a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.DeployState; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -29,7 +29,7 @@ public class BundleValidatorTest { // Valid jar file JarFile ok = createTemporaryJarFile("ok"); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), false, ok); + bundleValidator.validateJarFile(DeployState.createTestState(), ok); // No manifest validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); @@ -39,7 +39,7 @@ public class BundleValidatorTest { try { JarFile jarFile = createTemporaryJarFile(jarName); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), false, jarFile); + bundleValidator.validateJarFile(DeployState.createTestState(), jarFile); assert (false); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), exceptionMessage); @@ -50,47 +50,28 @@ public class BundleValidatorTest { public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); + DeployState state = createDeployState(buffer); JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); - new BundleValidator().validateJarFile(logger, false, jarFile); + new BundleValidator().validateJarFile(state, jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @Test public void outputs_deploy_warning_on_import_of_packages_from_deprecated_artifact() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); + DeployState state = createDeployState(buffer); BundleValidator validator = new BundleValidator(); JarFile jarFile = createTemporaryJarFile("import-warnings"); - validator.validateJarFile(logger, true, jarFile); + validator.validateJarFile(state, jarFile); assertThat(buffer.toString()) .contains("For JAR file 'import-warnings.jar': \n" + "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } - @Test - public void outputs_deploy_warnings_for_pom_xml() throws IOException { - StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); - BundleValidator validator = new BundleValidator(); - JarFile jarFile = createTemporaryJarFile("pom-xml-warnings"); - validator.validateJarFile(logger, true, jarFile); - String output = buffer.toString(); - assertThat(output) - .contains("The pom.xml of bundle 'pom-xml-warnings.jar' includes a dependency to the artifact " + - "'com.yahoo.vespa:vespa-http-client-extensions'. \n" + - "This artifact will be removed in Vespa 8. " + - "Programmatic use can be safely removed from system/staging tests. " + - "See internal Vespa 8 release notes for details.\n"); - assertThat(output) - .contains("\n" + - "<pluginRepositories> in pom.xml of 'pom-xml-warnings.jar' uses deprecated Maven repository " + - "'http://myartifactory:8000/artifactory/vespa-maven-libs-release-local'.\n See announcement."); - assertThat(output) - .contains("\n" + - "<repositories> in pom.xml of 'pom-xml-warnings.jar' uses deprecated Maven repository " + - "'http://myartifactory:8000/artifactory/vespa-maven-libs-release-local'.\n See announcement."); + private DeployState createDeployState(StringBuffer buffer) { + DeployLogger logger = (__, message) -> buffer.append(message).append('\n'); + return DeployState.createTestState(logger); } private JarFile createTemporaryJarFile(String testArtifact) throws IOException { @@ -114,7 +95,4 @@ public class BundleValidatorTest { return new JarFile(jarFile.toFile()); } - private DeployLogger createDeployLogger(StringBuffer buffer) { - return (__, message) -> buffer.append(message).append('\n'); - } } |