diff options
author | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2022-01-13 15:09:40 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-13 15:09:40 +0100 |
commit | aa0f7a0d495a53dbd60895f241454e9f762ac19d (patch) | |
tree | ec0ba8f90cd04c95ba5019367f9063e1df696d5c /config-model | |
parent | 9fac4b67e6fcdccbfc26cc8991ca1fa9b6ef76d7 (diff) | |
parent | 1018ff48b1bc5f463f434d69b3b1cfae1a81ac49 (diff) |
Merge pull request #20794 from vespa-engine/bjorncs/pom-file-validation
Bjorncs/pom file validation
Diffstat (limited to 'config-model')
22 files changed, 971 insertions, 39 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index b6b9190fedf..87a84911d3e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -10,8 +10,18 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.vespa.model.VespaModel; - +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -19,12 +29,15 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * A validator for bundles. Uses BND library for some of the validation. @@ -40,29 +53,31 @@ public class BundleValidator extends Validator { public void validate(VespaModel model, DeployState deployState) { ApplicationPackage app = deployState.getApplicationPackage(); for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); try { - Path path = Path.fromString(info.getPathRelativeToAppDir()); DeployLogger deployLogger = deployState.getDeployLogger(); deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); JarFile jarFile = new JarFile(app.getFileReference(path)); validateJarFile(deployLogger, jarFile); } catch (IOException e) { throw new IllegalArgumentException( - "Failed to validate JAR file '" + info.getPathRelativeToAppDir() + "'", e); + "Failed to validate JAR file '" + path.last() + "'", e); } } } void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException { Manifest manifest = jarFile.getManifest(); - String jarPath = jarFile.getName(); + String filename = Paths.get(jarFile.getName()).getFileName().toString(); if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarPath); + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename); } - validateManifest(deployLogger, jarPath, manifest); + validateManifest(deployLogger, filename, manifest); + getPomXmlContent(deployLogger, jarFile) + .ifPresent(pomXml -> validatePomXml(deployLogger, filename, pomXml)); } - void validateManifest(DeployLogger deployLogger, String jarPath, Manifest mf) { + private void validateManifest(DeployLogger deployLogger, String filename, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -74,35 +89,37 @@ public class BundleValidator extends Validator { for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + jarPath + "'"); + "' was not found in manifest in '" + filename + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarPath + + deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + filename + ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(deployLogger, jarPath, mf); + validateImportedPackages(deployLogger, filename, mf); } } - private static void validateImportedPackages(DeployLogger deployLogger, String jarPath, Manifest manifest) { + private static void validateImportedPackages(DeployLogger deployLogger, String filename, Manifest manifest) { Domain osgiHeaders = Domain.domain(manifest); Parameters importPackage = osgiHeaders.getImportPackage(); - Map<DeprecatedArtifact, List<String>> deprecatedPackagesInUse = new HashMap<>(); + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); importPackage.forEach((packageName, attrs) -> { VersionRange versionRange = attrs.getVersion() != null ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) : null; - for (DeprecatedArtifact deprecatedArtifact : DeprecatedArtifact.values()) { - if (deprecatedArtifact.javaPackages.contains(packageName) - && (versionRange == null || deprecatedArtifact.versionDiscriminator.test(versionRange))) { - deprecatedPackagesInUse.computeIfAbsent(deprecatedArtifact, __ -> new ArrayList<>()) - .add(packageName); + for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { + for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { + if (matcher.test(packageName) + && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) + .add(packageName); + } } } }); @@ -112,31 +129,94 @@ public class BundleValidator extends Validator { String.format("For JAR file '%s': \n" + "Manifest imports the following Java packages from '%s': %s. \n" + "%s", - jarPath, artifact.name, packagesInUse, artifact.description)); + filename, artifact.name, packagesInUse, artifact.description)); }); } - private enum DeprecatedArtifact { + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + + private Optional<String> getPomXmlContent(DeployLogger deployLogger, JarFile jarFile) { + return jarFile.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + return new String(jarFile.getInputStream(f).readAllBytes()); + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jarFile.getName())); + return null; + } + }); + } + + private void validatePomXml(DeployLogger deployLogger, String jarFilename, String pomXmlContent) { + try { + Document pom = DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(pomXmlContent))); + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/dependencies/dependency") + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element dependency = (Element) dependencies.item(i); + String groupId = dependency.getElementsByTagName("groupId").item(0).getTextContent(); + String artifactId = dependency.getElementsByTagName("artifactId").item(0).getTextContent(); + for (DeprecatedMavenArtifact deprecatedArtifact : DeprecatedMavenArtifact.values()) { + if (groupId.equals(deprecatedArtifact.groupId) && artifactId.equals(deprecatedArtifact.artifactId)) { + deployLogger.logApplicationPackage(Level.WARNING, + String.format( + "The pom.xml of bundle '%s' includes a dependency to the artifact '%s:%s'. \n%s", + jarFilename, groupId, artifactId, deprecatedArtifact.description)); + } + } + } + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (Exception e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", jarFilename)); + } + } + + private enum DeprecatedMavenArtifact { + VESPA_HTTP_CLIENT_EXTENSION("com.yahoo.vespa", "vespa-http-client-extensions", + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details."); + + final String groupId; + final String artifactId; + final String description; + + DeprecatedMavenArtifact(String groupId, String artifactId, String description) { + this.groupId = groupId; + this.artifactId = artifactId; + this.description = description; + } + } + + private enum DeprecatedProvidedBundle { ORG_JSON("org.json:json", "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", - Set.of("org.json")); + Set.of("org\\.json")); final String name; - final Collection<String> javaPackages; + final Collection<Predicate<String>> javaPackageMatchers; final Predicate<VersionRange> versionDiscriminator; final String description; - DeprecatedArtifact(String name, String description, Collection<String> javaPackages) { - this(name, description, __ -> true, javaPackages); + DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { + this(name, description, __ -> true, javaPackagePatterns); } - DeprecatedArtifact(String name, - String description, - Predicate<VersionRange> versionDiscriminator, - Collection<String> javaPackages) { + DeprecatedProvidedBundle(String name, + String description, + Predicate<VersionRange> versionDiscriminator, + Collection<String> javaPackagePatterns) { this.name = name; - this.javaPackages = javaPackages; + this.javaPackageMatchers = javaPackagePatterns.stream() + .map(s -> Pattern.compile(s).asMatchPredicate()) + .collect(Collectors.toList()); this.versionDiscriminator = versionDiscriminator; this.description = description; } diff --git a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF index 760a9ecf00f..760a9ecf00f 100644 --- a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF diff --git a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar b/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar Binary files differdeleted file mode 100644 index 84781c4802e..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar b/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar Binary files differdeleted file mode 100644 index f4f7dd4e127..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok.jar b/config-model/src/test/cfg/application/validation/testjars/ok.jar Binary files differdeleted file mode 100644 index fce043c6ff7..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/ok.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..53773b8b1cc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: ok +Bundle-SymbolicName: ok +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..1f88a5e6477 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: mybundle +Bundle-SymbolicName: mybundle +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml new file mode 100644 index 00000000000..1d28f307824 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.test</groupId> + <artifactId>mybundle</artifactId> + <packaging>container-plugin</packaging> + <version>1.0.0</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-http-client-extensions</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project>
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar Binary files differdeleted file mode 100644 index a395a52d17d..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..21c58490c13 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: vespa container maven plugin +Built-By: tonyv +Build-Jdk: 1.6.0_26 +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/jrt-5.1-SNAPSHOT.jar +Bundle-Version: 5.1.0.SNAPSHOT +Bundle-Name: container maven plugin test +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: TestBundle + diff --git a/config-model/src/test/cfg/application/validation/testjars/test.jar b/config-model/src/test/cfg/application/validation/testjars/test.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/test.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar Binary files differdeleted file mode 100644 index 31266f1e8f2..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar +++ /dev/null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index e2eae30d78d..ef4353d02fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -3,36 +3,41 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.Manifest; +import java.util.jar.JarOutputStream; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class BundleValidatorTest { - private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); @Test public void basicBundleValidation() throws Exception { // Valid jar file - JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); + JarFile ok = createTemporaryJarFile("ok"); BundleValidator bundleValidator = new BundleValidator(); bundleValidator.validateJarFile(new BaseDeployLogger(), ok); // No manifest - validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); + validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); } private void validateWithException(String jarName, String exceptionMessage) throws IOException { try { - JarFile jarFile = new JarFile(JARS_DIR + jarName); + JarFile jarFile = createTemporaryJarFile(jarName); BundleValidator bundleValidator = new BundleValidator(); bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile); assert (false); @@ -46,8 +51,8 @@ public class BundleValidatorTest { final StringBuffer buffer = new StringBuffer(); DeployLogger logger = createDeployLogger(buffer); - - new BundleValidator().validateJarFile(logger, new JarFile(JARS_DIR + "snapshot_bundle.jar")); + JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); + new BundleValidator().validateJarFile(logger, jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @@ -56,14 +61,50 @@ public class BundleValidatorTest { final StringBuffer buffer = new StringBuffer(); DeployLogger logger = createDeployLogger(buffer); BundleValidator validator = new BundleValidator(); - Manifest manifest = new Manifest(Files.newInputStream(Paths.get(JARS_DIR + "/manifest-producing-import-warnings.MF"))); - validator.validateManifest(logger, "my-app-bundle.jar", manifest); + JarFile jarFile = createTemporaryJarFile("import-warnings"); + validator.validateJarFile(logger, jarFile); assertThat(buffer.toString()) - .contains("For JAR file 'my-app-bundle.jar': \n" + + .contains("For JAR file 'import-warnings.jar': \n" + "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } + @Test + public void outputs_deploy_warning_on_deprecated_dependency() throws IOException { + StringBuffer buffer = new StringBuffer(); + DeployLogger logger = createDeployLogger(buffer); + BundleValidator validator = new BundleValidator(); + JarFile jarFile = createTemporaryJarFile("pom-xml-warnings"); + validator.validateJarFile(logger, jarFile); + assertThat(buffer.toString()) + .contains("The pom.xml of bundle 'pom-xml-warnings.jar' includes a dependency to the artifact " + + "'com.yahoo.vespa:vespa-http-client-extensions'. \n" + + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details.\n"); + } + + private JarFile createTemporaryJarFile(String testArtifact) throws IOException { + Path jarFile = tempDir.newFile(testArtifact + ".jar").toPath(); + Path artifactDirectory = Paths.get("src/test/cfg/application/validation/testjars/" + testArtifact); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jarFile))) { + Files.walk(artifactDirectory).forEach(path -> { + Path relativePath = artifactDirectory.relativize(path); + String zipName = relativePath.toString(); + uncheck(() -> { + if (Files.isDirectory(path)) { + out.putNextEntry(new JarEntry(zipName + "/")); + } else { + out.putNextEntry(new JarEntry(zipName)); + out.write(Files.readAllBytes(path)); + } + out.closeEntry(); + }); + }); + } + return new JarFile(jarFile.toFile()); + } + private DeployLogger createDeployLogger(StringBuffer buffer) { return (__, message) -> buffer.append(message).append('\n'); } |