Merge pull request #20756 from vespa-engine/bjorncs/bundle-import-validation

Bjorncs/bundle import validation [run-systemtest]

7 files changed, 241 insertions, 145 deletions

diff --git a/config-model/pom.xml b/config-model/pom.xml
index d42d5af8975..dc7bec27a3b 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -288,6 +288,22 @@
       <version>${project.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>biz.aQute.bnd</groupId>
+      <artifactId>biz.aQute.bndlib</artifactId>
+      <version>6.1.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <!-- These are not needed for our use of bndlib -->
+          <groupId>org.osgi</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java
new file mode 100644
index 00000000000..b6b9190fedf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java
@@ -0,0 +1,144 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import aQute.bnd.header.Parameters;
+import aQute.bnd.osgi.Domain;
+import aQute.bnd.version.VersionRange;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+import java.util.logging.Level;
+
+/**
+ * A validator for bundles.  Uses BND library for some of the validation.
+ *
+ * @author hmusum
+ * @author bjorncs
+ */
+public class BundleValidator extends Validator {
+
+    public BundleValidator() {}
+
+    @Override
+    public void validate(VespaModel model, DeployState deployState) {
+        ApplicationPackage app = deployState.getApplicationPackage();
+        for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) {
+            try {
+                Path path = Path.fromString(info.getPathRelativeToAppDir());
+                DeployLogger deployLogger = deployState.getDeployLogger();
+                deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path));
+                JarFile jarFile = new JarFile(app.getFileReference(path));
+                validateJarFile(deployLogger, jarFile);
+            } catch (IOException e) {
+                throw new IllegalArgumentException(
+                        "Failed to validate JAR file '" + info.getPathRelativeToAppDir() + "'", e);
+            }
+        }
+    }
+
+    void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException {
+        Manifest manifest = jarFile.getManifest();
+        String jarPath = jarFile.getName();
+        if (manifest == null) {
+            throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarPath);
+        }
+        validateManifest(deployLogger, jarPath, manifest);
+    }
+
+    void validateManifest(DeployLogger deployLogger, String jarPath, Manifest mf) {
+        // Check for required OSGI headers
+        Attributes attributes = mf.getMainAttributes();
+        HashSet<String> mfAttributes = new HashSet<>();
+        for (Map.Entry<Object,Object> entry : attributes.entrySet()) {
+            mfAttributes.add(entry.getKey().toString());
+        }
+        List<String> requiredOSGIHeaders = Arrays.asList(
+                "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version");
+        for (String header : requiredOSGIHeaders) {
+            if (!mfAttributes.contains(header)) {
+                throw new IllegalArgumentException("Required OSGI header '" + header +
+                        "' was not found in manifest in '" + jarPath + "'");
+            }
+        }
+
+        if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) {
+            deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarPath +
+                    ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in  the version specification in services.xml.");
+        }
+
+        if (attributes.getValue("Import-Package") != null) {
+            validateImportedPackages(deployLogger, jarPath, mf);
+        }
+    }
+
+    private static void validateImportedPackages(DeployLogger deployLogger, String jarPath, Manifest manifest) {
+        Domain osgiHeaders = Domain.domain(manifest);
+        Parameters importPackage = osgiHeaders.getImportPackage();
+        Map<DeprecatedArtifact, List<String>> deprecatedPackagesInUse = new HashMap<>();
+
+        importPackage.forEach((packageName, attrs) -> {
+            VersionRange versionRange = attrs.getVersion() != null
+                    ? VersionRange.parseOSGiVersionRange(attrs.getVersion())
+                    : null;
+
+            for (DeprecatedArtifact deprecatedArtifact : DeprecatedArtifact.values()) {
+                if (deprecatedArtifact.javaPackages.contains(packageName)
+                        && (versionRange == null || deprecatedArtifact.versionDiscriminator.test(versionRange))) {
+                    deprecatedPackagesInUse.computeIfAbsent(deprecatedArtifact, __ -> new ArrayList<>())
+                            .add(packageName);
+                }
+            }
+        });
+
+        deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> {
+            deployLogger.logApplicationPackage(Level.WARNING,
+                    String.format("For JAR file '%s': \n" +
+                            "Manifest imports the following Java packages from '%s': %s. \n" +
+                            "%s",
+                            jarPath, artifact.name, packagesInUse, artifact.description));
+        });
+    }
+
+    private enum DeprecatedArtifact {
+        ORG_JSON("org.json:json",
+                "The org.json library will no longer provided by jdisc runtime on Vespa 8. " +
+                        "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.",
+                Set.of("org.json"));
+
+        final String name;
+        final Collection<String> javaPackages;
+        final Predicate<VersionRange> versionDiscriminator;
+        final String description;
+
+        DeprecatedArtifact(String name, String description, Collection<String> javaPackages) {
+            this(name, description, __ -> true, javaPackages);
+        }
+
+        DeprecatedArtifact(String name,
+                           String description,
+                           Predicate<VersionRange> versionDiscriminator,
+                           Collection<String> javaPackages) {
+            this.name = name;
+            this.javaPackages = javaPackages;
+            this.versionDiscriminator = versionDiscriminator;
+            this.description = description;
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java
deleted file mode 100644
index 21e396959a7..00000000000
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.application.validation;
-
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.path.Path;
-import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.config.application.api.ComponentInfo;
-import com.yahoo.config.application.api.DeployLogger;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.jar.Attributes;
-import java.util.jar.JarFile;
-import java.util.jar.Manifest;
-import java.util.logging.Level;
-import java.util.zip.ZipException;
-
-/**
- * A validator for bundles.  Uses BND library for some of the validation (not active yet)
- *
- * @author hmusum
- * @since 2010-11-11
- */
-public class ComponentValidator extends Validator {
-    private JarFile jarFile;
-
-    public ComponentValidator() {
-    }
-
-    public ComponentValidator(JarFile jarFile) {
-        this.jarFile = jarFile;
-    }
-
-    @Override
-    public void validate(VespaModel model, DeployState deployState) {
-        ApplicationPackage app = deployState.getApplicationPackage();
-        for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) {
-            try {
-                this.jarFile = new JarFile(app.getFileReference(Path.fromString(info.getPathRelativeToAppDir())));
-            } catch (ZipException e) {
-                throw new IllegalArgumentException("Error opening jar file '" + info.getPathRelativeToAppDir() +
-                                                   "'. Please check that this is a valid jar file");
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-            try {
-                validateAll(deployState.getDeployLogger());
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
-    public void validateAll(DeployLogger deployLogger) throws IOException {
-        validateOSGIHeaders(deployLogger);
-    }
-
-    public void validateOSGIHeaders(DeployLogger deployLogger) throws IOException {
-        Manifest mf = jarFile.getManifest();
-        if (mf == null) {
-            throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarFile.getName());
-        }
-
-        // Check for required OSGI headers
-        Attributes attributes = mf.getMainAttributes();
-        HashSet<String> mfAttributes = new HashSet<>();
-        for (Object attributeSet : attributes.entrySet()) {
-            Map.Entry<Object, Object> e = (Map.Entry<Object, Object>) attributeSet;
-            mfAttributes.add(e.getKey().toString());
-        }
-        List<String> requiredOSGIHeaders = Arrays.asList(
-                "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version");
-        for (String header : requiredOSGIHeaders) {
-            if (!mfAttributes.contains(header)) {
-                throw new IllegalArgumentException("Required OSGI header '" + header +
-                        "' was not found in manifest in '" + jarFile.getName() + "'");
-            }
-        }
-
-        if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) {
-            deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarFile.getName() +
-                    ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in  the version specification in services.xml.");
-        }
-    }
-}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 5215fdcb301..08dc73a1bd0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -63,7 +63,7 @@ public class Validation {
             new RoutingSelectorValidator().validate(model, deployState);
         }
         new SchemasDirValidator().validate(model, deployState);
-        new ComponentValidator().validate(model, deployState);
+        new BundleValidator().validate(model, deployState);
         new SearchDataTypeValidator().validate(model, deployState);
         new ComplexAttributeFieldsValidator().validate(model, deployState);
         new StreamingValidator().validate(model, deployState);
diff --git a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF b/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF
new file mode 100644
index 00000000000..760a9ecf00f
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF
@@ -0,0 +1,10 @@
+Manifest-Version: 1.0
+Export-Package: com.yahoo.vespa.test.myapp;version=1.0.0
+Bundle-ManifestVersion: 2
+Bundle-SymbolicName: my-bundle
+Bundle-Version: 7.0.0
+Created-By: vespa container maven plugin
+Bundle-Name: my-bundle
+Bundle-Vendor: Yahoo!
+Import-Package: org.json;version="[0.0.0,1)"
+
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java
new file mode 100644
index 00000000000..e2eae30d78d
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java
@@ -0,0 +1,70 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class BundleValidatorTest {
+    private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/";
+
+    @Test
+    public void basicBundleValidation() throws Exception {
+        // Valid jar file
+        JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar"));
+        BundleValidator bundleValidator = new BundleValidator();
+        bundleValidator.validateJarFile(new BaseDeployLogger(), ok);
+
+        // No manifest
+        validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar");
+    }
+
+    private void validateWithException(String jarName, String exceptionMessage) throws IOException {
+        try {
+            JarFile jarFile = new JarFile(JARS_DIR + jarName);
+            BundleValidator bundleValidator = new BundleValidator();
+            bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile);
+            assert (false);
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), exceptionMessage);
+        }
+    }
+
+    @Test
+    public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException {
+        final StringBuffer buffer = new StringBuffer();
+
+        DeployLogger logger = createDeployLogger(buffer);
+
+        new BundleValidator().validateJarFile(logger, new JarFile(JARS_DIR + "snapshot_bundle.jar"));
+        assertTrue(buffer.toString().contains("Deploying snapshot bundle"));
+    }
+
+    @Test
+    public void outputs_deploy_warning_on_import_of_packages_from_deprecated_artifact() throws IOException {
+        final StringBuffer buffer = new StringBuffer();
+        DeployLogger logger = createDeployLogger(buffer);
+        BundleValidator validator = new BundleValidator();
+        Manifest manifest = new Manifest(Files.newInputStream(Paths.get(JARS_DIR + "/manifest-producing-import-warnings.MF")));
+        validator.validateManifest(logger, "my-app-bundle.jar", manifest);
+        assertThat(buffer.toString())
+                .contains("For JAR file 'my-app-bundle.jar': \n" +
+                        "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" +
+                        "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.");
+    }
+
+    private DeployLogger createDeployLogger(StringBuffer buffer) {
+        return (__, message) -> buffer.append(message).append('\n');
+    }
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java
deleted file mode 100644
index a375621d391..00000000000
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.application.validation;
-
-import org.junit.Test;
-
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.model.application.provider.BaseDeployLogger;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.jar.JarFile;
-import java.util.logging.Level;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class ComponentValidatorTest {
-    private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/";
-
-    @Test
-    public void basicComponentValidation() throws Exception {
-        // Valid jar file
-        JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar"));
-        ComponentValidator componentValidator = new ComponentValidator(ok);
-        componentValidator.validateAll(new BaseDeployLogger());
-
-        // No manifest
-        validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar");
-    }
-
-    private void validateWithException(String jarName, String exceptionMessage) throws IOException {
-        try {
-            JarFile jarFile = new JarFile(JARS_DIR + jarName);
-            ComponentValidator componentValidator = new ComponentValidator(jarFile);
-            componentValidator.validateAll(new BaseDeployLogger());
-            assert (false);
-        } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), exceptionMessage);
-        }
-    }
-
-    @Test
-    public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException {
-        final StringBuffer buffer = new StringBuffer();
-        
-        DeployLogger logger = new DeployLogger() {
-            @Override
-            public void log(Level level, String message) {
-                buffer.append(message).append('\n');
-            }
-        };
-        
-        new ComponentValidator(new JarFile(JARS_DIR + "snapshot_bundle.jar")).validateAll(logger);
-        assertTrue(buffer.toString().contains("Deploying snapshot bundle"));
-    }
-}