Merge pull request #27274 from vespa-engine/warn-for-using-non-PublicApi

Warn for using non public api

3 files changed, 101 insertions, 19 deletions

diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java
index 2b5941cc5aa..af6c82023ab 100644
--- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java
+++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java
@@ -7,12 +7,14 @@ import com.yahoo.container.plugin.util.JarFiles;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.jar.Manifest;
+import java.util.stream.Collectors;
 
 /**
  * Static utilities for analyzing jar files.
@@ -34,20 +36,42 @@ public class AnalyzeBundle {
     }
 
     static List<Export> exportedPackages(File jarFile) {
+        var manifest = getOsgiManifest(jarFile);
+        if (manifest == null) return Collections.emptyList();
         try {
-            Optional<Manifest> jarManifest = JarFiles.getManifest(jarFile);
-            if (jarManifest.isPresent()) {
-                Manifest manifest = jarManifest.get();
-                if (isOsgiManifest(manifest)) {
-                    return parseExports(manifest);
-                }
-            }
-            return Collections.emptyList();
+            return parseExports(manifest);
         } catch (Exception e) {
             throw new RuntimeException(String.format("Invalid manifest in bundle '%s'", jarFile.getPath()), e);
         }
     }
 
+    public static List<String> publicApiPackagesAggregated(Collection<File> jarFiles) {
+        return jarFiles.stream()
+                .map(AnalyzeBundle::publicApiPackages)
+                .flatMap(List::stream)
+                .distinct()
+                .toList();
+    }
+
+    static List<String> publicApiPackages(File jarFile) {
+        var manifest = getOsgiManifest(jarFile);
+        if (manifest == null) return Collections.emptyList();
+        return getMainAttributeValue(manifest, "X-JDisc-PublicApi-Package")
+                .map(s -> Arrays.asList(s.split(",")))
+                .orElseGet(ArrayList::new);
+    }
+
+    private static Manifest getOsgiManifest(File jarFile) {
+        Optional<Manifest> jarManifest = JarFiles.getManifest(jarFile);
+        if (jarManifest.isPresent()) {
+            Manifest manifest = jarManifest.get();
+            if (isOsgiManifest(manifest)) {
+                return manifest;
+            }
+        }
+        return null;
+    }
+
     public static Optional<String> bundleSymbolicName(File jarFile) {
         return JarFiles.getManifest(jarFile).flatMap(AnalyzeBundle::getBundleSymbolicName);
     }
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java
index 9eef8a55c01..48a128c2f0d 100644
--- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java
+++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java
@@ -1,8 +1,13 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.plugin.classanalysis;
 
+import com.yahoo.container.plugin.osgi.ImportPackages;
+
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Utility methods related to packages.
@@ -31,6 +36,22 @@ public class Packages {
         }
     }
 
+    /**
+     * Returns the imported Vespa packages that don't exist in the given set of allowed packages.
+     */
+    public static List<String> disallowedVespaImports(Map<String, ImportPackages.Import> imports, List<String> allowed) {
+        if (imports == null || imports.isEmpty()) return List.of();
+
+        var publicApi = allowed == null ? Set.of() : new HashSet<>(allowed);
+
+        Set<String> yahooImports = imports.keySet().stream()
+                .filter(pkg -> pkg.startsWith("com.yahoo") || pkg.startsWith("ai.vespa."))
+                .collect(Collectors.toSet());
+
+        List<String> disallowedImports = yahooImports.stream().collect(Collectors.groupingBy(publicApi::contains)).get(false);
+        return disallowedImports == null ? List.of() : disallowedImports;
+    }
+
     public static PackageMetaData analyzePackages(Set<ClassFileMetaData> allClasses) {
         Set<String> definedPackages = new HashSet<>();
         Set<String> referencedPackages = new HashSet<>();
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java
index 2781decdf79..eef74365b92 100644
--- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java
+++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java
@@ -26,6 +26,8 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.yahoo.container.plugin.bundle.AnalyzeBundle.exportedPackagesAggregated;
+import static com.yahoo.container.plugin.bundle.AnalyzeBundle.publicApiPackagesAggregated;
+import static com.yahoo.container.plugin.classanalysis.Packages.disallowedVespaImports;
 import static com.yahoo.container.plugin.osgi.ExportPackages.exportsByPackageName;
 import static com.yahoo.container.plugin.osgi.ImportPackages.calculateImports;
 import static com.yahoo.container.plugin.util.Files.allDescendantFiles;
@@ -66,6 +68,16 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
     private BundleType bundleType = BundleType.USER;
 
     @Parameter(defaultValue = "false")
+    private boolean suppressWarningMissingImportPackages;
+    @Parameter(defaultValue = "false")
+    private boolean suppressWarningPublicApi;
+    @Parameter(defaultValue = "false")
+    private boolean suppressWarningOverlappingPackages;
+
+    @Parameter(defaultValue = "false")
+    private boolean failOnWarnings;
+
+    @Parameter(defaultValue = "false")
     private boolean buildLegacyVespaPlatformBundle;
 
     public void execute() throws MojoExecutionException {
@@ -78,10 +90,12 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
             if (! isContainerDiscArtifact(project.getArtifact()))
                 throwIfInternalContainerArtifactsAreIncluded(artifactSet.getJarArtifactsToInclude());
 
-            List<Export> exportedPackagesFromProvidedJars = exportedPackagesAggregated(
-                    artifactSet.getJarArtifactsProvided().stream().map(Artifact::getFile).toList());
+            List<Artifact> providedJarArtifacts = artifactSet.getJarArtifactsProvided();
+            List<File> providedJarFiles = providedJarArtifacts.stream().map(Artifact::getFile).toList();
+            List<Export> exportedPackagesFromProvidedJars = exportedPackagesAggregated(providedJarFiles);
+            List<String> publicApiPackagesFromProvidedJars = publicApiPackagesAggregated(providedJarFiles);
 
-            // Packages from Export-Package headers in provided scoped jars
+            // Packages from Export-Package/PublicApi headers in provided scoped jars
             Set<String> exportedPackagesFromProvidedDeps = ExportPackages.packageNames(exportedPackagesFromProvidedJars);
 
             // Packaged defined in this project's code
@@ -95,11 +109,11 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
 
             logDebugPackageSets(exportedPackagesFromProvidedJars, includedPackages);
 
-            if (hasJdiscCoreProvided(artifactSet.getJarArtifactsProvided())) {
+            if (hasJdiscCoreProvided(providedJarArtifacts)) {
                 // jdisc_core being provided guarantees that log output does not contain its exported packages
                 logMissingPackages(exportedPackagesFromProvidedDeps, projectPackages, compileJarsPackages, includedPackages);
-            } else {
-                getLog().warn(("This project does not have '%s' as provided dependency, so the generated 'Import-Package' " +
+            } else if (! suppressWarningMissingImportPackages) {
+                warnOrThrow(("This project does not have '%s' as provided dependency, so the generated 'Import-Package' " +
                         "OSGi header may be missing important packages.").formatted(wantedProvidedDependency()));
             }
             logOverlappingPackages(projectPackages, exportedPackagesFromProvidedDeps);
@@ -109,9 +123,12 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
                                                                      includedPackages.definedPackages(),
                                                                      exportsByPackageName(exportedPackagesFromProvidedJars));
 
+            List<String> nonPublicApiUsed = disallowedVespaImports(calculatedImports, publicApiPackagesFromProvidedJars);
+            logNonPublicApiUsage(nonPublicApiUsed);
 
             Map<String, String> manifestContent = generateManifestContent(artifactSet.getJarArtifactsToInclude(), calculatedImports, includedPackages);
             addAdditionalManifestProperties(manifestContent, includedPackages);
+            addManifestPropertiesForUserBundles(manifestContent, nonPublicApiUsed);
             createManifestFile(Paths.get(project.getBuild().getOutputDirectory()), manifestContent);
 
         } catch (Exception e) {
@@ -142,6 +159,16 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
         addIfNotEmpty(manifestContent, "WebInfUrl", webInfUrl);
     }
 
+    private void addManifestPropertiesForUserBundles(Map<String, String> manifestContent, List<String> nonPublicApiUsed) {
+        if (effectiveBundleType() != BundleType.USER) return;
+        addIfNotEmpty(manifestContent, "X-JDisc-Non-PublicApi-Import-Package", String.join(",", nonPublicApiUsed));
+    }
+
+    private void logNonPublicApiUsage(List<String> nonPublicApiUsed) {
+        if (suppressWarningPublicApi || effectiveBundleType() != BundleType.USER || nonPublicApiUsed.isEmpty()) return;
+        warnOrThrow("This project uses packages that are not part of Vespa's public api: %s".formatted(nonPublicApiUsed));
+    }
+
     private static String publicApi(PackageTally tally) {
         return tally.publicApiPackages().stream().sorted().collect(Collectors.joining(","));
     }
@@ -181,10 +208,12 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
 
     private void logOverlappingPackages(PackageTally projectPackages,
                                         Set<String> exportedPackagesFromProvidedDeps) {
+        if (suppressWarningOverlappingPackages) return;
+
         Set<String> overlappingProjectPackages = Sets.intersection(projectPackages.definedPackages(), exportedPackagesFromProvidedDeps);
         if (! overlappingProjectPackages.isEmpty()) {
-            getLog().warn("This project defines packages that are also defined in provided scoped dependencies " +
-                          "(overlapping packages are strongly discouraged): " + overlappingProjectPackages);
+            warnOrThrow("This project defines packages that are also defined in provided scoped dependencies " +
+                         "(overlapping packages are strongly discouraged): " + overlappingProjectPackages);
         }
     }
 
@@ -211,9 +240,8 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
         List<Artifact> unsupportedArtifacts = nonJarArtifacts.stream().filter(a -> ! a.getType().equals("pom"))
                 .toList();
 
-        unsupportedArtifacts.forEach(artifact -> getLog()
-                .warn(String.format("Unsupported artifact '%s': Type '%s' is not supported. Please file a feature request.",
-                        artifact.getId(), artifact.getType())));
+        unsupportedArtifacts.forEach(artifact -> warnOrThrow(String.format("Unsupported artifact '%s': Type '%s' is not supported. Please file a feature request.",
+                                                                           artifact.getId(), artifact.getType())));
     }
 
     private void throwIfInternalContainerArtifactsAreIncluded(Collection<Artifact> includedArtifacts) throws MojoExecutionException {
@@ -246,4 +274,13 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo {
 
         return PackageTally.fromAnalyzedClassFiles(analyzedClasses);
     }
+
+    private void warnOrThrow(String... messages){
+        String message = String.join("\n", messages);
+        if (failOnWarnings) {
+            throw new RuntimeException(message);
+        }
+        getLog().warn(message);
+    }
+
 }