Merge pull request #22260 from vespa-engine/jonmv/add-test-categories-header

Move test package logic to separate class, and add validation

17 files changed, 609 insertions, 252 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java
index 721f3fd3460..c4fb7c29e7f 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java
@@ -56,13 +56,7 @@ public class Bundle {
         if (!bundleDir.isDirectory()) {
             return new ArrayList<>();
         }
-        return Arrays.asList(bundleDir.listFiles(
-                new FilenameFilter() {
-                    @Override
-                    public boolean accept(File dir, String name) {
-                        return name.endsWith(".jar");
-                    }
-                }));
+        return Arrays.asList(bundleDir.listFiles((dir, name) -> name.endsWith(".jar")));
     }
 
     public List<DefEntry> getDefEntries() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 613e3413ae8..6907747646e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -93,6 +93,8 @@ import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.logging.Level;
@@ -471,8 +473,10 @@ public class ApplicationController {
             RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision());
             ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision));
 
+            AtomicReference<RevisionId> lastRevision = new AtomicReference<>();
             try (Mutex lock = lock(applicationId)) {
                 LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
+                application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set);
                 Instance instance = application.get().require(job.application().instance());
 
                 if (   ! applicationPackage.trustedCertificates().isEmpty()
@@ -492,22 +496,25 @@ public class ApplicationController {
             var quotaUsage = deploymentQuotaUsage(zone, job.application());
 
             // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as
-            // the source since it's the same application, so it should have the same warnings
-            NotificationSource source = zone.environment().isManuallyDeployed() ?
-                    NotificationSource.from(deployment) : NotificationSource.from(applicationId);
-
-            @SuppressWarnings("deprecation")
-            List<String> warnings = Optional.ofNullable(result.prepareResponse().log)
-                    .map(logs -> logs.stream()
-                            .filter(log -> log.applicationPackage)
-                            .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue())
-                            .map(log -> log.message)
-                            .sorted()
-                            .distinct()
-                            .collect(Collectors.toList()))
-                    .orElseGet(List::of);
-            if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage);
-            else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings);
+            // the source since it's the same application, so it should have the same warnings.
+            // These notifications are only updated when the last submitted revision is deployed here.
+            NotificationSource source = zone.environment().isManuallyDeployed()
+                                        ? NotificationSource.from(deployment)
+                                        : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null;
+            if (source != null) {
+                @SuppressWarnings("deprecation")
+                List<String> warnings = Optional.ofNullable(result.prepareResponse().log)
+                                                .map(logs -> logs.stream()
+                                                                 .filter(log -> log.applicationPackage)
+                                                                 .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue())
+                                                                 .map(log -> log.message)
+                                                                 .sorted()
+                                                                 .distinct()
+                                                                 .collect(Collectors.toList()))
+                                                .orElseGet(List::of);
+                if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage);
+                else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings);
+            }
 
             lockApplicationOrThrow(applicationId, application ->
                     store(application.with(job.application().instance(),
@@ -542,7 +549,7 @@ public class ApplicationController {
         controller.jobController().deploymentStatus(application.get());
 
         for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) {
-            if ( ! notification.source().instance().map(declaredInstances::contains).orElse(false))
+            if ( ! notification.source().instance().map(declaredInstances::contains).orElse(true))
                 controller.notificationsDb().removeNotifications(notification.source());
             if (notification.source().instance().isPresent() &&
                     ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index ed794a9d929..0d39703b70d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -66,9 +66,9 @@ public class ApplicationPackage {
 
     private static final String trustedCertificatesFile = "security/clients.pem";
     private static final String buildMetaFile = "build-meta.json";
-    private static final String deploymentFile = "deployment.xml";
+    static final String deploymentFile = "deployment.xml";
     private static final String validationOverridesFile = "validation-overrides.xml";
-    private static final String servicesFile = "services.xml";
+    static final String servicesFile = "services.xml";
 
     private final String contentHash;
     private final String bundleHash;
@@ -212,7 +212,7 @@ public class ApplicationPackage {
                                                   entry -> entry.getValue().get())));
     }
 
-    static byte[] filesZip(Map<String, byte[]> files) {
+    public static byte[] filesZip(Map<String, byte[]> files) {
         try (ZipBuilder zipBuilder = new ZipBuilder(files.values().stream().mapToInt(bytes -> bytes.length).sum() + 512)) {
             files.forEach(zipBuilder::add);
             zipBuilder.close();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java
new file mode 100644
index 00000000000..fb352848911
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java
@@ -0,0 +1,318 @@
+package com.yahoo.vespa.hosted.controller.application.pkg;
+
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.DeploymentSpec.Step;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.path.Path;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.text.Text;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
+import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
+import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp;
+import com.yahoo.yolean.Exceptions;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.jar.JarInputStream;
+import java.util.jar.Manifest;
+import java.util.regex.Pattern;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system;
+import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile;
+import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile;
+import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.mapping;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Validation and manipulation of test package.
+ *
+ * @author jonmv
+ */
+public class TestPackage {
+
+    // Must match exactly the advertised resources of an AWS instance type. Also consider that the container
+    // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259).
+    static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any);
+    static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any);
+
+    private final ApplicationPackage applicationPackage;
+    private final X509Certificate certificate;
+
+    public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp,
+                       DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) {
+
+        // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip.
+        Map<String, byte[]> entries = new HashMap<>();
+        entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]);
+        entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]);
+
+        entries.put(servicesFile,
+                    servicesXml( ! isPublicSystem,
+                                certificateValidFrom != null,
+                                testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())),
+                                testerApp));
+
+        entries.put(deploymentFile,
+                    deploymentXml(id.tester(),
+                                  spec.athenzDomain(),
+                                  spec.requireInstance(id.application().instance())
+                                      .athenzService(id.type().zone().environment(), id.type().zone().region())));
+
+        if (certificateValidFrom != null) {
+            KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048);
+            X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number());
+            this.certificate = X509CertificateBuilder.fromKeypair(keyPair,
+                                                                  subject,
+                                                                  certificateValidFrom,
+                                                                  certificateValidFrom.plus(certificateValidDuration),
+                                                                  SignatureAlgorithm.SHA512_WITH_RSA,
+                                                                  BigInteger.valueOf(1))
+                                                     .build();
+            entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8));
+            entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8));
+        }
+        else {
+            this.certificate = null;
+        }
+
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000);
+        transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries);
+        this.applicationPackage = new ApplicationPackage(buffer.toByteArray());
+    }
+
+    public ApplicationPackage asApplicationPackage() {
+        return applicationPackage;
+    }
+
+    public X509Certificate certificate() {
+        return Objects.requireNonNull(certificate);
+    }
+
+    public static TestSummary validateTests(DeploymentSpec spec, byte[] testPackage) {
+        return validateTests(expectedSuites(spec.steps()), testPackage);
+    }
+
+    static TestSummary validateTests(Collection<Suite> expectedSuites, byte[] testPackage) {
+        List<String> problems = new ArrayList<>();
+        Set<Suite> suites = new LinkedHashSet<>();
+        ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream()
+                  .map(entry -> Path.fromString(entry.name()))
+                  .collect(groupingBy(path -> path.elements().size() > 1 ? path.elements().get(0) : "",
+                                      mapping(path -> (path.elements().size() > 1 ? path.getChildPath() : path).getRelative(), toList())))
+                  .forEach((directory, paths) -> {
+                      switch (directory) {
+                          case "components": {
+                              for (String path : paths) {
+                                  if (path.endsWith("-tests.jar")) {
+                                      try {
+                                          byte[] testsJar = ZipEntries.readFile(testPackage, "components/" + path, 1 << 30);
+                                          Manifest manifest = new JarInputStream(new ByteArrayInputStream(testsJar)).getManifest();
+                                          for (String suite : manifest.getMainAttributes().getValue("X-JDisc-Test-Bundle-Categories").split(","))
+                                              switch (suite.trim()) {
+                                                  case "SystemTest": suites.add(system); break;
+                                                  case "StagingSetup": suites.add(staging_setup); break;
+                                                  case "StagingTest": suites.add(staging); break;
+                                                  case "ProductionTest": suites.add(production); break;
+                                                  default: problems.add("unexpected test suite name '" + suite + "' in bundle manifest");
+                                              }
+                                      }
+                                      catch (Exception e) {
+                                          problems.add("failed reading test bundle manifest: " + Exceptions.toMessageString(e));
+                                      }
+                                  }
+                              }
+                          }
+                          break;
+                          case "tests": {
+                              if (paths.stream().anyMatch(Pattern.compile("system-test/.+\\.json").asMatchPredicate())) suites.add(system);
+                              if (paths.stream().anyMatch(Pattern.compile("staging-setup/.+\\.json").asMatchPredicate())) suites.add(staging_setup);
+                              if (paths.stream().anyMatch(Pattern.compile("staging-test/.+\\.json").asMatchPredicate())) suites.add(staging);
+                              if (paths.stream().anyMatch(Pattern.compile("production-test/.+\\.json").asMatchPredicate())) suites.add(production);
+                          }
+                          break;
+                          case "artifacts": {
+                              if (paths.stream().anyMatch(Pattern.compile(".+-tests.jar").asMatchPredicate()))
+                                  suites.addAll(expectedSuites); // ಠ_ಠ
+
+                              for (String forbidden : List.of("key", "cert"))
+                                  if (paths.contains(forbidden))
+                                      problems.add("test package contains 'artifacts/" + forbidden +
+                                                   "'; this conflicts with credentials used to run tests in Vespa Cloud");
+                          }
+                          break;
+                      }
+                  });
+
+        if (expectedSuites.contains(system) && ! suites.contains(system))
+            problems.add("test package has no system tests, but <test /> is declared in deployment.xml");
+
+        if (suites.contains(staging) != suites.contains(staging_setup))
+            problems.add("test package has " + (suites.contains(staging) ? "staging tests" : "staging setup") +
+                         ", so it should also include " + (suites.contains(staging) ? "staging setup" : "staging tests"));
+        else if (expectedSuites.contains(staging) && ! suites.contains(staging))
+            problems.add("test package has no staging setup and tests, but <staging /> is declared in deployment.xml");
+
+        if (suites.contains(production) != expectedSuites.contains(production))
+            problems.add("test package has " + (suites.contains(production) ? "" : "no ") + "production tests, " +
+                         "but " + (suites.contains(production) ? "no " : "") + "production tests are declared in deployment.xml");
+
+        if ( ! problems.isEmpty())
+            problems.add("see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa");
+
+        return new TestSummary(problems, suites);
+    }
+
+    public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) {
+        NodeResources nodeResources = spec.steps().stream()
+                                          .filter(step -> step.concerns(zone.environment()))
+                                          .findFirst()
+                                          .flatMap(step -> step.zones().get(0).testerFlavor())
+                                          .map(NodeResources::fromLegacyName)
+                                          .orElse(zone.region().value().contains("aws-") ? DEFAULT_TESTER_RESOURCES_AWS
+                                                                                         : DEFAULT_TESTER_RESOURCES);
+        return nodeResources.with(NodeResources.DiskSpeed.any);
+    }
+
+    /** Returns the generated services.xml content for the tester application. */
+    public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate,
+                                     NodeResources resources, ControllerConfig.Steprunner.Testerapp config) {
+        int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?).
+        int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb());
+
+        // Of the remaining memory, split 50/50 between Surefire running the tests and the rest
+        int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2);
+
+        String resourceString = Text.format("<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>",
+                                            resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name());
+
+        String runtimeProviderClass = config.runtimeProviderClass();
+        String tenantCdBundle = config.tenantCdBundle();
+
+        String servicesXml =
+                "<?xml version='1.0' encoding='UTF-8'?>\n" +
+                "<services xmlns:deploy='vespa' version='1.0'>\n" +
+                "    <container version='1.0' id='tester'>\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" +
+                "            <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "                <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" +
+                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+                "                <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" +
+                "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" +
+                "            <binding>http://*/tester/v1/*</binding>\n" +
+                "        </handler>\n" +
+                "\n" +
+                "        <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+                "            <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+                "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+                "            <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "                <testsPath>tests</testsPath>\n" +
+                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+                "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <nodes count=\"1\">\n" +
+                "            <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" +
+                "            " + resourceString + "\n" +
+                "        </nodes>\n" +
+                "    </container>\n" +
+                "</services>\n";
+
+        return servicesXml.getBytes(UTF_8);
+    }
+
+    /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */
+    public static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) {
+        String deploymentSpec =
+                "<?xml version='1.0' encoding='UTF-8'?>\n" +
+                "<deployment version=\"1.0\" " +
+                athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") +
+                athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" +
+                "  <instance id=\"" + id.id().instance().value() + "\" />" +
+                "</deployment>";
+        return deploymentSpec.getBytes(UTF_8);
+    }
+
+    static Set<Suite> expectedSuites(List<Step> steps) {
+        Set<Suite> suites = new HashSet<>();
+        if (steps.isEmpty()) return suites;
+        for (Step step : steps) {
+            if (step.isTest()) {
+                if (step.concerns(Environment.prod)) suites.add(production);
+                if (step.concerns(Environment.test)) suites.add(system);
+                if (step.concerns(Environment.staging)) { suites.add(staging); suites.add(staging_setup); }
+            }
+            else
+                suites.addAll(expectedSuites(step.steps()));
+        }
+        return suites;
+    }
+
+
+    public static class TestSummary {
+
+        private final List<String> problems;
+        private final List<Suite> suites;
+
+        public TestSummary(List<String> problems, Set<Suite> suites) {
+            this.problems = List.copyOf(problems);
+            this.suites = List.copyOf(suites);
+        }
+
+        public List<String> problems() {
+            return problems;
+        }
+
+        public List<Suite> suites() {
+            return suites;
+        }
+
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
index a6cb7f23fc3..8392a77bad5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
@@ -13,6 +13,8 @@ import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Predicate;
@@ -35,19 +37,28 @@ public class ZipEntries {
 
     /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */
     public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) {
+        transferAndWrite(out, in, Map.of(name, content));
+    }
+
+    /** Copies the zipped content from in to out, adding/overwriting/removing (on {@code null}) entries as specified. */
+    public static void transferAndWrite(OutputStream out, InputStream in, Map<String, byte[]> entries) {
         try (ZipOutputStream zipOut = new ZipOutputStream(out);
              ZipInputStream zipIn = new ZipInputStream(in)) {
             for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) {
-                if (entry.getName().equals(name))
+                if (entries.containsKey(entry.getName()))
                     continue;
 
                 zipOut.putNextEntry(new ZipEntry(entry.getName()));
                 zipIn.transferTo(zipOut);
                 zipOut.closeEntry();
             }
-            zipOut.putNextEntry(new ZipEntry(name));
-            zipOut.write(content);
-            zipOut.closeEntry();
+            for (Entry<String, byte[]> entry : entries.entrySet()) {
+                if (entry.getValue() != null) {
+                    zipOut.putNextEntry(new ZipEntry(entry.getKey()));
+                    zipOut.write(entry.getValue());
+                    zipOut.closeEntry();
+                }
+            }
         }
         catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -76,6 +87,10 @@ public class ZipEntries {
         return new ZipEntries(entries);
     }
 
+    public static byte[] readFile(byte[] zip, String name, int maxEntrySizeInBytes) {
+        return from(zip, name::equals, maxEntrySizeInBytes, true).asList().get(0).contentOrThrow();
+    }
+
     public List<ZipEntryWithContent> asList() { return entries; }
 
     public static class ZipEntryWithContent {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 9e551c7ce78..52e5431b552 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -3,16 +3,12 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import ai.vespa.http.DomainName;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.Notifications;
 import com.yahoo.config.application.api.Notifications.When;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.AthenzDomain;
-import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -22,7 +18,6 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.text.Text;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -38,7 +33,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
 import com.yahoo.vespa.hosted.controller.application.ActivateResult;
@@ -46,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
+import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
@@ -124,13 +118,6 @@ public class InternalStepRunner implements StepRunner {
 
     private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName());
 
-    static final NodeResources DEFAULT_TESTER_RESOURCES =
-            new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any);
-    // Must match exactly the advertised resources of an AWS instance type. Also consider that the container
-    // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259).
-    static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
-            new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any);
-
     private final Controller controller;
     private final TestConfigSerializer testConfigSerializer;
     private final DeploymentFailureMails mails;
@@ -907,48 +894,20 @@ public class InternalStepRunner implements StepRunner {
     private ApplicationPackage testerPackage(RunId id) {
         RevisionId revision = controller.jobController().run(id).get().versions().targetRevision();
         DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec();
-
-        ZoneId zone = id.type().zone();
+        byte[] testZip = controller.applications().applicationStore().getTester(id.application().tenant(),
+                                                                                id.application().application(), revision);
         boolean useTesterCertificate = useTesterCertificate(id);
 
-        byte[] servicesXml = servicesXml( ! controller.system().isPublic(),
-                                         useTesterCertificate,
-                                         testerResourcesFor(zone, spec.requireInstance(id.application().instance())),
-                                         controller.controllerConfig().steprunner().testerapp());
-        byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), revision);
-        byte[] deploymentXml = deploymentXml(id.tester(),
-                                             spec.athenzDomain(),
-                                             spec.requireInstance(id.application().instance()).athenzService(zone.environment(), zone.region()));
-
-        try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + deploymentXml.length + 1000)) {
-            // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip.
-            zipBuilder.add(testPackage);
-            zipBuilder.add("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]);
-            zipBuilder.add("tests/.ignore-" + UUID.randomUUID(), new byte[0]);
-
-            zipBuilder.add("services.xml", servicesXml);
-            zipBuilder.add("deployment.xml", deploymentXml);
-            if (useTesterCertificate)
-                appendAndStoreCertificate(zipBuilder, id);
-
-            zipBuilder.close();
-            return new ApplicationPackage(zipBuilder.toByteArray());
-        }
-    }
+        TestPackage testPackage = new TestPackage(testZip,
+                                                  controller.system().isPublic(),
+                                                  id,
+                                                  controller.controllerConfig().steprunner().testerapp(),
+                                                  spec,
+                                                  useTesterCertificate ? controller.clock().instant() : null,
+                                                  timeouts.testerCertificate());
+        if (useTesterCertificate) controller.jobController().storeTesterCertificate(id, testPackage.certificate());
 
-    private void appendAndStoreCertificate(ZipBuilder zipBuilder, RunId id) {
-        KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048);
-        X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number());
-        X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair,
-                                                                         subject,
-                                                                         controller.clock().instant(),
-                                                                         controller.clock().instant().plus(timeouts.testerCertificate()),
-                                                                         SignatureAlgorithm.SHA512_WITH_RSA,
-                                                                         BigInteger.valueOf(1))
-                                                            .build();
-        controller.jobController().storeTesterCertificate(id, certificate);
-        zipBuilder.add("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8));
-        zipBuilder.add("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8));
+        return testPackage.asApplicationPackage();
     }
 
     private DeploymentId getTesterDeploymentId(RunId runId) {
@@ -956,90 +915,6 @@ public class InternalStepRunner implements StepRunner {
         return new DeploymentId(runId.tester().id(), zoneId);
     }
 
-    static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) {
-        NodeResources nodeResources = spec.steps().stream()
-                   .filter(step -> step.concerns(zone.environment()))
-                   .findFirst()
-                   .flatMap(step -> step.zones().get(0).testerFlavor())
-                   .map(NodeResources::fromLegacyName)
-                   .orElse(zone.region().value().contains("aws-") ?
-                           DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES);
-        return nodeResources.with(NodeResources.DiskSpeed.any);
-    }
-
-    /** Returns the generated services.xml content for the tester application. */
-    static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate,
-                              NodeResources resources, ControllerConfig.Steprunner.Testerapp config) {
-        int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?).
-        int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb());
-
-        // Of the remaining memory, split 50/50 between Surefire running the tests and the rest
-        int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2);
-
-        String resourceString = Text.format(
-                                              "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>",
-                                              resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name());
-
-        String runtimeProviderClass = config.runtimeProviderClass();
-        String tenantCdBundle = config.tenantCdBundle();
-
-        String servicesXml =
-                "<?xml version='1.0' encoding='UTF-8'?>\n" +
-                "<services xmlns:deploy='vespa' version='1.0'>\n" +
-                "    <container version='1.0' id='tester'>\n" +
-                "\n" +
-                "        <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" +
-                "            <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" +
-                "                <artifactsPath>artifacts</artifactsPath>\n" +
-                "                <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" +
-                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
-                "                <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" +
-                "            </config>\n" +
-                "        </component>\n" +
-                "\n" +
-                "        <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" +
-                "            <binding>http://*/tester/v1/*</binding>\n" +
-                "        </handler>\n" +
-                "\n" +
-                "        <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" +
-                "\n" +
-                "        <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
-                "            <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" +
-                "                <artifactsPath>artifacts</artifactsPath>\n" +
-                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
-                "            </config>\n" +
-                "        </component>\n" +
-                "\n" +
-                "        <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
-                "            <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
-                "                <artifactsPath>artifacts</artifactsPath>\n" +
-                "                <testsPath>tests</testsPath>\n" +
-                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
-                "            </config>\n" +
-                "        </component>\n" +
-                "\n" +
-                "        <nodes count=\"1\">\n" +
-                "            <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" +
-                "            " + resourceString + "\n" +
-                "        </nodes>\n" +
-                "    </container>\n" +
-                "</services>\n";
-
-        return servicesXml.getBytes(UTF_8);
-    }
-
-    /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */
-    private static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) {
-        String deploymentSpec =
-                "<?xml version='1.0' encoding='UTF-8'?>\n" +
-                "<deployment version=\"1.0\" " +
-                athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") +
-                athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" +
-                "  <instance id=\"" + id.id().instance().value() + "\" />" +
-                "</deployment>";
-        return deploymentSpec.getBytes(UTF_8);
-    }
-
     /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */
     private class DualLogger {
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 69d9ba504a5..91d127976ce 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -28,6 +28,11 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff;
+import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage;
+import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary;
+import com.yahoo.vespa.hosted.controller.notification.Notification;
+import com.yahoo.vespa.hosted.controller.notification.Notification.Type;
+import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
@@ -489,6 +494,16 @@ public class JobController {
             application = application.withRevisions(revisions -> revisions.with(version.get()));
             application = withPrunedPackages(application);
 
+            TestSummary testSummary = TestPackage.validateTests(applicationPackage.deploymentSpec(), testPackageBytes);
+            if (testSummary.problems().isEmpty())
+                controller.notificationsDb().removeNotification(NotificationSource.from(id),
+                                                                Type.testPackage);
+            else
+                controller.notificationsDb().setNotification(NotificationSource.from(id),
+                                                             Type.testPackage,
+                                                             Notification.Level.warning,
+                                                             testSummary.problems());
+
             applications.storeWithUpdatedConfig(application, applicationPackage);
             applications.deploymentTrigger().triggerNewRevision(id);
         });
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
index c39ee031e27..38cb09355ca 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java
@@ -66,10 +66,13 @@ public class Notification {
     }
 
     public enum Type {
-        /** Related to contents of application package, e.g. usage of deprecated features/syntax */
+        /** Related to contents of application package, e.g., usage of deprecated features/syntax */
         applicationPackage,
 
-        /** Related to deployment of application, e.g. system test failure, node allocation failure, internal errors, etc. */
+        /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */
+        testPackage,
+
+        /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */
         deployment,
 
         /** Application cluster is (near) external feed blocked */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
index c882681632e..570dbdd870e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java
@@ -92,6 +92,7 @@ public class NotificationsSerializer {
     private static String asString(Notification.Type type) {
         switch (type) {
             case applicationPackage: return "applicationPackage";
+            case testPackage: return "testPackage";
             case deployment: return "deployment";
             case feedBlock: return "feedBlock";
             case reindex: return "reindex";
@@ -102,6 +103,7 @@ public class NotificationsSerializer {
     private static Notification.Type typeFrom(Inspector field) {
         switch (field.asString()) {
             case "applicationPackage": return Notification.Type.applicationPackage;
+            case "testPackage": return Notification.Type.testPackage;
             case "deployment": return Notification.Type.deployment;
             case "feedBlock": return Notification.Type.feedBlock;
             case "reindex": return Notification.Type.reindex;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index c07794ea39c..4df53be57f2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -745,6 +745,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
     private static String notificationTypeAsString(Notification.Type type) {
         switch (type) {
             case applicationPackage: return "applicationPackage";
+            case testPackage: return "testPackage";
             case deployment: return "deployment";
             case feedBlock: return "feedBlock";
             case reindex: return "reindex";
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index dff379a2249..de03bbfb767 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -225,6 +225,7 @@ class JobControllerApiHandlerHelper {
             case aborted:                    return "aborted";
             case error:                      return "error";
             case testFailure:                return "testFailure";
+            case noTests:                    return "noTests";
             case endpointCertificateTimeout: return "endpointCertificateTimeout";
             case nodeAllocationFailure:      return "nodeAllocationFailure";
             case installationFailed:         return "installationFailed";
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index e9619297a2f..20b64419f28 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller;
 import com.google.common.collect.Sets;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.Notifications;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
@@ -38,6 +39,10 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.notification.Notification;
+import com.yahoo.vespa.hosted.controller.notification.Notification.Level;
+import com.yahoo.vespa.hosted.controller.notification.Notification.Type;
+import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
@@ -1104,6 +1109,27 @@ public class ControllerTest {
     }
 
     @Test
+    public void testTestPackageWarnings() {
+        String deploymentXml = "<deployment version='1.0'>\n" +
+                               "  <prod>\n" +
+                               "    <region>us-west-1</region>\n" +
+                               "  </prod>\n" +
+                               "</deployment>\n";
+        ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentXml);
+        byte[] testPackage = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0]));
+        var app = tester.newDeploymentContext();
+        tester.jobs().submit(app.application().id(), Optional.empty(), Optional.empty(), Optional.empty(), 1,
+                             applicationPackage, testPackage, Optional.empty(), 0);
+        assertEquals(List.of(new Notification(tester.clock().instant(),
+                                              Type.testPackage,
+                                              Level.warning,
+                                              NotificationSource.from(app.application().id()),
+                                              List.of("test package has staging tests, so it should also include staging setup",
+                                                      "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"))),
+                     tester.controller().notificationsDb().listNotifications(NotificationSource.from(app.application().id()), true));
+    }
+
+    @Test
     public void testCompileVersion() {
         DeploymentContext context = tester.newDeploymentContext();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region("us-west-1").build();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
index 4e155e937b9..99e22302c73 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
@@ -24,24 +24,24 @@ import static org.junit.Assert.fail;
  */
 public class ApplicationPackageTest {
 
-    private static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
-                                                "<deployment version=\"1.0\">\n" +
-                                                "    <test />\n" +
-                                                "    <prod>\n" +
-                                                "        <parallel>\n" +
-                                                "            <region active=\"true\">us-central-1</region>\n" +
-                                                "        </parallel>\n" +
-                                                "    </prod>\n" +
-                                                "</deployment>\n";
-
-    private static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" +
-                                              "    <preprocess:include file='jdisc.xml' />\n" +
-                                              "    <content version='1.0' if='foo' />\n" +
-                                              "    <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" +
-                                              "        <preprocess:include file='content/content.xml' />\n" +
-                                              "    </content>\n" +
-                                              "    <preprocess:include file='not_found.xml' required='false' />\n" +
-                                              "</services>\n";
+    static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+                                        "<deployment version=\"1.0\">\n" +
+                                        "    <test />\n" +
+                                        "    <prod>\n" +
+                                        "        <parallel>\n" +
+                                        "            <region active=\"true\">us-central-1</region>\n" +
+                                        "        </parallel>\n" +
+                                        "    </prod>\n" +
+                                        "</deployment>\n";
+
+    static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" +
+                                      "    <preprocess:include file='jdisc.xml' />\n" +
+                                      "    <content version='1.0' if='foo' />\n" +
+                                      "    <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" +
+                                      "        <preprocess:include file='content/content.xml' />\n" +
+                                      "    </content>\n" +
+                                      "    <preprocess:include file='not_found.xml' required='false' />\n" +
+                                      "</services>\n";
 
     private static final String jdiscXml = "<container id='stateless' version='1.0' />\n";
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java
new file mode 100644
index 00000000000..8588bb9ea16
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java
@@ -0,0 +1,157 @@
+package com.yahoo.vespa.hosted.controller.application.pkg;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary;
+import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system;
+import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.validateTests;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class TestPackageTest {
+
+    static byte[] testsJar(String... suites) throws IOException {
+        String manifest = "Manifest-Version: 1.0\n" +
+                          "Created-By: vespa container maven plugin\n" +
+                          "Build-Jdk-Spec: 17\n" +
+                          "Bundle-ManifestVersion: 2\n" +
+                          "Bundle-SymbolicName: canary-application-test\n" +
+                          "Bundle-Version: 1.0.1\n" +
+                          "Bundle-Name: Test & verification application for Vespa\n" +
+                          "X-JDisc-Test-Bundle-Version: 1.0\n" +
+                          "Bundle-Vendor: Yahoo!\n" +
+                          "Bundle-ClassPath: .,dependencies/fest-assert-1.4.jar,dependencies/fest-u\n" +
+                          " til-1.1.6.jar\n" +
+                          "Import-Package: ai.vespa.feed.client;version=\"[1.0.0,2)\",ai.vespa.hosted\n" +
+                          " .cd;version=\"[1.0.0,2)\",com.yahoo.config;version=\"[1.0.0,2)\",com.yahoo.\n" +
+                          " container.jdisc;version=\"[1.0.0,2)\",com.yahoo.jdisc.http;version=\"[1.0.\n" +
+                          " 0,2)\",com.yahoo.slime;version=\"[1.0.0,2)\",java.awt.image;version=\"[0.0.\n" +
+                          " 0,1)\",java.awt;version=\"[0.0.0,1)\",java.beans;version=\"[0.0.0,1)\",java.\n" +
+                          " io;version=\"[0.0.0,1)\",java.lang.annotation;version=\"[0.0.0,1)\",java.la\n" +
+                          " ng.reflect;version=\"[0.0.0,1)\",java.lang;version=\"[0.0.0,1)\",java.math;\n" +
+                          " version=\"[0.0.0,1)\",java.net.http;version=\"[0.0.0,1)\",java.net;version=\n" +
+                          " \"[0.0.0,1)\",java.nio.file;version=\"[0.0.0,1)\",java.security;version=\"[0\n" +
+                          " .0.0,1)\",java.text;version=\"[0.0.0,1)\",java.time.temporal;version=\"[0.0\n" +
+                          " .0,1)\",java.time;version=\"[0.0.0,1)\",java.util.concurrent;version=\"[0.0\n" +
+                          " .0,1)\",java.util.function;version=\"[0.0.0,1)\",java.util.stream;version=\n" +
+                          " \"[0.0.0,1)\",java.util;version=\"[0.0.0,1)\",javax.imageio;version=\"[0.0.0\n" +
+                          " ,1)\",org.junit.jupiter.api;version=\"[5.8.1,6)\"\n" +
+                          "X-JDisc-Test-Bundle-Categories: " + String.join(",", suites) + "\n" +
+                          "\n";
+
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        try (JarOutputStream out = new JarOutputStream(buffer)) {
+            write("META-INF/MANIFEST.MF", manifest, out);
+            write("dependencies/foo.jar", "bar", out);
+            write("META-INF/maven/ai.vespa.test/app/pom.xml", "<project />", out);
+            write("ai/vespa/test/Test.class", "baz", out);
+        }
+        return buffer.toByteArray();
+    }
+
+    static void write(String name, String content, JarOutputStream out) throws IOException {
+        out.putNextEntry(new ZipEntry(name));
+        out.write(content.getBytes(UTF_8));
+        out.closeEntry();
+    }
+
+    @Test
+    public void testBundleValidation() throws IOException {
+        byte[] testZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "StagingSetup", "ProductionTest"),
+                                                            "artifacts/key", new byte[0]));
+        TestSummary summary = validateTests(List.of(system), testZip);
+
+        assertEquals(List.of(system, staging_setup, production), summary.suites());
+        assertEquals(List.of("test package contains 'artifacts/key'; this conflicts with credentials used to run tests in Vespa Cloud",
+                             "test package has staging setup, so it should also include staging tests",
+                             "test package has production tests, but no production tests are declared in deployment.xml",
+                             "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"),
+                     summary.problems());
+    }
+
+    @Test
+    public void testFatTestsValidation() {
+        byte[] testZip = ApplicationPackage.filesZip(Map.of("artifacts/foo-tests.jar", new byte[0]));
+        TestSummary summary = validateTests(List.of(staging, production), testZip);
+
+        assertEquals(List.of(staging, production), summary.suites());
+        assertEquals(List.of("test package has staging tests, so it should also include staging setup",
+                             "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"),
+                     summary.problems());
+    }
+
+    @Test
+    public void testBasicTestsValidation() {
+        byte[] testZip = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0],
+                                                            "tests/staging-setup/foo.json", new byte[0]));
+        TestSummary summary = validateTests(List.of(system, production), testZip);
+        assertEquals(List.of(staging_setup, staging), summary.suites());
+        assertEquals(List.of("test package has no system tests, but <test /> is declared in deployment.xml",
+                             "test package has no production tests, but production tests are declared in deployment.xml",
+                             "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"),
+                     summary.problems());
+    }
+
+    @Test
+    public void generates_correct_tester_flavor() {
+        DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" +
+                                                     "    <instance id='first'>\n" +
+                                                     "        <test tester-flavor=\"d-6-16-100\" />\n" +
+                                                     "        <prod>\n" +
+                                                     "            <region active=\"true\">us-west-1</region>\n" +
+                                                     "            <test>us-west-1</test>\n" +
+                                                     "        </prod>\n" +
+                                                     "    </instance>\n" +
+                                                     "    <instance id='second'>\n" +
+                                                     "        <test />\n" +
+                                                     "        <staging />\n" +
+                                                     "        <prod tester-flavor=\"d-6-16-100\">\n" +
+                                                     "            <parallel>\n" +
+                                                     "                <region active=\"true\">us-east-3</region>\n" +
+                                                     "                <region active=\"true\">us-central-1</region>\n" +
+                                                     "            </parallel>\n" +
+                                                     "            <region active=\"true\">us-west-1</region>\n" +
+                                                     "            <test>us-west-1</test>\n" +
+                                                     "        </prod>\n" +
+                                                     "    </instance>\n" +
+                                                     "</deployment>\n");
+
+        NodeResources firstResources = TestPackage.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first"));
+        assertEquals(TestPackage.DEFAULT_TESTER_RESOURCES, firstResources);
+
+        NodeResources secondResources = TestPackage.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second"));
+        assertEquals(6, secondResources.vcpu(), 1e-9);
+        assertEquals(16, secondResources.memoryGb(), 1e-9);
+        assertEquals(100, secondResources.diskGb(), 1e-9);
+    }
+
+    @Test
+    public void generates_correct_services_xml() throws IOException {
+        assertEquals(Files.readString(Paths.get("src/test/resources/test_runner_services.xml-cd")),
+                     new String(TestPackage.servicesXml(true,
+                                                        false,
+                                                        new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local),
+                                                        new ControllerConfig.Steprunner.Testerapp.Builder().build()),
+                                UTF_8));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 0dde6bd882f..031cdaa84ae 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -13,9 +13,6 @@ import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -27,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage;
 import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
@@ -35,6 +33,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -47,8 +46,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Supplier;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info;
@@ -63,6 +60,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.time.temporal.ChronoUnit.SECONDS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -556,62 +554,4 @@ public class InternalStepRunnerTest {
                                            "3554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" +
                                            "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)";
 
-    @Test
-    public void generates_correct_tester_flavor() {
-        DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" +
-                                                     "    <instance id='first'>\n" +
-                                                     "        <test tester-flavor=\"d-6-16-100\" />\n" +
-                                                     "        <prod>\n" +
-                                                     "            <region active=\"true\">us-west-1</region>\n" +
-                                                     "            <test>us-west-1</test>\n" +
-                                                     "        </prod>\n" +
-                                                     "    </instance>\n" +
-                                                     "    <instance id='second'>\n" +
-                                                     "        <test />\n" +
-                                                     "        <staging />\n" +
-                                                     "        <prod tester-flavor=\"d-6-16-100\">\n" +
-                                                     "            <parallel>\n" +
-                                                     "                <region active=\"true\">us-east-3</region>\n" +
-                                                     "                <region active=\"true\">us-central-1</region>\n" +
-                                                     "            </parallel>\n" +
-                                                     "            <region active=\"true\">us-west-1</region>\n" +
-                                                     "            <test>us-west-1</test>\n" +
-                                                     "        </prod>\n" +
-                                                     "    </instance>\n" +
-                                                     "</deployment>\n");
-
-        NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first"));
-        assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources);
-
-        NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second"));
-        assertEquals(6, secondResources.vcpu(), 1e-9);
-        assertEquals(16, secondResources.memoryGb(), 1e-9);
-        assertEquals(100, secondResources.diskGb(), 1e-9);
-    }
-
-    @Test
-    public void generates_correct_services_xml() {
-        generates_correct_services_xml("test_runner_services.xml-cd");
-    }
-
-    private void generates_correct_services_xml(String filenameExpectedOutput) {
-        ControllerConfig.Steprunner.Testerapp config = new ControllerConfig.Steprunner.Testerapp.Builder().build();
-        assertFile(filenameExpectedOutput,
-                new String(InternalStepRunner.servicesXml(
-                        true,
-                        false,
-                        new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local),
-                        config)));
-    }
-
-    private void assertFile(String resourceName, String actualContent) {
-        try {
-            Path path = Paths.get("src/test/resources/").resolve(resourceName);
-            String expectedContent = new String(Files.readAllBytes(path));
-            assertEquals(expectedContent, actualContent);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index ac8bdafa2bd..d2698afdc48 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -8,10 +8,13 @@ import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.notification.Notification.Type;
+import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index c59c693a789..8a0446ac2e6 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -153,7 +153,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         try {
             var jsonDescriptor = IOUtils.readAll(resource.openStream(), Charset.defaultCharset()).trim();
             testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor);
-            logger.info( "Test classes in bundle :" + testDescriptor.toString());
+            logger.info("Test classes in bundle: " + testDescriptor);
             return Optional.of(testDescriptor);
         } catch (IOException e) {
             return Optional.empty();