Revert changes to OSGi test runner back to 5.8.1 compatibility

9 files changed, 77 insertions, 92 deletions

diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index 06a4680e982..21890bab569 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -22,7 +22,6 @@ import java.time.Clock;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
@@ -33,8 +32,6 @@ import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import java.util.logging.Logger;
 
-import static java.util.stream.Collectors.toSet;
-
 
 /**
  * @author mortent
@@ -118,11 +115,9 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         List<Class<?>> testClasses = classLoader.apply(suite);
         if (testClasses == null)
             return  null;
-        Set<String> testClassNames = testClasses.stream().map(Class::getName).collect(toSet());
 
         testRuntimeProvider.initialize(testConfig);
         TestReportGeneratingListener testReportListener = new TestReportGeneratingListener(suite,
-                                                                                           testClassNames,
                                                                                            record -> logRecords.put(record.getSequenceNumber(), record),
                                                                                            stdoutTee,
                                                                                            stderrTee,
@@ -156,12 +151,16 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
 
     static TestRunner.Status testRunnerStatus(TestReport report) {
         if (report == null) return Status.NO_TESTS;
-        return switch (report.root().status()) {
-            case error, failed                -> Status.FAILURE;
-            case inconclusive                 -> Status.INCONCLUSIVE;
-            case successful, skipped, aborted -> report.root().tally().containsKey(TestReport.Status.successful) ? Status.SUCCESS
-                                                                                                                 : Status.NO_TESTS;
-        };
+        switch (report.root().status()) {
+            case error:
+            case failed:       return Status.FAILURE;
+            case inconclusive: return Status.INCONCLUSIVE;
+            case successful:
+            case skipped:
+            case aborted:     return report.root().tally().containsKey(TestReport.Status.successful) ? Status.SUCCESS
+                                                                                                     : Status.NO_TESTS;
+            default: throw new IllegalStateException("unknown status '" + report.root().status() + "'");
+        }
     }
 
     @Override
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
index 0b25c6f6402..a2ac86309d9 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
@@ -20,7 +20,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
 
@@ -33,31 +32,28 @@ public class TestReport {
 
     private final Object monitor = new Object();
     private final Set<TestIdentifier> complete = new HashSet<>();
-    private final Set<String> testClassNames;
     private final Clock clock;
     private final ContainerNode root;
     private final Suite suite;
     private NamedNode current;
     private TestPlan plan;
 
-    private TestReport(Clock clock, Suite suite, Set<String> testClassNames, ContainerNode root) {
+    private TestReport(Clock clock, Suite suite, ContainerNode root) {
         this.clock = clock;
         this.root = root;
-        this.testClassNames = Set.copyOf(testClassNames);
         this.current = root;
         this.suite = suite;
     }
 
-    TestReport(Clock clock, Suite suite, Set<String> testClassNames) {
-        this(clock, suite, testClassNames, new ContainerNode(null, null, toString(suite), clock.instant()));
+    TestReport(Clock clock, Suite suite) {
+        this(clock, suite, new ContainerNode(null, null, toString(suite), clock.instant()));
     }
 
     static TestReport createFailed(Clock clock, Suite suite, Throwable thrown) {
-        if (thrown instanceof OutOfMemoryError oome) throw oome;
-        if (thrown instanceof ExecutionException ee) thrown = ee.getCause();
-        TestReport failed = new TestReport(clock, suite, Set.of());
+        if (thrown instanceof OutOfMemoryError) throw (Error) thrown;
+        TestReport failed = new TestReport(clock, suite);
         failed.complete();
-        failed.root().children.add(new FailureNode(failed.root(), clock.instant(), thrown, suite, Set.of()));
+        failed.root().children.add(new FailureNode(failed.root(), clock.instant(), thrown, suite));
         return failed;
     }
 
@@ -131,7 +127,7 @@ public class TestReport {
         synchronized (monitor) {
             Status status = Status.successful;
             if (thrown != null) {
-                FailureNode failure = new FailureNode(current, clock.instant(), thrown, suite, testClassNames);
+                FailureNode failure = new FailureNode(current, clock.instant(), thrown, suite);
                 current.children.add(failure);
                 status = failure.status();
             }
@@ -142,7 +138,7 @@ public class TestReport {
 
     void log(LogRecord record) {
         synchronized (monitor) {
-            if (record.getThrown() != null) trimStackTraces(record.getThrown(), testClassNames);
+            if (record.getThrown() != null) trimStackTraces(record.getThrown(), JunitRunner.class.getName());
             if ( ! (current.children.peekLast() instanceof OutputNode))
                 current.children.add(new OutputNode(current));
 
@@ -162,9 +158,7 @@ public class TestReport {
                 ContainerNode newRoot = new ContainerNode(null, null, root.name(), root.start());
                 newRoot.children.addAll(root.children);
                 newRoot.children.addAll(other.root.children);
-                Set<String> testClassNames = new HashSet<>(this.testClassNames);
-                testClassNames.addAll(other.testClassNames);
-                TestReport merged = new TestReport(clock, suite, testClassNames, newRoot);
+                TestReport merged = new TestReport(clock, suite, newRoot);
                 merged.complete();
                 return merged;
             }
@@ -283,9 +277,9 @@ public class TestReport {
         private final Throwable thrown;
         private final Suite suite;
 
-        public FailureNode(NamedNode parent, Instant now, Throwable thrown, Suite suite, Set<String> testClassNames) {
+        public FailureNode(NamedNode parent, Instant now, Throwable thrown, Suite suite) {
             super(parent, null, thrown.toString(), now);
-            trimStackTraces(thrown, testClassNames);
+            trimStackTraces(thrown, JunitRunner.class.getName());
             this.thrown = thrown;
             this.suite = suite;
 
@@ -297,6 +291,10 @@ public class TestReport {
             children.add(child);
         }
 
+        public Throwable thrown() {
+            return thrown;
+        }
+
         @Override
         public Duration duration() {
             return Duration.ZERO;
@@ -329,20 +327,26 @@ public class TestReport {
 
     /**
      * Recursively trims stack traces for the given throwable and its causes/suppressed.
-     * This is based on the assumption that the relevant stack is anything from the first
-     * test bundle class frame, and upwards; the exception is for dynamic tests, where a
-     * specific dynamic test factory method is right below the first user frame.
+     * This is based on the assumption that the relevant stack is anything above the first native
+     * reflection invocation, above any frame in the given root class.
      */
-    static void trimStackTraces(Throwable thrown, Set<String> testClassNames) {
+    static void trimStackTraces(Throwable thrown, String testFrameworkRootClass) {
         if (thrown == null)
             return;
 
         StackTraceElement[] stack = thrown.getStackTrace();
-        int i = -1;
-        int cutoff = stack.length;
+        int i = 0;
+        int firstReflectFrame = -1;
+        int cutoff = 0;
+        boolean rootedInTestFramework = false;
         while (++i < stack.length) {
-            for (String name : testClassNames) if (stack[i].getClassName().startsWith(name))
-                cutoff = i + 1;
+            rootedInTestFramework |= testFrameworkRootClass.equals(stack[i].getClassName());
+            if (firstReflectFrame == -1 && stack[i].getClassName().startsWith("jdk.internal.reflect."))
+                firstReflectFrame = i; // jdk.internal.reflect class invokes the first user test frame, on both jdk 17 and 21.
+            if (rootedInTestFramework && firstReflectFrame > 0) {
+                cutoff = firstReflectFrame;
+                break;
+            }
             boolean isDynamicTestInvocation = "org.junit.jupiter.engine.descriptor.DynamicTestTestDescriptor".equals(stack[i].getClassName());
             if (isDynamicTestInvocation) {
                 cutoff = i;
@@ -352,9 +356,9 @@ public class TestReport {
         thrown.setStackTrace(copyOf(stack, cutoff));
 
         for (Throwable suppressed : thrown.getSuppressed())
-            trimStackTraces(suppressed, testClassNames);
+            trimStackTraces(suppressed, testFrameworkRootClass);
 
-        trimStackTraces(thrown.getCause(), testClassNames);
+        trimStackTraces(thrown.getCause(), testFrameworkRootClass);
     }
 
     private static String toString(Suite suite) {
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java
index 487cc8c501a..5bc9fda6835 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java
@@ -18,7 +18,6 @@ import java.time.Clock;
 import java.time.ZoneOffset;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Consumer;
 import java.util.logging.Handler;
 import java.util.logging.Level;
@@ -42,8 +41,8 @@ class TestReportGeneratingListener implements TestExecutionListener {
     private final Handler handler;              // Captures logging from test code.
     private final Clock clock;
 
-    TestReportGeneratingListener(Suite suite, Set<String> testClassNames, Consumer<LogRecord> logger, TeeStream stdoutTee, TeeStream stderrTee, Clock clock) {
-        this.report = new TestReport(clock, suite, testClassNames);
+    TestReportGeneratingListener(Suite suite, Consumer<LogRecord> logger, TeeStream stdoutTee, TeeStream stderrTee, Clock clock) {
+        this.report = new TestReport(clock, suite);
         this.logger = logger;
         this.stdoutTee = stdoutTee;
         this.stderrTee = stderrTee;
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java
index c2f70cda635..842ce89e63a 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java
@@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test;
 public class WrongBeforeAllTest {
 
     @BeforeAll
-    void wrong() { } // Intentionally wrong, to verify this isn't run, but instead causes an error.
+    void wrong() { }
 
     @Test
     void test() { }
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
index 0c55645eb33..2690c89e1c1 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
@@ -12,7 +12,6 @@ import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
@@ -149,6 +148,11 @@ class AggregateTestRunnerTest {
     }
 
     @Test
+    void testReportStatus() {
+        assertEquals(FAILURE, JunitRunner.testRunnerStatus(TestReport.createFailed(Clock.systemUTC(), Suite.SYSTEM_TEST, new RuntimeException("hello"))));
+    }
+
+    @Test
     void testStackTrimming() {
         try {
             try {
@@ -159,22 +163,17 @@ class AggregateTestRunnerTest {
             }
         }
         catch (Exception e) {
-            TestReport.trimStackTraces(e, Set.of(AggregateTestRunnerTest.class.getName()));
+            TestReport.trimStackTraces(e, "org.junit.platform.commons.util.ReflectionUtils");
             assertEquals("""
-                         java.lang.RuntimeException: java.lang.RuntimeException: inner
-                         	at com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:158)
-                         Caused by: java.lang.RuntimeException: inner
-                         	at com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:155)
-                         """,
+                            java.lang.RuntimeException: java.lang.RuntimeException: inner
+                            \tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:162)
+                            Caused by: java.lang.RuntimeException: inner
+                            \tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:159)
+                            """,
                          ExceptionUtils.getStackTraceAsString(e));
         }
     }
 
-    @Test
-    void testReportStatus() {
-        assertEquals(FAILURE, JunitRunner.testRunnerStatus(TestReport.createFailed(Clock.systemUTC(), Suite.SYSTEM_TEST, new RuntimeException("hello"))));
-    }
-
     static class MockTestRunner implements TestRunner {
 
         final List<LogRecord> log = new ArrayList<>();
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
index 6878579b950..09aaeba081f 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
@@ -28,18 +28,13 @@ import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.annotation.Annotation;
-import java.net.URL;
-import java.net.URLClassLoader;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 
 import static ai.vespa.hosted.cd.internal.TestRuntimeProvider.testRuntime;
@@ -92,30 +87,27 @@ class JunitRunnerTest {
     static Suite getSuite(Class<?> testClass) {
         for (Annotation annotation : testClass.getAnnotations()) {
             switch (annotation.annotationType().getSimpleName()) {
-                case "SystemTest" -> { return Suite.SYSTEM_TEST; }
-                case "StagingSetup" -> { return Suite.STAGING_SETUP_TEST; }
-                case "StagingTest" -> { return Suite.STAGING_TEST; }
-                case "ProductionTest" -> { return Suite.PRODUCTION_TEST; }
+                case "SystemTest": return Suite.SYSTEM_TEST;
+                case "StagingSetup": return Suite.STAGING_SETUP_TEST;
+                case "StagingTest": return Suite.STAGING_TEST;
+                case "ProductionTest": return Suite.PRODUCTION_TEST;
             }
         }
         return null;
     }
 
     static TestRunner test(Suite suite, byte[] testConfig, Class<?>... testClasses) {
+        JunitRunner runner = new JunitRunner(clock,
+                                             config -> { assertSame(testConfig, config); testRuntime.set(new MockTestRuntime()); },
+                                             __ -> List.of(testClasses),
+                                             JunitRunnerTest::execute);
         try {
-            JunitRunner runner = new JunitRunner(clock,
-                                                 config -> {
-                                                     assertSame(testConfig, config);
-                                                     testRuntime.set(new MockTestRuntime());
-                                                 },
-                                                 __ -> List.of(testClasses),
-                                                 JunitRunnerTest::execute);
             runner.test(suite, testConfig).get();
-            return runner;
         }
         catch (Exception e) {
-            throw new AssertionError(e);
+            fail(e);
         }
+        return runner;
     }
 
 
@@ -141,7 +133,6 @@ class JunitRunnerTest {
             this.listeners = List.of(listeners);
         }
 
-        @SuppressWarnings("deprecation")
         private TestIdentifier getTestIdentifier(TestDescriptor testDescriptor) {
             return plan.getTestIdentifier(testDescriptor.getUniqueId().toString());
         }
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
index 9edd9a23c68..6d6fbbf2cf1 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
@@ -19,15 +19,10 @@ import org.junit.jupiter.api.Test;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLClassLoader;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
@@ -52,18 +47,16 @@ class TestRunnerHandlerTest {
     private TestRunner aggregateRunner;
 
     @BeforeEach
-    void setup() throws Exception {
+    void setup() {
         TestReport moreTestsReport = JunitRunnerTest.test(Suite.PRODUCTION_TEST,
                                                           new byte[0],
                                                           FailingTestAndBothAftersTest.class,
                                                           WrongBeforeAllTest.class,
                                                           FailingExtensionTest.class)
                                                     .getReport();
-        Exception cnfe = new ClassNotFoundException("School's out all summer!");
-        cnfe.setStackTrace(Arrays.copyOf(cnfe.getStackTrace(), 1));
         TestReport failedReport = TestReport.createFailed(Clock.fixed(testInstant, ZoneId.of("UTC")),
                                                           Suite.PRODUCTION_TEST,
-                                                          cnfe);
+                                                          new ClassNotFoundException("School's out all summer!"));
         aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS,
                                                                         AggregateTestRunnerTest.report.mergedWith(moreTestsReport)
                                                                                                       .mergedWith(failedReport))));
diff --git a/vespa-osgi-testrunner/src/test/resources/output.json b/vespa-osgi-testrunner/src/test/resources/output.json
index f98c50fb841..2b4aa9e5599 100644
--- a/vespa-osgi-testrunner/src/test/resources/output.json
+++ b/vespa-osgi-testrunner/src/test/resources/output.json
@@ -34,7 +34,7 @@
       "id": 11,
       "at": 0,
       "type": "error",
-      "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n\tat org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n\tat org.junit.jupiter.api.AssertEquals.failNotEqual(AssertEquals.java:197)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n"
+      "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n"
     },
     {
       "id": 15,
@@ -94,7 +94,7 @@
       "id": 34,
       "at": 0,
       "type": "error",
-      "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n"
+      "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n"
     },
     {
       "id": 38,
@@ -106,7 +106,7 @@
       "id": 67,
       "at": 0,
       "type": "error",
-      "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:34)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n"
+      "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n"
     },
     {
       "id": 69,
@@ -118,7 +118,7 @@
       "id": 72,
       "at": 0,
       "type": "error",
-      "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n\tat java.base/java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092)\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n"
+      "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n"
     },
     {
       "id": 76,
@@ -130,7 +130,7 @@
       "id": 80,
       "at": 12000,
       "type": "error",
-      "message": "java.lang.ClassNotFoundException: School's out all summer!\n\tat com.yahoo.vespa.testrunner.TestRunnerHandlerTest.setup(TestRunnerHandlerTest.java:62)\n"
+      "message": "java.lang.ClassNotFoundException: School's out all summer!\n"
     }
   ]
 }
diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json
index 2a99e69c9ab..66ae6dd398c 100644
--- a/vespa-osgi-testrunner/src/test/resources/report.json
+++ b/vespa-osgi-testrunner/src/test/resources/report.json
@@ -110,7 +110,7 @@
                         "type": "output",
                         "children": [
                           {
-                            "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n\tat org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n\tat org.junit.jupiter.api.AssertEquals.failNotEqual(AssertEquals.java:197)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n",
+                            "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n",
                             "at": 0,
                             "level": "error"
                           }
@@ -265,7 +265,7 @@
                                 "type": "output",
                                 "children": [
                                   {
-                                    "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n",
+                                    "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n",
                                     "at": 0,
                                     "level": "error"
                                   }
@@ -350,7 +350,7 @@
                         "type": "output",
                         "children": [
                           {
-                            "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:34)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n",
+                            "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n",
                             "at": 0,
                             "level": "error"
                           }
@@ -399,7 +399,7 @@
                     "type": "output",
                     "children": [
                       {
-                        "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n\tat java.base/java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092)\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n",
+                        "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n",
                         "at": 0,
                         "level": "error"
                       }
@@ -473,7 +473,7 @@
             "type": "output",
             "children": [
               {
-                "message": "java.lang.ClassNotFoundException: School's out all summer!\n\tat com.yahoo.vespa.testrunner.TestRunnerHandlerTest.setup(TestRunnerHandlerTest.java:62)\n",
+                "message": "java.lang.ClassNotFoundException: School's out all summer!\n",
                 "at": 12000,
                 "level": "error"
               }