Reimplement JUnit integratino, with unit tests, and structured report

30 files changed, 766 insertions, 66 deletions

diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java
new file mode 100644
index 00000000000..795bf8c6a1e
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java
@@ -0,0 +1,17 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@Disabled("because")
+@Expect(skipped = 2, status = 1)
+public class DisabledClassTest {
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java
new file mode 100644
index 00000000000..561ec81e865
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java
@@ -0,0 +1,14 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@Expect(skipped = 1, status = 1)
+public class DisabledTest {
+
+    @Test
+    @Disabled("because")
+    void test() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterAllTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterAllTest.java
new file mode 100644
index 00000000000..4c7132fc01a
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterAllTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+
+@Expect(successful = 2, status = 5)
+public class FailingAfterAllTest {
+
+    @AfterAll
+    static void fail() { throw new RuntimeException(); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterEachTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterEachTest.java
new file mode 100644
index 00000000000..b1ec3cb13fd
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAfterEachTest.java
@@ -0,0 +1,16 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+@Expect(error = 1, status = 5)
+public class FailingAfterEachTest {
+
+    @AfterEach
+    void fail() { throw new RuntimeException(); }
+
+    @Test
+    void test() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssertionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssertionTest.java
new file mode 100644
index 00000000000..4dd8be898ec
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssertionTest.java
@@ -0,0 +1,13 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@Expect(failed = 1, status = 4)
+public class FailingAssertionTest {
+
+    @Test
+    void fail() { Assertions.fail(); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssumptionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssumptionTest.java
new file mode 100644
index 00000000000..1b542a7dd7d
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingAssumptionTest.java
@@ -0,0 +1,14 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+@Expect(aborted = 1, status = 2)
+public class FailingAssumptionTest {
+
+    @Test
+    void test() { assumeTrue(false, "assumption"); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllAssertionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllAssertionTest.java
new file mode 100644
index 00000000000..15e67f2c51c
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllAssertionTest.java
@@ -0,0 +1,20 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@Expect(skipped = 2, status = 4)
+public class FailingBeforeAllAssertionTest {
+
+    @BeforeAll
+    static void fail() { Assertions.fail(); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTest.java
new file mode 100644
index 00000000000..ae26b3fd038
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@Expect(skipped = 2, status = 5)
+public class FailingBeforeAllTest {
+
+    @BeforeAll
+    static void fail() { throw new RuntimeException(); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTestFactoryTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTestFactoryTest.java
new file mode 100644
index 00000000000..89457b145c9
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeAllTestFactoryTest.java
@@ -0,0 +1,26 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestFactory;
+
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+@Expect(skipped = 1, status = 5)
+public class FailingBeforeAllTestFactoryTest {
+
+    @BeforeAll
+    static void fail() { throw new RuntimeException(); }
+
+    @TestFactory
+    Stream<DynamicTest> tests() {
+        return IntStream.range(0, 3).mapToObj(i -> DynamicTest.dynamicTest("test-" + i, () -> { }));
+    }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeEachTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeEachTest.java
new file mode 100644
index 00000000000..5e5ebe47c99
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingBeforeEachTest.java
@@ -0,0 +1,16 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+@Expect(error = 1, status = 5)
+public class FailingBeforeEachTest {
+
+    @BeforeEach
+    void fail() { throw new RuntimeException(); }
+
+    @Test
+    void test() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassAssumptionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassAssumptionTest.java
new file mode 100644
index 00000000000..2a1085c3db3
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassAssumptionTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Test;
+
+@Expect(aborted = 2, status = 2)
+public class FailingClassAssumptionTest {
+
+    { Assumptions.assumeTrue(false, "assumption"); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
+
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassLoadingTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassLoadingTest.java
new file mode 100644
index 00000000000..fc2b33ee03f
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingClassLoadingTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@Expect(failed = 1, error = 1, status = 5)
+public class FailingClassLoadingTest {
+
+    static { Assertions.fail(); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
+
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingExtensionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingExtensionTest.java
new file mode 100644
index 00000000000..c7dba666daf
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingExtensionTest.java
@@ -0,0 +1,23 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+
+import static java.util.Objects.requireNonNull;
+
+@Expect(error = 1, status = 5)
+public class FailingExtensionTest {
+
+    @Test
+    @ExtendWith(FailingExtension.class)
+    void test() { }
+
+    static class FailingExtension implements Extension {
+
+        { requireNonNull(null); }
+
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInnerClassTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInnerClassTest.java
new file mode 100644
index 00000000000..85ed49fcf0b
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInnerClassTest.java
@@ -0,0 +1,28 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+@Expect(successful = 1, failed = 1, status = 4)
+public class FailingInnerClassTest {
+
+    @Nested
+    class Failing {
+
+        @Test
+        void test() { fail(); }
+
+    }
+
+    @Nested
+    class Succeeding {
+
+        @Test
+        void test() { }
+
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationAssertionTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationAssertionTest.java
new file mode 100644
index 00000000000..87c19872f13
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationAssertionTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+@Expect(failed = 2, status = 4)
+public class FailingInstantiationAssertionTest {
+
+    { fail(); }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java
index 4c4e5bf3e36..50e0c6a43b7 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java
@@ -1,9 +1,9 @@
 package com.yahoo.vespa.test.samples;
 
-import ai.vespa.hosted.cd.SystemTest;
+import com.yahoo.vespa.testrunner.Expect;
 import org.junit.jupiter.api.Test;
 
-@SystemTest
+@Expect(error = 2, status = 5)
 public class FailingInstantiationTest {
 
     final int i = Integer.parseInt("");
@@ -11,4 +11,7 @@ public class FailingInstantiationTest {
     @Test
     void test() { }
 
+    @Test
+    void fest() { }
+
 }
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTest.java
new file mode 100644
index 00000000000..f2a65c58728
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTest.java
@@ -0,0 +1,12 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+@Expect(error = 1, status = 5)
+public class FailingTest {
+
+    @Test
+    void test() { throw new RuntimeException(); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestAndBothAftersTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestAndBothAftersTest.java
new file mode 100644
index 00000000000..5ca1f43b976
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestAndBothAftersTest.java
@@ -0,0 +1,21 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@Expect(failed = 1, status = 5)
+public class FailingTestAndBothAftersTest {
+
+    @AfterAll
+    static void fail() { throw new RuntimeException(); }
+
+    @AfterEach
+    void moreFail() { throw new RuntimeException(); }
+
+    @Test
+    void test() { Assertions.fail(); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestFactoryTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestFactoryTest.java
new file mode 100644
index 00000000000..fa7a39eea7d
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingTestFactoryTest.java
@@ -0,0 +1,26 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestFactory;
+
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+@Expect(skipped = 1, status = 5)
+public class FailingTestFactoryTest {
+
+    @BeforeAll
+    static void fail() { throw new RuntimeException(); }
+
+    @TestFactory
+    Stream<DynamicTest> tests() {
+        return IntStream.range(0, 3).mapToObj(i -> DynamicTest.dynamicTest("test-" + i, () -> { throw new RuntimeException("error"); }));
+    }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/InconclusiveTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/InconclusiveTest.java
new file mode 100644
index 00000000000..868568e8bb5
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/InconclusiveTest.java
@@ -0,0 +1,15 @@
+package com.yahoo.vespa.test.samples;
+
+import ai.vespa.hosted.cd.InconclusiveTestException;
+import ai.vespa.hosted.cd.ProductionTest;
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+@ProductionTest
+@Expect(inconclusive = 1, status = 3)
+public class InconclusiveTest {
+
+    @Test
+    void test() { throw new InconclusiveTestException("soon"); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/NotInconclusiveTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/NotInconclusiveTest.java
new file mode 100644
index 00000000000..fea1e827260
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/NotInconclusiveTest.java
@@ -0,0 +1,13 @@
+package com.yahoo.vespa.test.samples;
+
+import ai.vespa.hosted.cd.InconclusiveTestException;
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+@Expect(error = 1, status = 5)
+public class NotInconclusiveTest {
+
+    @Test
+    void test() { throw new InconclusiveTestException("soon"); }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java
new file mode 100644
index 00000000000..bc878353d4b
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java
@@ -0,0 +1,119 @@
+package com.yahoo.vespa.test.samples;
+
+import ai.vespa.hosted.cd.InconclusiveTestException;
+import ai.vespa.hosted.cd.ProductionTest;
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.ClassOrderer;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestClassOrder;
+import org.junit.jupiter.api.TestFactory;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.TestReporter;
+
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Stream;
+
+import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.INFO;
+import static java.util.logging.Level.WARNING;
+import static org.junit.jupiter.api.DynamicTest.dynamicTest;
+
+@ProductionTest
+@Expect(successful = 3, skipped = 2, aborted = 1, inconclusive = 1, failed = 2, error = 1, status = 5)
+@TestMethodOrder(MethodOrderer.DisplayName.class)
+@TestClassOrder(ClassOrderer.DisplayName.class)
+public class SampleTest {
+
+    static Handler consoleHandler = null;
+
+    @BeforeAll
+    static void setupLogging() {
+        Handler[] handlers = Logger.getLogger("").getHandlers();
+        for (Handler handler : handlers)
+            if (handler instanceof ConsoleHandler)
+                consoleHandler = handler;
+        Logger.getLogger("").removeHandler(consoleHandler);
+    }
+
+    @AfterAll
+    static void restoreLogging() {
+        Logger.getLogger("").addHandler(consoleHandler);
+    }
+
+    private static final Logger log = Logger.getLogger(SampleTest.class.getName());
+
+    @BeforeEach
+    void spam() {
+        System.err.println("spam");
+    }
+
+    @Test
+    @Disabled("disabled for test purposes")
+    void ignored() { }
+
+    @Test
+    void aborted() {
+        Assumptions.assumeTrue(false, "thou shalt not pass!");
+    }
+
+    @Test
+    void successful() {
+        log.log(new Level("html", INFO.intValue()) { }, "<body />");
+        log.log(INFO, "Very informative");
+        log.log(WARNING, "Oh no", new IllegalArgumentException("error", new RuntimeException("wrapped")));
+    }
+
+    @Test
+    void failing() {
+        log.log(INFO, "I have a bad feeling about this");
+        Assertions.assertEquals("foo", "bar", "baz");
+    }
+
+    @Test
+    void error() {
+        log.log(FINE, "What could possibly go wrong this time?");
+        throw new NoClassDefFoundError();
+    }
+
+    @Test
+    void inconclusive(TestReporter reporter) {
+        reporter.publishEntry("I'm here with Erwin today; Erwin, what can you tell us about your cat?");
+        throw new InconclusiveTestException("the cat is both dead _and_ alive");
+    }
+
+    @Nested
+    class Inner {
+
+        @Test
+        void first() { }
+
+        @TestFactory
+        Stream<DynamicTest> others() {
+            return Stream.of(dynamicTest("second", () -> System.out.println("Catch me if you can!")),
+                             dynamicTest("third", () -> Assertions.fail("no charm")));
+        }
+
+    }
+
+    @Nested
+    @Disabled
+    class Skipped {
+
+        @Test
+        void disabled() { }
+
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java
new file mode 100644
index 00000000000..59a56a1c9c7
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java
@@ -0,0 +1,13 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+@Expect(successful = 1, status = 0)
+public class SucceedingTest {
+
+    @Test
+    void test() { }
+
+}
+
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/TimingOutTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/TimingOutTest.java
new file mode 100644
index 00000000000..b248fe065fb
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/TimingOutTest.java
@@ -0,0 +1,18 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.util.concurrent.TimeUnit;
+
+@Expect(error = 1, status = 5)
+public class TimingOutTest {
+
+    @Test
+    @Timeout(value = 1, unit = TimeUnit.MILLISECONDS)
+    void test() throws InterruptedException {
+        Thread.sleep(10_000);
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java
new file mode 100644
index 00000000000..acb41bb42be
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java
@@ -0,0 +1,22 @@
+package com.yahoo.vespa.test.samples;
+
+import ai.vespa.cloud.Environment;
+import ai.vespa.hosted.cd.TestRuntime;
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@Expect(successful = 1, status = 0)
+public class UsingTestRuntimeTest {
+
+    @Test
+    void testTestRuntime() {
+        TestRuntime runtime = TestRuntime.get();
+        assertEquals(Environment.test, runtime.zone().environment());
+        assertEquals("name", runtime.zone().region());
+        assertNull(runtime.deploymentToTest().endpoint("dummy"));
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java
new file mode 100644
index 00000000000..842ce89e63a
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/WrongBeforeAllTest.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.test.samples;
+
+import com.yahoo.vespa.testrunner.Expect;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@Expect(skipped = 2, status = 5)
+public class WrongBeforeAllTest {
+
+    @BeforeAll
+    void wrong() { }
+
+    @Test
+    void test() { }
+
+    @Test
+    void fest() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
index 8bd72d35737..2245f8db68e 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
@@ -2,9 +2,13 @@
 package com.yahoo.vespa.testrunner;
 
 import com.yahoo.exception.ExceptionUtils;
-import com.yahoo.vespa.testrunner.TestReport.Failure;
+import com.yahoo.vespa.test.samples.SampleTest;
+import com.yahoo.vespa.testrunner.TestReport.Node;
+import com.yahoo.vespa.testrunner.TestReport.Status;
+import com.yahoo.vespa.testrunner.TestRunner.Suite;
 import org.junit.jupiter.api.Test;
 
+import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -32,6 +36,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 class AggregateTestRunnerTest {
 
+    static final TestReport report = JunitRunnerTest.test(Suite.SYSTEM_TEST, new byte[0], SampleTest.class).getReport();
+
     @Test
     void onlySupportedRunnersAreUsed() {
         MockTestRunner unsupported = new MockTestRunner();
@@ -125,15 +131,6 @@ class AggregateTestRunnerTest {
         // Verify reports are merged.
         assertNull(runner.getReport());
 
-        Failure failure = new Failure("test", null);
-        TestReport report = TestReport.builder()
-                                      .withLogs(List.of(record1))
-                                      .withFailures(List.of(failure))
-                                      .withSuccessCount(8)
-                                      .withIgnoredCount(4)
-                                      .withFailedCount(2)
-                                      .withAbortedCount(1)
-                                      .build();
         first.report = report;
         assertSame(report, runner.getReport());
         second.report = report;
@@ -141,24 +138,19 @@ class AggregateTestRunnerTest {
 
         second.future.complete(null);
         TestReport merged = runner.getReport();
-        assertEquals(List.of(record1, record1), merged.logLines);
-        assertEquals(List.of(failure, failure), merged.failures);
-        assertEquals(16, merged.successCount);
-        assertEquals(8, merged.ignoredCount);
-        assertEquals(4, merged.failedCount);
-        assertEquals(2, merged.abortedCount);
-
+        List<Node> expected = new ArrayList<>(first.report.root().children());
+        expected.addAll(second.report.root().children());
+        assertEquals(expected, new ArrayList<>(merged.root().children()));
+
+        for (Status status : Status.values())
+            assertEquals(  first.report.root().tally().getOrDefault(status, 0L)
+                         + second.report.root().tally().getOrDefault(status, 0L),
+                         merged.root().tally().getOrDefault(status, 0L));
     }
 
     @Test
     void testReportStatus() {
-        assertEquals(NO_TESTS, TestReport.builder().build().status());
-        assertEquals(SUCCESS, TestReport.builder().withSuccessCount(1).build().status());
-        assertEquals(INCONCLUSIVE, TestReport.builder().withSuccessCount(1).withInconclusiveCount(1).build().status());
-        assertEquals(FAILURE, TestReport.builder().withSuccessCount(1).withFailedCount(1).build().status());
-        assertEquals(NO_TESTS, TestReport.builder().withAbortedCount(1).build().status());
-        assertEquals(NO_TESTS, TestReport.builder().withIgnoredCount(1).build().status());
-        assertEquals(FAILURE, JunitRunner.createReportWithFailedInitialization(new RuntimeException("hello")).status());
+        assertEquals(FAILURE, JunitRunner.testRunnerStatus(TestReport.createFailed(Clock.systemUTC(), Suite.SYSTEM_TEST, new RuntimeException("hello"))));
     }
 
     @Test
@@ -172,11 +164,11 @@ class AggregateTestRunnerTest {
             }
         }
         catch (Exception e) {
-            TestReport.trimStackTraces(e, "org.junit.platform.launcher.core.SessionPerRequestLauncher");
+            TestReport.trimStackTraces(e, "org.junit.platform.commons.util.ReflectionUtils");
             assertEquals("java.lang.RuntimeException: java.lang.RuntimeException: inner\n" +
-                         "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:171)\n" +
+                         "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:163)\n" +
                          "Caused by: java.lang.RuntimeException: inner\n" +
-                         "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:168)\n",
+                         "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:160)\n",
                          ExceptionUtils.getStackTraceAsString(e));
         }
     }
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/Expect.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/Expect.java
new file mode 100644
index 00000000000..88278b3feb6
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/Expect.java
@@ -0,0 +1,29 @@
+package com.yahoo.vespa.testrunner;
+
+import org.junit.jupiter.api.Tag;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * @author jonmv
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("integration")
+public @interface Expect {
+
+    int status();
+
+    long aborted() default 0;
+
+    long skipped() default 0;
+
+    long successful() default 0;
+
+    long inconclusive() default 0;
+
+    long failed() default 0;
+
+    long error() default 0;
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
index 64a4c3dbc80..8e330b0228e 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
@@ -1,8 +1,13 @@
 package com.yahoo.vespa.testrunner;
 
-import com.yahoo.vespa.test.samples.FailingInstantiationTest;
-import com.yahoo.vespa.testrunner.TestRunner.Status;
-import org.junit.jupiter.api.Test;
+import ai.vespa.cloud.Environment;
+import ai.vespa.cloud.Zone;
+import ai.vespa.hosted.cd.Deployment;
+import ai.vespa.hosted.cd.TestRuntime;
+import com.yahoo.vespa.testrunner.TestReport.Status;
+import com.yahoo.vespa.testrunner.TestRunner.Suite;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.TestFactory;
 import org.junit.jupiter.engine.JupiterTestEngine;
 import org.junit.platform.engine.EngineExecutionListener;
 import org.junit.platform.engine.TestDescriptor;
@@ -17,12 +22,25 @@ import org.junit.platform.launcher.core.EngineDiscoveryOrchestrator;
 import org.junit.platform.launcher.core.EngineExecutionOrchestrator;
 import org.junit.platform.launcher.core.LauncherDiscoveryResult;
 
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.annotation.Annotation;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Stream;
 
+import static ai.vespa.hosted.cd.internal.TestRuntimeProvider.testRuntime;
+import static java.util.Objects.requireNonNull;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.DynamicTest.dynamicTest;
 import static org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.Phase.EXECUTION;
 
 /**
@@ -30,24 +48,70 @@ import static org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.Phase
  */
 class JunitRunnerTest {
 
-    @Test
-    void test() throws ExecutionException, InterruptedException {
-        AtomicReference<byte[]> testRuntime = new AtomicReference<>();
-        JunitRunner runner = new JunitRunner(testRuntime::set,
-                                            __ -> List.of(FailingInstantiationTest.class),
-                                            this::execute);
-
-        runner.test(null, null).get();
-        assertEquals(Status.FAILURE, runner.getStatus());
-        assertEquals(0, runner.getReport().successCount);
-        assertEquals("java.lang.NumberFormatException: For input string: \"\"",
-                     runner.getReport().failures.get(0).exception().toString());
+    static final Clock clock = Clock.fixed(Instant.EPOCH, ZoneId.of("UTC"));
+
+    @TestFactory
+    Stream<DynamicTest> runSampleTests() {
+        String packageName = "com.yahoo.vespa.test.samples";
+        InputStream classes = getClass().getClassLoader().getResourceAsStream(packageName.replace(".", "/"));
+        BufferedReader reader = new BufferedReader(new InputStreamReader(requireNonNull(classes, packageName + " should contain sample tests")));
+        return reader.lines()
+                     .filter(line -> line.endsWith("Test.class"))
+                     .map(name -> {
+                         try {
+                             Class<?> testClass = getClass().getClassLoader().loadClass(packageName + "." + name.replace(".class", ""));
+                             return dynamicTest(testClass.getSimpleName(), () -> verify(testClass));
+                         }
+                         catch (ClassNotFoundException e) {
+                             throw new IllegalStateException(e);
+                         }
+                     });
+    }
+
+    static void verify(Class<?> testClass) {
+        Expect expected = requireNonNull(testClass.getAnnotation(Expect.class), "sample tests must be annotated with @Expect");
+        TestReport report = test(getSuite(testClass), new byte[0], testClass).getReport();
+        assertEquals(Status.values()[expected.status()], report.root().status());
+        Map<Status, Long> tally = new EnumMap<>(Status.class);
+        if (expected.successful() > 0) tally.put(Status.successful, expected.successful());
+        if (expected.skipped() > 0) tally.put(Status.skipped, expected.skipped());
+        if (expected.aborted() > 0) tally.put(Status.aborted, expected.aborted());
+        if (expected.inconclusive() > 0) tally.put(Status.inconclusive, expected.inconclusive());
+        if (expected.failed() > 0) tally.put(Status.failed, expected.failed());
+        if (expected.error() > 0) tally.put(Status.error, expected.error());
+        assertEquals(tally, report.root().tally());
+    }
+
+    static Suite getSuite(Class<?> testClass) {
+        for (Annotation annotation : testClass.getAnnotations()) {
+            switch (annotation.annotationType().getSimpleName()) {
+                case "SystemTest": return Suite.SYSTEM_TEST;
+                case "StagingSetup": return Suite.STAGING_SETUP_TEST;
+                case "StagingTest": return Suite.STAGING_TEST;
+                case "ProductionTest": return Suite.PRODUCTION_TEST;
+            }
+        }
+        return null;
+    }
+
+    static TestRunner test(Suite suite, byte[] testConfig, Class<?>... testClasses) {
+        JunitRunner runner = new JunitRunner(clock,
+                                             config -> { assertSame(testConfig, config); testRuntime.set(new MockTestRuntime()); },
+                                             __ -> List.of(testClasses),
+                                             JunitRunnerTest::execute);
+        try {
+            runner.test(suite, testConfig).get();
+        }
+        catch (Exception e) {
+            fail(e);
+        }
+        return runner;
     }
 
 
     // For some inane reason, the JUnit test framework makes it impossible to simply launch a new instance of itself
-    // from inside a unit test (run by itself) in the standard way, so all this kludge is necessary to work around that.
-    void execute(LauncherDiscoveryRequest discoveryRequest, TestExecutionListener... listeners) {
+    // from inside a unit test (run by itself) in the standard way, so this kludge is necessary to work around that.
+    static void execute(LauncherDiscoveryRequest discoveryRequest, TestExecutionListener... listeners) {
         TestEngine testEngine = new JupiterTestEngine();
         LauncherDiscoveryResult discoveryResult = new EngineDiscoveryOrchestrator(Set.of(testEngine), Set.of()).discover(discoveryRequest, EXECUTION);
         TestDescriptor engineTestDescriptor = discoveryResult.getEngineTestDescriptor(testEngine);
@@ -100,4 +164,19 @@ class JunitRunnerTest {
 
     }
 
+
+    static class MockTestRuntime implements TestRuntime {
+
+        @Override
+        public Deployment deploymentToTest() {
+            return __ -> null;
+        }
+
+        @Override
+        public Zone zone() {
+            return new Zone(Environment.test, "name");
+        }
+
+    }
+
 }
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
index 49fadebe58b..c0843cfff8e 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
@@ -5,6 +5,11 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.testrunner.TestReport.Node;
+import com.yahoo.vespa.testrunner.TestReport.OutputNode;
 import com.yahoo.vespa.testrunner.TestRunner.Status;
 import com.yahoo.vespa.testrunner.TestRunner.Suite;
 import org.junit.jupiter.api.BeforeEach;
@@ -12,11 +17,16 @@ import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import java.util.stream.Collectors;
@@ -42,16 +52,8 @@ class TestRunnerHandlerTest {
         List<LogRecord> logRecords = List.of(logRecord("Tests started"));
         Throwable exception = new RuntimeException("org.junit.ComparisonFailure: expected:<foo> but was:<bar>");
         exception.setStackTrace(new StackTraceElement[]{new StackTraceElement("Foo", "bar", "Foo.java", 1123)});
-        TestReport testReport = TestReport.builder()
-                .withSuccessCount(1)
-                .withFailedCount(2)
-                .withIgnoredCount(3)
-                .withAbortedCount(4)
-                .withInconclusiveCount(5)
-                .withFailures(List.of(new TestReport.Failure("Foo.bar()", exception)))
-                .withLogs(logRecords).build();
-
-        aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS, testReport)));
+
+        aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS, AggregateTestRunnerTest.report)));
         testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner);
     }
 
@@ -61,7 +63,7 @@ class TestRunnerHandlerTest {
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
-        assertEquals(new String(toJsonBytes(jsonToSlimeOrThrow("{\"summary\":{\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"inconclusive\":5,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"00:00:12.000 Tests started\"]}").get(), false), UTF_8),
+        assertEquals(new String(toJsonBytes(jsonToSlimeOrThrow(readTestResource("/report.json")).get(), false), UTF_8),
                      new String(toJsonBytes(jsonToSlimeOrThrow(out.toByteArray()).get(), false), UTF_8));
     }
 
@@ -73,16 +75,42 @@ class TestRunnerHandlerTest {
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
-        assertEquals(new String(toJsonBytes(jsonToSlimeOrThrow("{\"logRecords\":[{\"id\":0,\"at\":12000,\"type\":\"info\",\"message\":\"Tests started\"}]}").get(), false), UTF_8),
-                     new String(toJsonBytes(jsonToSlimeOrThrow(out.toByteArray()).get(), false), UTF_8));
+        Inspector actualRoot = jsonToSlimeOrThrow(out.toByteArray()).get();
+        Inspector expectedRoot = jsonToSlimeOrThrow(readTestResource("/output.json")).get();
+        boolean ok = expectedRoot.field("logRecords").entries() == actualRoot.field("logRecords").entries();
+        // Need custom comparison, because sequence ID may be influenced by other tests.
+        for (int i = 0; i < expectedRoot.field("logRecords").entries(); i++) {
+            Inspector expectedEntry = expectedRoot.field("logRecords").entry(i);
+            Inspector actualEntry = actualRoot.field("logRecords").entry(i);
+            ok &= expectedEntry.field("at").equalTo(actualEntry.field("at"));
+            ok &= expectedEntry.field("type").equalTo(actualEntry.field("type"));
+            ok &= expectedEntry.field("message").equalTo(actualEntry.field("message"));
+        }
+        if ( ! ok)
+            assertEquals(new String(toJsonBytes(expectedRoot, false), UTF_8),
+                         new String(toJsonBytes(actualRoot, false), UTF_8));
 
         // Should not get old log
-        response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log?after=0", GET));
+
+        long last = SlimeUtils.entriesStream(jsonToSlimeOrThrow (readTestResource("/output.json")).get().field("logRecords"))
+                              .mapToLong(recordObject -> recordObject.field("id").asLong())
+                              .max()
+                              .orElse(0L);
+        response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log?after=" + last, GET));
         out = new ByteArrayOutputStream();
         response.render(out);
         assertEquals("{\"logRecords\":[]}", out.toString(UTF_8));
     }
 
+    static byte[] readTestResource(String name) {
+        try {
+            return TestRunnerHandlerTest.class.getResourceAsStream(name).readAllBytes();
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     @Test
     public void returnsEmptyResponsesWhenReportNotReady() throws IOException {
         testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(),
@@ -130,10 +158,19 @@ class TestRunnerHandlerTest {
 
         @Override
         public Collection<LogRecord> getLog(long after) {
-            return getReport() == null ? List.of()
-                                       : getReport().logLines().stream()
-                                                    .filter(entry -> entry.getSequenceNumber() > after)
-                                                    .collect(Collectors.toList());
+            List<LogRecord> log = new ArrayList<>();
+            if (testReport != null) addLog(log, testReport.root(), after);
+            return log;
+        }
+
+        private void addLog(List<LogRecord> log, Node node, long after) {
+            if (node instanceof OutputNode)
+                for (LogRecord record : ((OutputNode) node).log())
+                    if (record.getSequenceNumber() > after)
+                        log.add(record);
+
+            for (Node child : node.children())
+                addLog(log, child, after);
         }
 
         @Override