Include more tests in sample response

2 files changed, 137 insertions, 13 deletions

diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
index c0843cfff8e..1d723699427 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
@@ -5,9 +5,11 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.test.samples.FailingExtensionTest;
+import com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest;
+import com.yahoo.vespa.test.samples.WrongBeforeAllTest;
 import com.yahoo.vespa.testrunner.TestReport.Node;
 import com.yahoo.vespa.testrunner.TestReport.OutputNode;
 import com.yahoo.vespa.testrunner.TestRunner.Status;
@@ -18,18 +20,16 @@ import org.junit.jupiter.api.Test;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.time.Clock;
 import java.time.Instant;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
-import java.util.stream.Collectors;
 
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
 import static com.yahoo.slime.SlimeUtils.jsonToSlimeOrThrow;
@@ -49,11 +49,18 @@ class TestRunnerHandlerTest {
 
     @BeforeEach
     void setup() {
-        List<LogRecord> logRecords = List.of(logRecord("Tests started"));
-        Throwable exception = new RuntimeException("org.junit.ComparisonFailure: expected:<foo> but was:<bar>");
-        exception.setStackTrace(new StackTraceElement[]{new StackTraceElement("Foo", "bar", "Foo.java", 1123)});
-
-        aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS, AggregateTestRunnerTest.report)));
+        TestReport moreTestsReport = JunitRunnerTest.test(Suite.PRODUCTION_TEST,
+                                                          new byte[0],
+                                                          FailingTestAndBothAftersTest.class,
+                                                          WrongBeforeAllTest.class,
+                                                          FailingExtensionTest.class)
+                                                    .getReport();
+        TestReport failedReport = TestReport.createFailed(Clock.fixed(testInstant, ZoneId.of("UTC")),
+                                                          Suite.PRODUCTION_TEST,
+                                                          new ClassNotFoundException("School's out all summer!"));
+        aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS,
+                                                                        AggregateTestRunnerTest.report.mergedWith(moreTestsReport)
+                                                                                                      .mergedWith(failedReport))));
         testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner);
     }
 
@@ -91,7 +98,6 @@ class TestRunnerHandlerTest {
                          new String(toJsonBytes(actualRoot, false), UTF_8));
 
         // Should not get old log
-
         long last = SlimeUtils.entriesStream(jsonToSlimeOrThrow (readTestResource("/output.json")).get().field("logRecords"))
                               .mapToLong(recordObject -> recordObject.field("id").asLong())
                               .max()
diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json
index 16c491ef61c..281665d18a7 100644
--- a/vespa-osgi-testrunner/src/test/resources/report.json
+++ b/vespa-osgi-testrunner/src/test/resources/report.json
@@ -262,13 +262,106 @@
             ]
           }
         ]
+      },
+      {
+        "type": "container",
+        "name": "JUnit Jupiter",
+        "status": "error",
+        "start": 0,
+        "end": 0,
+        "children": [
+          {
+            "type": "container",
+            "name": "FailingTestAndBothAftersTest",
+            "status": "error",
+            "start": 0,
+            "end": 0,
+            "children": [
+              {
+                "type": "test",
+                "name": "test()",
+                "status": "failed",
+                "start": 0,
+                "end": 0,
+                "children": [
+                  {
+                    "type": "failure",
+                    "status": "failed",
+                    "trace": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n"
+                  }
+                ]
+              },
+              {
+                "type": "failure",
+                "status": "error",
+                "trace": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n"
+              }
+            ]
+          },
+          {
+            "type": "container",
+            "name": "WrongBeforeAllTest",
+            "status": "error",
+            "start": 0,
+            "end": 0,
+            "children": [
+              {
+                "type": "failure",
+                "status": "error",
+                "trace": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n"
+              },
+              {
+                "type": "test",
+                "name": "fest()",
+                "status": "skipped",
+                "start": 0,
+                "end": 0
+              },
+              {
+                "type": "test",
+                "name": "test()",
+                "status": "skipped",
+                "start": 0,
+                "end": 0
+              }
+            ]
+          },
+          {
+            "type": "container",
+            "name": "FailingExtensionTest",
+            "status": "error",
+            "start": 0,
+            "end": 0,
+            "children": [
+              {
+                "type": "test",
+                "name": "test()",
+                "status": "error",
+                "start": 0,
+                "end": 0,
+                "children": [
+                  {
+                    "type": "failure",
+                    "status": "error",
+                    "trace": "java.lang.NullPointerException\n\tat java.base/java.util.Objects.requireNonNull(Objects.java:208)\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "failure",
+        "status": "error",
+        "trace": "java.lang.ClassNotFoundException: School's out all summer!\n"
       }
     ]
   },
   "summary": {
     "success": 3,
-    "failed": 3,
-    "ignored": 2,
+    "failed": 5,
+    "ignored": 4,
     "aborted": 1,
     "inconclusive": 1,
     "failures": [
@@ -291,6 +384,31 @@
         "testName": "third",
         "testError": "no charm",
         "exception": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n"
+      },
+      {
+        "testName": "test()",
+        "testError": "",
+        "exception": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n"
+      },
+      {
+        "testName": "FailingTestAndBothAftersTest",
+        "testError": null,
+        "exception": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n"
+      },
+      {
+        "testName": "WrongBeforeAllTest",
+        "testError": "@BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).",
+        "exception": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n"
+      },
+      {
+        "testName": "test()",
+        "testError": null,
+        "exception": "java.lang.NullPointerException\n\tat java.base/java.util.Objects.requireNonNull(Objects.java:208)\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n"
+      },
+      {
+        "testName": "Production test",
+        "testError": "School's out all summer!",
+        "exception": "java.lang.ClassNotFoundException: School's out all summer!\n"
       }
     ]
   },