Ensure errors outside test methods count as failure

4 files changed, 30 insertions, 6 deletions

diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml
index 0fba7d371aa..4fa651a6930 100644
--- a/vespa-osgi-testrunner/pom.xml
+++ b/vespa-osgi-testrunner/pom.xml
@@ -85,6 +85,13 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <excludedGroups>integration</excludedGroups>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
index bcb865c55ff..c7f4620693b 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
@@ -40,7 +40,7 @@ public class TestReport {
     }
 
     public TestRunner.Status status() {
-        return failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : successCount > 0 ? SUCCESS : NO_TESTS;
+        return failures.size() > 0 || failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : successCount > 0 ? SUCCESS : NO_TESTS;
     }
 
     public static Builder builder(){
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java
new file mode 100644
index 00000000000..4c4e5bf3e36
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java
@@ -0,0 +1,14 @@
+package com.yahoo.vespa.test.samples;
+
+import ai.vespa.hosted.cd.SystemTest;
+import org.junit.jupiter.api.Test;
+
+@SystemTest
+public class FailingInstantiationTest {
+
+    final int i = Integer.parseInt("");
+
+    @Test
+    void test() { }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
index 42be19b0f76..64a4c3dbc80 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java
@@ -1,6 +1,7 @@
 package com.yahoo.vespa.testrunner;
 
-import com.yahoo.vespa.testrunner.TestRunner.Suite;
+import com.yahoo.vespa.test.samples.FailingInstantiationTest;
+import com.yahoo.vespa.testrunner.TestRunner.Status;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.engine.JupiterTestEngine;
 import org.junit.platform.engine.EngineExecutionListener;
@@ -33,12 +34,14 @@ class JunitRunnerTest {
     void test() throws ExecutionException, InterruptedException {
         AtomicReference<byte[]> testRuntime = new AtomicReference<>();
         JunitRunner runner = new JunitRunner(testRuntime::set,
-                                            __ -> List.of(HtmlLoggerTest.class),
+                                            __ -> List.of(FailingInstantiationTest.class),
                                             this::execute);
 
-        runner.test(Suite.SYSTEM_TEST, new byte[0]).get();
-        assertEquals(1, runner.getReport().successCount);
-        assertEquals(0, runner.getReport().failedCount);
+        runner.test(null, null).get();
+        assertEquals(Status.FAILURE, runner.getStatus());
+        assertEquals(0, runner.getReport().successCount);
+        assertEquals("java.lang.NumberFormatException: For input string: \"\"",
+                     runner.getReport().failures.get(0).exception().toString());
     }