Use InconclusiveTestException an dexit code 4 to signal inconclusive tests

8 files changed, 46 insertions, 15 deletions

diff --git a/tenant-cd-api/abi-spec.json b/tenant-cd-api/abi-spec.json
index 2e277a42378..4840943b7b1 100644
--- a/tenant-cd-api/abi-spec.json
+++ b/tenant-cd-api/abi-spec.json
@@ -29,6 +29,17 @@
     ],
     "fields": []
   },
+  "ai.vespa.hosted.cd.InconclusiveTestException": {
+    "superClass": "java.lang.RuntimeException",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()"
+    ],
+    "fields": []
+  },
   "ai.vespa.hosted.cd.IntegrationTest": {
     "superClass": "java.lang.Object",
     "interfaces": [
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java
index 82c1f7194d0..5637d25df71 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java
@@ -52,12 +52,14 @@ public class AggregateTestRunner implements TestRunner {
             return Status.NOT_STARTED;
 
         boolean failed = false;
+        boolean inconclusive = false;
         for (int i = 0; i <= current.get(); i++) {
             if (i == wrapped.size())
-                return failed ? Status.FAILURE : Status.SUCCESS;
+                return failed ? Status.FAILURE : inconclusive ? Status.INCONCLUSIVE : Status.SUCCESS;
 
             switch (wrapped.get(i).getStatus()) {
                 case ERROR: return Status.ERROR;
+                case INCONCLUSIVE: inconclusive = true; break;
                 case FAILURE: failed = true;
             }
         }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index 6aa36c62416..837b298d76f 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -5,6 +5,7 @@ import ai.vespa.cloud.Environment;
 import ai.vespa.cloud.SystemInfo;
 import ai.vespa.cloud.Zone;
 import ai.vespa.hosted.api.TestDescriptor;
+import ai.vespa.hosted.cd.InconclusiveTestException;
 import ai.vespa.hosted.cd.internal.TestRuntimeProvider;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
@@ -204,11 +205,13 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         var failures = report.getFailures().stream()
                 .map(failure -> new TestReport.Failure(failure.getTestIdentifier().getUniqueId(), failure.getException()))
                 .collect(Collectors.toList());
+        long inconclusive = failures.stream().filter(failure -> failure.exception() instanceof InconclusiveTestException).count();
         return TestReport.builder()
                 .withSuccessCount(report.getTestsSucceededCount())
                 .withAbortedCount(report.getTestsAbortedCount())
                 .withIgnoredCount(report.getTestsSkippedCount())
-                .withFailedCount(report.getTestsFailedCount())
+                .withFailedCount(report.getTestsFailedCount() - inconclusive)
+                .withInconclusiveCount(inconclusive)
                 .withFailures(failures)
                 .withLogs(logRecords.values())
                 .build();
@@ -230,12 +233,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         if (execution == null) return TestRunner.Status.NOT_STARTED;
         if (!execution.isDone()) return TestRunner.Status.RUNNING;
         try {
-            TestReport report = execution.get();
-            if (report.isSuccess()) {
-                return TestRunner.Status.SUCCESS;
-            } else {
-                return TestRunner.Status.FAILURE;
-            }
+            return execution.get().status();
         } catch (InterruptedException|ExecutionException e) {
             logger.log(Level.WARNING, "Error while getting test report", e);
             return TestRunner.Status.ERROR;
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
index b3ca47e2480..d01e9fffe9f 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
@@ -6,6 +6,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.logging.LogRecord;
 
+import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
+
 /**
  * @author mortent
  */
@@ -14,15 +18,17 @@ public class TestReport {
     final long totalCount;
     final long successCount;
     final long failedCount;
+    final long inconclusiveCount;
     final long ignoredCount;
     final long abortedCount;
     final List<Failure> failures;
     final List<LogRecord> logLines;
 
-    private TestReport(long totalCount, long successCount, long failedCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) {
+    private TestReport(long totalCount, long successCount, long failedCount, long inconclusiveCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) {
         this.totalCount = totalCount;
         this.successCount = successCount;
         this.failedCount = failedCount;
+        this.inconclusiveCount = inconclusiveCount;
         this.ignoredCount = ignoredCount;
         this.abortedCount = abortedCount;
         this.failures = failures;
@@ -33,8 +39,8 @@ public class TestReport {
         return logLines;
     }
 
-    public boolean isSuccess() {
-        return (failedCount + abortedCount) == 0;
+    public TestRunner.Status status() {
+        return failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : SUCCESS;
     }
 
     public static Builder builder(){
@@ -47,13 +53,14 @@ public class TestReport {
         private long totalCount;
         private long successCount;
         private long failedCount;
+        private long inconclusiveCount;
         private long ignoredCount;
         private long abortedCount;
         private List<Failure> failures = Collections.emptyList();
         private List<LogRecord> logLines = Collections.emptyList();
 
         public TestReport build() {
-            return new TestReport(totalCount, successCount, failedCount, ignoredCount, abortedCount, failures, logLines);
+            return new TestReport(totalCount, successCount, failedCount, inconclusiveCount, ignoredCount, abortedCount, failures, logLines);
         }
 
         public Builder withTotalCount(long totalCount) {
@@ -71,6 +78,11 @@ public class TestReport {
             return this;
         }
 
+        public Builder withInconclusiveCount(long failedCount) {
+            this.failedCount = failedCount;
+            return this;
+        }
+
         public Builder withIgnoredCount(long ignoredCount) {
             this.ignoredCount = ignoredCount;
             return this;
@@ -95,6 +107,7 @@ public class TestReport {
 
 
     public static class Failure {
+
         private final String testId;
         private final Throwable exception;
 
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
index d70a3f60c7d..20ff19266cf 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
@@ -22,7 +22,7 @@ public interface TestRunner {
     default TestReport getReport() { return null; }
 
     enum Status {
-        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
+        NOT_STARTED, RUNNING, FAILURE, INCONCLUSIVE, ERROR, SUCCESS
     }
 
     enum Suite {
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
index a383d435a26..e7b0fe406c7 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
@@ -27,7 +27,7 @@ import static com.yahoo.jdisc.Response.Status;
 
 /**
  * @author valerijf
- * @author jvenstad
+ * @author jonmv
  * @author mortent
  */
 public class TestRunnerHandler extends ThreadedHttpRequestHandler {
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
index 3cccafb4a8b..9ae75fbae43 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
@@ -26,6 +26,7 @@ import java.util.stream.Stream;
 
 import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
 import static com.yahoo.yolean.Exceptions.uncheck;
@@ -88,7 +89,8 @@ public class VespaCliTestRunner implements TestRunner {
             HtmlLogger htmlLogger = new HtmlLogger();
             BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
             in.lines().forEach(line -> log(htmlLogger.toLog(line)));
-            status.set(process.waitFor() == 0 ? SUCCESS : process.waitFor() == 3 ? FAILURE : ERROR);
+            int exitCode = process.waitFor();
+            status.set(exitCode == 0 ? SUCCESS : exitCode == 3 ? FAILURE : exitCode == 4 ? INCONCLUSIVE : ERROR);
         }
         catch (NoTestsException e) {
             log(Level.WARNING, "Did not find expected basic HTTP tests", e);
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
index 64f9079643a..a7ddbbf3fcc 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
@@ -12,6 +12,7 @@ import java.util.logging.LogRecord;
 
 import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING;
 import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
@@ -85,6 +86,10 @@ class AggregateTestRunnerTest {
         second.future.complete(null);
         assertEquals(SUCCESS, runner.getStatus());
 
+        // An inconclusive test means inconclusive.
+        first.status = INCONCLUSIVE;
+        assertEquals(INCONCLUSIVE, runner.getStatus());
+
         // A failure means failure.
         second.status = FAILURE;
         assertEquals(FAILURE, runner.getStatus());