diff options
Diffstat (limited to 'vespa-osgi-testrunner')
7 files changed, 35 insertions, 15 deletions
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java index 82c1f7194d0..5637d25df71 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java @@ -52,12 +52,14 @@ public class AggregateTestRunner implements TestRunner { return Status.NOT_STARTED; boolean failed = false; + boolean inconclusive = false; for (int i = 0; i <= current.get(); i++) { if (i == wrapped.size()) - return failed ? Status.FAILURE : Status.SUCCESS; + return failed ? Status.FAILURE : inconclusive ? Status.INCONCLUSIVE : Status.SUCCESS; switch (wrapped.get(i).getStatus()) { case ERROR: return Status.ERROR; + case INCONCLUSIVE: inconclusive = true; break; case FAILURE: failed = true; } } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 6aa36c62416..837b298d76f 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -5,6 +5,7 @@ import ai.vespa.cloud.Environment; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; import ai.vespa.hosted.api.TestDescriptor; +import ai.vespa.hosted.cd.InconclusiveTestException; import ai.vespa.hosted.cd.internal.TestRuntimeProvider; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; @@ -204,11 +205,13 @@ public class JunitRunner extends AbstractComponent implements TestRunner { var failures = report.getFailures().stream() .map(failure -> new TestReport.Failure(failure.getTestIdentifier().getUniqueId(), failure.getException())) .collect(Collectors.toList()); + long inconclusive = failures.stream().filter(failure -> failure.exception() instanceof InconclusiveTestException).count(); return TestReport.builder() .withSuccessCount(report.getTestsSucceededCount()) .withAbortedCount(report.getTestsAbortedCount()) .withIgnoredCount(report.getTestsSkippedCount()) - .withFailedCount(report.getTestsFailedCount()) + .withFailedCount(report.getTestsFailedCount() - inconclusive) + .withInconclusiveCount(inconclusive) .withFailures(failures) .withLogs(logRecords.values()) .build(); @@ -230,12 +233,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { if (execution == null) return TestRunner.Status.NOT_STARTED; if (!execution.isDone()) return TestRunner.Status.RUNNING; try { - TestReport report = execution.get(); - if (report.isSuccess()) { - return TestRunner.Status.SUCCESS; - } else { - return TestRunner.Status.FAILURE; - } + return execution.get().status(); } catch (InterruptedException|ExecutionException e) { logger.log(Level.WARNING, "Error while getting test report", e); return TestRunner.Status.ERROR; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java index b3ca47e2480..d01e9fffe9f 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java @@ -6,6 +6,10 @@ import java.util.Collections; import java.util.List; import java.util.logging.LogRecord; +import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; + /** * @author mortent */ @@ -14,15 +18,17 @@ public class TestReport { final long totalCount; final long successCount; final long failedCount; + final long inconclusiveCount; final long ignoredCount; final long abortedCount; final List<Failure> failures; final List<LogRecord> logLines; - private TestReport(long totalCount, long successCount, long failedCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) { + private TestReport(long totalCount, long successCount, long failedCount, long inconclusiveCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) { this.totalCount = totalCount; this.successCount = successCount; this.failedCount = failedCount; + this.inconclusiveCount = inconclusiveCount; this.ignoredCount = ignoredCount; this.abortedCount = abortedCount; this.failures = failures; @@ -33,8 +39,8 @@ public class TestReport { return logLines; } - public boolean isSuccess() { - return (failedCount + abortedCount) == 0; + public TestRunner.Status status() { + return failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : SUCCESS; } public static Builder builder(){ @@ -47,13 +53,14 @@ public class TestReport { private long totalCount; private long successCount; private long failedCount; + private long inconclusiveCount; private long ignoredCount; private long abortedCount; private List<Failure> failures = Collections.emptyList(); private List<LogRecord> logLines = Collections.emptyList(); public TestReport build() { - return new TestReport(totalCount, successCount, failedCount, ignoredCount, abortedCount, failures, logLines); + return new TestReport(totalCount, successCount, failedCount, inconclusiveCount, ignoredCount, abortedCount, failures, logLines); } public Builder withTotalCount(long totalCount) { @@ -71,6 +78,11 @@ public class TestReport { return this; } + public Builder withInconclusiveCount(long failedCount) { + this.failedCount = failedCount; + return this; + } + public Builder withIgnoredCount(long ignoredCount) { this.ignoredCount = ignoredCount; return this; @@ -95,6 +107,7 @@ public class TestReport { public static class Failure { + private final String testId; private final Throwable exception; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java index d70a3f60c7d..20ff19266cf 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java @@ -22,7 +22,7 @@ public interface TestRunner { default TestReport getReport() { return null; } enum Status { - NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS + NOT_STARTED, RUNNING, FAILURE, INCONCLUSIVE, ERROR, SUCCESS } enum Suite { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index a383d435a26..e7b0fe406c7 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -27,7 +27,7 @@ import static com.yahoo.jdisc.Response.Status; /** * @author valerijf - * @author jvenstad + * @author jonmv * @author mortent */ public class TestRunnerHandler extends ThreadedHttpRequestHandler { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index 3cccafb4a8b..9ae75fbae43 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; import static com.yahoo.yolean.Exceptions.uncheck; @@ -88,7 +89,8 @@ public class VespaCliTestRunner implements TestRunner { HtmlLogger htmlLogger = new HtmlLogger(); BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); in.lines().forEach(line -> log(htmlLogger.toLog(line))); - status.set(process.waitFor() == 0 ? SUCCESS : process.waitFor() == 3 ? FAILURE : ERROR); + int exitCode = process.waitFor(); + status.set(exitCode == 0 ? SUCCESS : exitCode == 3 ? FAILURE : exitCode == 4 ? INCONCLUSIVE : ERROR); } catch (NoTestsException e) { log(Level.WARNING, "Did not find expected basic HTTP tests", e); diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java index 64f9079643a..a7ddbbf3fcc 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java @@ -12,6 +12,7 @@ import java.util.logging.LogRecord; import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED; import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; @@ -85,6 +86,10 @@ class AggregateTestRunnerTest { second.future.complete(null); assertEquals(SUCCESS, runner.getStatus()); + // An inconclusive test means inconclusive. + first.status = INCONCLUSIVE; + assertEquals(INCONCLUSIVE, runner.getStatus()); + // A failure means failure. second.status = FAILURE; assertEquals(FAILURE, runner.getStatus()); |