diff options
Diffstat (limited to 'vespa-osgi-testrunner')
13 files changed, 387 insertions, 229 deletions
diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index ebb1240a198..d89fbe1cf89 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -27,6 +27,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -37,23 +43,47 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>5.8.2</version> + <version>5.8.1</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> </exclusion> + <exclusion> + <groupId>org.junit.platform</groupId> + <artifactId>junit-platform-commons</artifactId> + </exclusion> + <exclusion> + <groupId>org.opentest4j</groupId> + <artifactId>opentest4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.apiguardian</groupId> + <artifactId>apiguardian-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> <groupId>org.junit.platform</groupId> <artifactId>junit-platform-launcher</artifactId> - <version>1.8.2</version> + <version>1.8.1</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> </exclusion> + <exclusion> + <groupId>org.junit.platform</groupId> + <artifactId>junit-platform-commons</artifactId> + </exclusion> + <exclusion> + <groupId>org.opentest4j</groupId> + <artifactId>opentest4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.apiguardian</groupId> + <artifactId>apiguardian-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -65,12 +95,6 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -80,6 +104,11 @@ <artifactId>org.apache.felix.framework</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 54f0941208d..d0e8663731a 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -4,28 +4,23 @@ package com.yahoo.vespa.testrunner; import ai.vespa.cloud.Environment; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; -import ai.vespa.hosted.cd.InconclusiveTestException; import ai.vespa.hosted.cd.internal.TestRuntimeProvider; import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.testrunner.TestReport.ContainerNode; -import com.yahoo.vespa.testrunner.TestReport.FailureNode; -import com.yahoo.vespa.testrunner.TestReport.Status; import org.junit.jupiter.engine.JupiterTestEngine; +import org.junit.platform.engine.discovery.ClassSelector; import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.core.LauncherConfig; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; -import org.junit.platform.launcher.listeners.SummaryGeneratingListener; import java.time.Clock; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.SortedMap; import java.util.concurrent.CompletableFuture; @@ -64,9 +59,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { this(Clock.systemUTC(), testRuntimeProvider, new TestBundleLoader(osgiFramework)::loadTestClasses, - (discoveryRequest, listeners) -> LauncherFactory.create(LauncherConfig.builder() - .addTestEngines(new JupiterTestEngine()) - .build()).execute(discoveryRequest, listeners)); + JunitRunner::executeTests); uglyHackSetCredentialsRootSystemProperty(config, systemInfo.zone()); @@ -82,6 +75,24 @@ public class JunitRunner extends AbstractComponent implements TestRunner { this.testRuntimeProvider = testRuntimeProvider; } + private static void executeTests(LauncherDiscoveryRequest discoveryRequest, TestExecutionListener[] listeners) { + var launcher = LauncherFactory.create(LauncherConfig.builder() + .addTestEngines(new JupiterTestEngine()) + .build()); + ClassLoader context = Thread.currentThread().getContextClassLoader(); + try { + // Pick the bundle class loader of the first user test class, from the test class selector. + discoveryRequest.getSelectorsByType(ClassSelector.class).stream() + .map(selector -> selector.getJavaClass().getClassLoader()) + .findAny().ifPresent(Thread.currentThread()::setContextClassLoader); + + launcher.execute(discoveryRequest, listeners); + } + finally { + Thread.currentThread().setContextClassLoader(context); + } + } + @Override public CompletableFuture<?> test(Suite suite, byte[] testConfig) { if (execution != null && ! execution.isDone()) { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java index 9aae329d7fb..1641bd7802f 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.logging.Level; import java.util.logging.LogRecord; import static java.util.Arrays.copyOf; @@ -52,7 +53,7 @@ public class TestReport { if (thrown instanceof OutOfMemoryError) throw (Error) thrown; TestReport failed = new TestReport(clock, suite); failed.complete(); - failed.root().children.add(new FailureNode(failed.root(), thrown, suite)); + failed.root().children.add(new FailureNode(failed.root(), clock.instant(), thrown, suite)); return failed; } @@ -126,7 +127,7 @@ public class TestReport { synchronized (monitor) { Status status = Status.successful; if (thrown != null) { - FailureNode failure = new FailureNode(current, thrown, suite); + FailureNode failure = new FailureNode(current, clock.instant(), thrown, suite); current.children.add(failure); status = failure.status(); } @@ -271,22 +272,35 @@ public class TestReport { } - public static class FailureNode extends Node { + public static class FailureNode extends NamedNode { private final Throwable thrown; private final Suite suite; - public FailureNode(NamedNode parent, Throwable thrown, Suite suite) { - super(parent); - this.thrown = thrown; + public FailureNode(NamedNode parent, Instant now, Throwable thrown, Suite suite) { + super(parent, null, thrown.toString(), now); trimStackTraces(thrown, JunitRunner.class.getName()); + this.thrown = thrown; this.suite = suite; + + LogRecord record = new LogRecord(levelOf(status()), null); + record.setThrown(thrown); + record.setInstant(now); + OutputNode child = new OutputNode(this); + child.log.add(record); + children.add(child); } public Throwable thrown() { return thrown; } + @Override + public Duration duration() { + return Duration.ZERO; + } + + @Override public Status status() { return suite == Suite.PRODUCTION_TEST && thrown instanceof InconclusiveTestException ? Status.inconclusive @@ -297,16 +311,20 @@ public class TestReport { public enum Status { - // Must be kept in order of increasing severity. - successful, + // Must be kept in order of increasing importance. skipped, aborted, + successful, inconclusive, failed, error; } + static Level levelOf(Status status) { + return status.compareTo(Status.failed) >= 0 ? Level.SEVERE : status == Status.successful ? Level.INFO : Level.WARNING; + } + /** * Recursively trims stack traces for the given throwable and its causes/suppressed. * This is based on the assumption that the relevant stack is anything above the first native diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java index 0d767f5aa8a..5bc9fda6835 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReportGeneratingListener.java @@ -110,7 +110,7 @@ class TestReportGeneratingListener implements TestExecutionListener { ? report.abort(testIdentifier) : report.complete(testIdentifier, testExecutionResult.getThrowable().orElse(null)); Status status = node.status(); - Level level = status.compareTo(Status.failed) >= 0 ? SEVERE : status.compareTo(Status.skipped) >= 0 ? WARNING : INFO; + Level level = TestReport.levelOf(status); if (testIdentifier.isContainer()) { if (testIdentifier.getParentIdObject().isPresent()) { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index b1ca6c84b75..756c3f55ab3 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.EmptyResponse; @@ -9,11 +11,8 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.exception.ExceptionUtils; import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.testrunner.TestReport.ContainerNode; import com.yahoo.vespa.testrunner.TestReport.FailureNode; +import com.yahoo.vespa.testrunner.TestReport.NamedNode; import com.yahoo.vespa.testrunner.TestReport.Node; import com.yahoo.vespa.testrunner.TestReport.OutputNode; import com.yahoo.vespa.testrunner.TestReport.TestNode; @@ -21,6 +20,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -42,6 +42,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class TestRunnerHandler extends ThreadedHttpRequestHandler { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS"); + private static final JsonFactory factory = new JsonFactory(); private final TestRunner testRunner; @@ -79,15 +80,13 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { long fetchRecordsAfter = Optional.ofNullable(request.getProperty("after")) .map(Long::parseLong) .orElse(-1L); - return new SlimeJsonResponse(logToSlime(testRunner.getLog(fetchRecordsAfter))); + return new CustomJsonResponse(out -> render(out, testRunner.getLog(fetchRecordsAfter))); case "/tester/v1/status": return new MessageResponse(testRunner.getStatus().name()); case "/tester/v1/report": TestReport report = testRunner.getReport(); - if (report == null) - return new EmptyResponse(200); - - return new SlimeJsonResponse(toSlime(report)); + if (report == null) return new EmptyResponse(204); + else return new CustomJsonResponse(out -> render(out, report)); } return new MessageResponse(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); } @@ -113,31 +112,30 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return path.substring(lastSlash + 1); } - static Slime logToSlime(Collection<LogRecord> log) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor recordArray = root.setArray("logRecords"); - logArrayToSlime(recordArray, log); - return slime; - } - - static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) { - log.forEach(record -> { - Cursor recordObject = recordArray.addObject(); - recordObject.setLong("id", record.getSequenceNumber()); - recordObject.setLong("at", record.getMillis()); - recordObject.setString("type", typeOf(record.getLevel())); - String message = record.getMessage(); + private static void render(OutputStream out, Collection<LogRecord> log) throws IOException { + var json = factory.createGenerator(out); + json.writeStartObject(); + json.writeArrayFieldStart("logRecords"); + for (LogRecord record : log) { + String message = record.getMessage() == null ? "" : record.getMessage(); if (record.getThrown() != null) { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); record.getThrown().printStackTrace(new PrintStream(buffer)); - message += "\n" + buffer; + message += (message.isEmpty() ? "" : "\n") + buffer; } - recordObject.setString("message", message); - }); + json.writeStartObject(); + json.writeNumberField("id", record.getSequenceNumber()); + json.writeNumberField("at", record.getMillis()); + json.writeStringField("type", typeOf(record.getLevel())); + json.writeStringField("message", message); + json.writeEndObject(); + } + json.writeEndArray(); + json.writeEndObject(); + json.close(); } - public static String typeOf(Level level) { + private static String typeOf(Level level) { return level.getName().equals("html") ? "html" : level.intValue() < Level.INFO.intValue() ? "debug" : level.intValue() < Level.WARNING.intValue() ? "info" @@ -145,85 +143,54 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { : "error"; } - private static Slime toSlime(TestReport report) { - var slime = new Slime(); - var root = slime.setObject(); + private static void render(OutputStream out, TestReport report) throws IOException { + JsonGenerator json = factory.createGenerator(out); + json.writeStartObject(); - toSlime(root.setObject("report"), (Node) report.root()); + json.writeFieldName("report"); + render(json, (Node) report.root()); - // TODO jonmv: remove - Map<TestReport.Status, Long> tally = report.root().tally(); - var summary = root.setObject("summary"); - summary.setLong("success", tally.getOrDefault(TestReport.Status.successful, 0L)); - summary.setLong("failed", tally.getOrDefault(TestReport.Status.failed, 0L) + tally.getOrDefault(TestReport.Status.error, 0L)); - summary.setLong("ignored", tally.getOrDefault(TestReport.Status.skipped, 0L)); - summary.setLong("aborted", tally.getOrDefault(TestReport.Status.aborted, 0L)); - summary.setLong("inconclusive", tally.getOrDefault(TestReport.Status.inconclusive, 0L)); - toSlime(summary.setArray("failures"), root.setArray("output"), report.root()); - - return slime; + json.writeEndObject(); + json.close(); } - static void toSlime(Cursor failuresArray, Cursor outputArray, Node node) { - for (Node child : node.children()) - TestRunnerHandler.toSlime(failuresArray, outputArray, child); - - if (node instanceof FailureNode) { - Cursor failureObject = failuresArray.addObject(); - failureObject.setString("testName", node.parent.name()); - failureObject.setString("testError", ((FailureNode) node).thrown().getMessage()); - failureObject.setString("exception", ExceptionUtils.getStackTraceAsString(((FailureNode) node).thrown())); - } - if (node instanceof OutputNode) - for (LogRecord record : ((OutputNode) node).log()) - outputArray.addString(formatter.format(record.getInstant().atOffset(ZoneOffset.UTC)) + " " + record.getMessage()); - } - - static void toSlime(Cursor nodeObject, Node node) { - if (node instanceof ContainerNode) toSlime(nodeObject, (ContainerNode) node); - if (node instanceof TestNode) toSlime(nodeObject, (TestNode) node); - if (node instanceof OutputNode) toSlime(nodeObject, (OutputNode) node); - if (node instanceof FailureNode) toSlime(nodeObject, (FailureNode) node); + private static void render(JsonGenerator json, Node node) throws IOException { + json.writeStartObject(); + if (node instanceof NamedNode) render(json, (NamedNode) node); + if (node instanceof OutputNode) render(json, (OutputNode) node); if ( ! node.children().isEmpty()) { - Cursor childrenArray = nodeObject.setArray("children"); - for (Node child : node.children) - toSlime(childrenArray.addObject(), child); + json.writeArrayFieldStart("children"); + for (Node child : node.children) { + render(json, child); + } + json.writeEndArray(); } + json.writeEndObject(); } - static void toSlime(Cursor nodeObject, ContainerNode node) { - nodeObject.setString("type", "container"); - nodeObject.setString("name", node.name()); - nodeObject.setString("status", node.status().name()); - nodeObject.setLong("start", node.start().toEpochMilli()); - nodeObject.setLong("duration", node.duration().toMillis()); - } - - static void toSlime(Cursor nodeObject, TestNode node) { - nodeObject.setString("type", "test"); - nodeObject.setString("name", node.name()); - nodeObject.setString("status", node.status().name()); - nodeObject.setLong("start", node.start().toEpochMilli()); - nodeObject.setLong("duration", node.duration().toMillis()); + private static void render(JsonGenerator json, NamedNode node) throws IOException { + String type = node instanceof FailureNode ? "failure" : node instanceof TestNode ? "test" : "container"; + json.writeStringField("type", type); + json.writeStringField("name", node.name()); + json.writeStringField("status", node.status().name()); + json.writeNumberField("start", node.start().toEpochMilli()); + json.writeNumberField("duration", node.duration().toMillis()); } - static void toSlime(Cursor nodeObject, OutputNode node) { - nodeObject.setString("type", "output"); - Cursor childrenArray = nodeObject.setArray("children"); + private static void render(JsonGenerator json, OutputNode node) throws IOException { + json.writeStringField("type", "output"); + json.writeArrayFieldStart("children"); for (LogRecord record : node.log()) { - Cursor recordObject = childrenArray.addObject(); - recordObject.setString("message", (record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + record.getMessage()); - recordObject.setLong("at", record.getInstant().toEpochMilli()); - recordObject.setString("level", typeOf(record.getLevel())); - if (record.getThrown() != null) recordObject.setString("trace", traceToString(record.getThrown())); + json.writeStartObject(); + json.writeStringField("message", (record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + + (record.getMessage() != null ? record.getMessage() : "") + + (record.getThrown() != null ? (record.getMessage() != null ? "\n" : "") + traceToString(record.getThrown()) : "")); + json.writeNumberField("at", record.getInstant().toEpochMilli()); + json.writeStringField("level", typeOf(record.getLevel())); + json.writeEndObject(); } - } - - static void toSlime(Cursor nodeObject, FailureNode node) { - nodeObject.setString("type", "failure"); - nodeObject.setString("status", node.status().name()); - nodeObject.setString("trace", traceToString(node.thrown())); + json.writeEndArray(); } private static String traceToString(Throwable thrown) { @@ -232,4 +199,36 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return buffer.toString(UTF_8); } + private interface Renderer { + + void render(OutputStream out) throws IOException; + + } + + private static class CustomJsonResponse extends HttpResponse { + + private final Renderer renderer; + + private CustomJsonResponse(Renderer renderer) { + super(200); + this.renderer = renderer; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + renderer.render(outputStream); + } + + @Override + public String getContentType() { + return "application/json"; + } + + @Override + public long maxPendingBytes() { + return 1 << 25; // 32MB + } + + } + } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java index 795bf8c6a1e..417ca4b6c9e 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledClassTest.java @@ -5,7 +5,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @Disabled("because") -@Expect(skipped = 2, status = 1) +@Expect(skipped = 2, status = 2) public class DisabledClassTest { @Test diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java index 561ec81e865..be36954d1bb 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/DisabledTest.java @@ -4,7 +4,7 @@ import com.yahoo.vespa.testrunner.Expect; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -@Expect(skipped = 1, status = 1) +@Expect(skipped = 1, status = 2) public class DisabledTest { @Test diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java index bc878353d4b..b0e5119c06e 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java @@ -71,7 +71,7 @@ public class SampleTest { @Test void successful() { log.log(new Level("html", INFO.intValue()) { }, "<body />"); - log.log(INFO, "Very informative"); + log.log(INFO, "Very informative: \"\\n\": \n"); log.log(WARNING, "Oh no", new IllegalArgumentException("error", new RuntimeException("wrapped"))); } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java index 59a56a1c9c7..8fd25d618a9 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SucceedingTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.test.samples; import com.yahoo.vespa.testrunner.Expect; import org.junit.jupiter.api.Test; -@Expect(successful = 1, status = 0) +@Expect(successful = 1, status = 2) public class SucceedingTest { @Test diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java index 62547bd34bf..67b236f75a2 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/UsingTestRuntimeTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -@Expect(successful = 1, status = 0) +@Expect(successful = 1, status = 2) public class UsingTestRuntimeTest { @Test diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index 5ce737d7649..6d6fbbf2cf1 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; -import java.util.logging.Level; import java.util.logging.LogRecord; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -136,14 +135,6 @@ class TestRunnerHandlerTest { } } - /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */ - private static LogRecord logRecord(String logMessage) { - LogRecord logRecord = new LogRecord(Level.INFO, logMessage); - logRecord.setInstant(testInstant); - logRecord.setSequenceNumber(0); - return logRecord; - } - private static class MockRunner implements TestRunner { private final TestRunner.Status status; diff --git a/vespa-osgi-testrunner/src/test/resources/output.json b/vespa-osgi-testrunner/src/test/resources/output.json index 04682fa9f31..2b4aa9e5599 100644 --- a/vespa-osgi-testrunner/src/test/resources/output.json +++ b/vespa-osgi-testrunner/src/test/resources/output.json @@ -1,18 +1,24 @@ { "logRecords": [ { - "id": 3, + "id": 2, "at": 0, "type": "info", "message": "spam" }, { - "id": 6, + "id": 5, "at": 0, "type": "info", "message": "spam" }, { + "id": 6, + "at": 0, + "type": "error", + "message": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n" + }, + { "id": 9, "at": 0, "type": "info", @@ -25,58 +31,106 @@ "message": "I have a bad feeling about this" }, { - "id": 14, + "id": 11, + "at": 0, + "type": "error", + "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n" + }, + { + "id": 15, "at": 0, "type": "info", "message": "spam" }, { - "id": 15, + "id": 16, "at": 0, "type": "info", "message": "I'm here with Erwin today; Erwin, what can you tell us about your cat?" }, { - "id": 18, + "id": 17, + "at": 0, + "type": "warning", + "message": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n" + }, + { + "id": 20, "at": 0, "type": "info", "message": "spam" }, { - "id": 19, + "id": 21, "at": 0, "type": "info", "message": "<body />" }, { - "id": 20, + "id": 22, "at": 0, "type": "info", - "message": "Very informative" + "message": "Very informative: \"\\n\": \n" }, { - "id": 21, + "id": 23, "at": 0, "type": "warning", "message": "Oh no\njava.lang.IllegalArgumentException: error\n\tat com.yahoo.vespa.test.samples.SampleTest.successful(SampleTest.java:75)\nCaused by: java.lang.RuntimeException: wrapped\n\t... 1 more\n" }, { - "id": 25, + "id": 27, "at": 0, "type": "info", "message": "spam" }, { - "id": 28, + "id": 30, "at": 0, "type": "info", "message": "Catch me if you can!" }, { - "id": 35, + "id": 34, + "at": 0, + "type": "error", + "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n" + }, + { + "id": 38, "at": 0, "type": "info", "message": "spam" + }, + { + "id": 67, + "at": 0, + "type": "error", + "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n" + }, + { + "id": 69, + "at": 0, + "type": "error", + "message": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n" + }, + { + "id": 72, + "at": 0, + "type": "error", + "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n" + }, + { + "id": 76, + "at": 0, + "type": "error", + "message": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n" + }, + { + "id": 80, + "at": 12000, + "type": "error", + "message": "java.lang.ClassNotFoundException: School's out all summer!\n" } ] } diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json index 9c41a83a6b5..66ae6dd398c 100644 --- a/vespa-osgi-testrunner/src/test/resources/report.json +++ b/vespa-osgi-testrunner/src/test/resources/report.json @@ -58,8 +58,22 @@ }, { "type": "failure", + "name": "java.lang.NoClassDefFoundError", "status": "error", - "trace": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] }, @@ -87,8 +101,22 @@ }, { "type": "failure", + "name": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>", "status": "failed", - "trace": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] }, @@ -123,8 +151,22 @@ }, { "type": "failure", + "name": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive", "status": "inconclusive", - "trace": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n", + "at": 0, + "level": "warning" + } + ] + } + ] } ] }, @@ -149,15 +191,14 @@ "level": "info" }, { - "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative", + "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative: \"\\n\": \n", "at": 0, "level": "info" }, { - "message": "com.yahoo.vespa.test.samples.SampleTest: Oh no", + "message": "com.yahoo.vespa.test.samples.SampleTest: Oh no\njava.lang.IllegalArgumentException: error\n\tat com.yahoo.vespa.test.samples.SampleTest.successful(SampleTest.java:75)\nCaused by: java.lang.RuntimeException: wrapped\n\t... 1 more\n", "at": 0, - "level": "warning", - "trace": "java.lang.IllegalArgumentException: error\n\tat com.yahoo.vespa.test.samples.SampleTest.successful(SampleTest.java:75)\nCaused by: java.lang.RuntimeException: wrapped\n\t... 1 more\n" + "level": "warning" } ] } @@ -215,8 +256,22 @@ "children": [ { "type": "failure", + "name": "org.opentest4j.AssertionFailedError: no charm", "status": "failed", - "trace": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] } @@ -286,15 +341,43 @@ "children": [ { "type": "failure", + "name": "org.opentest4j.AssertionFailedError", "status": "failed", - "trace": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] }, { "type": "failure", + "name": "java.lang.RuntimeException", "status": "error", - "trace": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] }, @@ -307,8 +390,22 @@ "children": [ { "type": "failure", + "name": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).", "status": "error", - "trace": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n", + "at": 0, + "level": "error" + } + ] + } + ] }, { "type": "test", @@ -342,8 +439,22 @@ "children": [ { "type": "failure", + "name": "java.lang.NullPointerException", "status": "error", - "trace": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n" + "start": 0, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n", + "at": 0, + "level": "error" + } + ] + } + ] } ] } @@ -353,78 +464,23 @@ }, { "type": "failure", + "name": "java.lang.ClassNotFoundException: School's out all summer!", "status": "error", - "trace": "java.lang.ClassNotFoundException: School's out all summer!\n" - } - ] - }, - "summary": { - "success": 3, - "failed": 5, - "ignored": 4, - "aborted": 1, - "inconclusive": 1, - "failures": [ - { - "testName": "error()", - "testError": null, - "exception": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n" - }, - { - "testName": "failing()", - "testError": "baz ==> expected: <foo> but was: <bar>", - "exception": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n" - }, - { - "testName": "inconclusive(TestReporter)", - "testError": "the cat is both dead _and_ alive", - "exception": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n" - }, - { - "testName": "third", - "testError": "no charm", - "exception": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n" - }, - { - "testName": "test()", - "testError": "", - "exception": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n" - }, - { - "testName": "FailingTestAndBothAftersTest", - "testError": null, - "exception": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n" - }, - { - "testName": "WrongBeforeAllTest", - "testError": "@BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).", - "exception": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n" - }, - { - "testName": "test()", - "testError": null, - "exception": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n" - }, - { - "testName": "Production test", - "testError": "School's out all summer!", - "exception": "java.lang.ClassNotFoundException: School's out all summer!\n" + "start": 12000, + "duration": 0, + "children": [ + { + "type": "output", + "children": [ + { + "message": "java.lang.ClassNotFoundException: School's out all summer!\n", + "at": 12000, + "level": "error" + } + ] + } + ] } ] - }, - "output": [ - "00:00:00.000 spam", - "00:00:00.000 spam", - "00:00:00.000 spam", - "00:00:00.000 I have a bad feeling about this", - "00:00:00.000 spam", - "00:00:00.000 I'm here with Erwin today; Erwin, what can you tell us about your cat?", - "00:00:00.000 spam", - "00:00:00.000 <body />", - "00:00:00.000 Very informative", - "00:00:00.000 Oh no", - "00:00:00.000 spam", - "00:00:00.000 Catch me if you can!", - "00:00:00.000 spam" - ] + } } |