diff options
Diffstat (limited to 'vespa-osgi-testrunner/src/test/java/com/yahoo/vespa')
2 files changed, 232 insertions, 38 deletions
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java new file mode 100644 index 00000000000..64f9079643a --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java @@ -0,0 +1,174 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; +import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED; +import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; +import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author jonmv + */ +class AggregateTestRunnerTest { + + @Test + void onlySupportedRunnersAreUsed() { + MockTestRunner unsupported = new MockTestRunner(false); + MockTestRunner suppported = new MockTestRunner(true); + TestRunner runner = AggregateTestRunner.of(List.of(unsupported, suppported)); + CompletableFuture<?> future = runner.test(null, null); + assertFalse(future.isDone()); + assertNull(unsupported.future); + assertNotNull(suppported.future); + suppported.future.complete(null); + assertTrue(future.isDone()); + } + + @Test + void noTestsResultInFailure() { + TestRunner runner = AggregateTestRunner.of(List.of(new MockTestRunner(false))); + assertEquals("No tests were found", runner.getLog(-1).iterator().next().getMessage()); + assertSame(FAILURE, runner.getStatus()); + } + + @Test + void chainedRunners() { + LogRecord record1 = new LogRecord(Level.INFO, "one"); + LogRecord record2 = new LogRecord(Level.INFO, "two"); + MockTestRunner first = new MockTestRunner(true); + MockTestRunner second = new MockTestRunner(true); + TestRunner runner = AggregateTestRunner.of(List.of(first, second)); + assertSame(NOT_STARTED, runner.getStatus()); + assertEquals(List.of(), runner.getLog(-1)); + + // First wrapped runner is started. + CompletableFuture<?> future = runner.test(null, null); + assertNotNull(first.future); + assertNull(second.future); + assertEquals(RUNNING, runner.getStatus()); + + // Logs from first wrapped runner are returned. + assertEquals(List.of(), runner.getLog(-1)); + first.log.add(record1); + assertEquals(List.of(record1), runner.getLog(-1)); + assertEquals(List.of(), runner.getLog(record1.getSequenceNumber())); + + // First wrapped runner completes, second is started. + first.status = SUCCESS; + first.future.complete(null); + assertNotNull(second.future); + assertFalse(future.isDone()); + assertEquals(RUNNING, runner.getStatus()); + + // Logs from second runner are available. + second.log.add(record2); + assertEquals(List.of(record1, record2), runner.getLog(-1)); + + // No failures means success. + second.future.complete(null); + assertEquals(SUCCESS, runner.getStatus()); + + // A failure means failure. + second.status = FAILURE; + assertEquals(FAILURE, runner.getStatus()); + + // An error means error. + first.status = ERROR; + assertEquals(ERROR, runner.getStatus()); + + // Runner is re-used. Ensure nothing from the second wrapped runner is visible. + runner.test(null, null); + assertFalse(first.future.isDone()); + assertTrue(second.future.isDone()); + assertEquals(List.of(record1), runner.getLog(-1)); + assertEquals(ERROR, runner.getStatus()); + + // First wrapped runner completes exceptionally, but the second should be started as usual. + first.future.completeExceptionally(new RuntimeException("error")); + assertFalse(second.future.isDone()); + assertEquals(List.of(record1, record2), runner.getLog(-1)); + + // Verify reports are merged. + assertNull(runner.getReport()); + + TestReport.Failure failure = new TestReport.Failure("test", null); + TestReport report = TestReport.builder() + .withLogs(List.of(record1)) + .withFailures(List.of(failure)) + .withTotalCount(15) + .withSuccessCount(8) + .withIgnoredCount(4) + .withFailedCount(2) + .withAbortedCount(1) + .build(); + first.report = report; + assertSame(report, runner.getReport()); + + second.report = report; + TestReport merged = runner.getReport(); + assertEquals(List.of(record1, record1), merged.logLines); + assertEquals(List.of(failure, failure), merged.failures); + assertEquals(30, merged.totalCount); + assertEquals(16, merged.successCount); + assertEquals(8, merged.ignoredCount); + assertEquals(4, merged.failedCount); + assertEquals(2, merged.abortedCount); + } + + static class MockTestRunner implements TestRunner { + + final List<LogRecord> log = new ArrayList<>(); + final boolean supported; + CompletableFuture<?> future; + Status status = NOT_STARTED; + TestReport report; + + public MockTestRunner(boolean supported) { + this.supported = supported; + } + + @Override + public Collection<LogRecord> getLog(long after) { + return log.stream().filter(record -> record.getSequenceNumber() > after).collect(toList()); + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public CompletableFuture<?> test(Suite suite, byte[] config) { + return future = new CompletableFuture<>(); + } + + @Override + public boolean isSupported() { + return supported; + } + + @Override + public TestReport getReport() { + return report; + } + + } + +} diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index 0caac95ed34..37ab8550357 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -1,24 +1,28 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; -import ai.vespa.hosted.api.TestDescriptor; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Instant; +import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.LogRecord; +import java.util.stream.Collectors; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static org.junit.Assert.assertEquals; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,13 +30,15 @@ import static org.mockito.Mockito.when; /** * @author mortent */ -public class TestRunnerHandlerTest { +class TestRunnerHandlerTest { private static final Instant testInstant = Instant.ofEpochMilli(1598432151660L); - private static TestRunnerHandler testRunnerHandler; - @BeforeAll - public static void setup() { + private TestRunnerHandler testRunnerHandler; + private TestRunner aggregateRunner; + + @BeforeEach + void setup() { List<LogRecord> logRecords = List.of(logRecord("Tests started")); Throwable exception = new RuntimeException("org.junit.ComparisonFailure: expected:<foo> but was:<bar>"); exception.setStackTrace(new StackTraceElement[]{new StackTraceElement("Foo", "bar", "Foo.java", 1123)}); @@ -45,10 +51,8 @@ public class TestRunnerHandlerTest { .withFailures(List.of(new TestReport.Failure("Foo.bar()", exception))) .withLogs(logRecords).build(); - testRunnerHandler = new TestRunnerHandler( - Executors.newSingleThreadExecutor(), - new MockJunitRunner(LegacyTestRunner.Status.SUCCESS, testReport), - null); + aggregateRunner = AggregateTestRunner.of(List.of(new MockJunitRunner(TestRunner.Status.SUCCESS, testReport))); + testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner); } @Test @@ -56,58 +60,69 @@ public class TestRunnerHandlerTest { HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); - JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}"); - + JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}"); } @Test public void returnsCorrectLog() throws IOException { + // Prime the aggregate runner to actually consider the wrapped runner for logs. + aggregateRunner.test(TestRunner.Suite.SYSTEM_TEST, new byte[0]); + HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); - JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Tests started\"}]}"); + JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Tests started\"}]}"); // Should not get old log response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log?after=0", GET)); out = new ByteArrayOutputStream(); response.render(out); - assertEquals("{\"logRecords\":[]}", new String(out.toByteArray())); - + assertEquals("{\"logRecords\":[]}", out.toString(UTF_8)); } @Test - public void returnsEmptyLogWhenReportNotReady() throws IOException { + public void returnsEmptyResponsesWhenReportNotReady() throws IOException { TestRunner testRunner = mock(TestRunner.class); when(testRunner.isSupported()).thenReturn(true); when(testRunner.getReport()).thenReturn(null); testRunnerHandler = new TestRunnerHandler( Executors.newSingleThreadExecutor(), - testRunner, null); + ComponentRegistry.singleton(new ComponentId("runner"), testRunner)); - HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - response.render(out); - assertEquals("{\"logRecords\":[]}", new String(out.toByteArray())); + { + HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + response.render(out); + assertEquals("{\"logRecords\":[]}", out.toString(UTF_8)); + } + + { + HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + response.render(out); + assertEquals("", out.toString(UTF_8)); + } } @Test public void usesLegacyTestRunnerWhenNotSupported() throws IOException { TestRunner testRunner = mock(TestRunner.class); when(testRunner.isSupported()).thenReturn(false); - LegacyTestRunner legacyTestRunner = mock(LegacyTestRunner.class); + TestRunner legacyTestRunner = mock(TestRunner.class); + when(legacyTestRunner.isSupported()).thenReturn(true); when(legacyTestRunner.getLog(anyLong())).thenReturn(List.of(logRecord("Legacy log message"))); + TestRunner aggregate = AggregateTestRunner.of(List.of(testRunner, legacyTestRunner)); + testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregate); - testRunnerHandler = new TestRunnerHandler( - Executors.newSingleThreadExecutor(), - testRunner, legacyTestRunner); - + // Prime the aggregate to check for logs in the wrapped runners. + aggregate.test(TestRunner.Suite.PRODUCTION_TEST, new byte[0]); HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); - JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}"); + JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}"); } - /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization format */ + /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */ private static LogRecord logRecord(String logMessage) { LogRecord logRecord = new LogRecord(Level.INFO, logMessage); logRecord.setInstant(testInstant); @@ -117,17 +132,25 @@ public class TestRunnerHandlerTest { private static class MockJunitRunner implements TestRunner { - private final LegacyTestRunner.Status status; + private final TestRunner.Status status; private final TestReport testReport; - public MockJunitRunner(LegacyTestRunner.Status status, TestReport testReport) { + public MockJunitRunner(TestRunner.Status status, TestReport testReport) { this.status = status; this.testReport = testReport; } @Override - public void executeTests(TestDescriptor.TestCategory category, byte[] testConfig) { + public CompletableFuture<?> test(Suite suite, byte[] testConfig) { + return CompletableFuture.completedFuture(null); + } + + @Override + public Collection<LogRecord> getLog(long after) { + return getReport().logLines().stream() + .filter(entry -> entry.getSequenceNumber() > after) + .collect(Collectors.toList()); } @Override @@ -136,7 +159,7 @@ public class TestRunnerHandlerTest { } @Override - public LegacyTestRunner.Status getStatus() { + public TestRunner.Status getStatus() { return status; } @@ -145,9 +168,6 @@ public class TestRunnerHandlerTest { return testReport; } - @Override - public String getReportAsJson() { - return getReport().toJson(); - } } + } |