diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2021-11-17 19:28:27 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2021-11-17 19:28:27 +0100 |
commit | fbdc8549df12ffd68f470bdbabff72d49951cf61 (patch) | |
tree | 99990861e8a28b1573f1e979f44b8d67903a7faf /vespa-osgi-testrunner/src/test | |
parent | 7cddf841ac8774ecb1d359cac7e1c65059314733 (diff) |
Have registry injected, and wrap runners in an aggregate
Diffstat (limited to 'vespa-osgi-testrunner/src/test')
2 files changed, 199 insertions, 18 deletions
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java new file mode 100644 index 00000000000..64f9079643a --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java @@ -0,0 +1,174 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; +import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED; +import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; +import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author jonmv + */ +class AggregateTestRunnerTest { + + @Test + void onlySupportedRunnersAreUsed() { + MockTestRunner unsupported = new MockTestRunner(false); + MockTestRunner suppported = new MockTestRunner(true); + TestRunner runner = AggregateTestRunner.of(List.of(unsupported, suppported)); + CompletableFuture<?> future = runner.test(null, null); + assertFalse(future.isDone()); + assertNull(unsupported.future); + assertNotNull(suppported.future); + suppported.future.complete(null); + assertTrue(future.isDone()); + } + + @Test + void noTestsResultInFailure() { + TestRunner runner = AggregateTestRunner.of(List.of(new MockTestRunner(false))); + assertEquals("No tests were found", runner.getLog(-1).iterator().next().getMessage()); + assertSame(FAILURE, runner.getStatus()); + } + + @Test + void chainedRunners() { + LogRecord record1 = new LogRecord(Level.INFO, "one"); + LogRecord record2 = new LogRecord(Level.INFO, "two"); + MockTestRunner first = new MockTestRunner(true); + MockTestRunner second = new MockTestRunner(true); + TestRunner runner = AggregateTestRunner.of(List.of(first, second)); + assertSame(NOT_STARTED, runner.getStatus()); + assertEquals(List.of(), runner.getLog(-1)); + + // First wrapped runner is started. + CompletableFuture<?> future = runner.test(null, null); + assertNotNull(first.future); + assertNull(second.future); + assertEquals(RUNNING, runner.getStatus()); + + // Logs from first wrapped runner are returned. + assertEquals(List.of(), runner.getLog(-1)); + first.log.add(record1); + assertEquals(List.of(record1), runner.getLog(-1)); + assertEquals(List.of(), runner.getLog(record1.getSequenceNumber())); + + // First wrapped runner completes, second is started. + first.status = SUCCESS; + first.future.complete(null); + assertNotNull(second.future); + assertFalse(future.isDone()); + assertEquals(RUNNING, runner.getStatus()); + + // Logs from second runner are available. + second.log.add(record2); + assertEquals(List.of(record1, record2), runner.getLog(-1)); + + // No failures means success. + second.future.complete(null); + assertEquals(SUCCESS, runner.getStatus()); + + // A failure means failure. + second.status = FAILURE; + assertEquals(FAILURE, runner.getStatus()); + + // An error means error. + first.status = ERROR; + assertEquals(ERROR, runner.getStatus()); + + // Runner is re-used. Ensure nothing from the second wrapped runner is visible. + runner.test(null, null); + assertFalse(first.future.isDone()); + assertTrue(second.future.isDone()); + assertEquals(List.of(record1), runner.getLog(-1)); + assertEquals(ERROR, runner.getStatus()); + + // First wrapped runner completes exceptionally, but the second should be started as usual. + first.future.completeExceptionally(new RuntimeException("error")); + assertFalse(second.future.isDone()); + assertEquals(List.of(record1, record2), runner.getLog(-1)); + + // Verify reports are merged. + assertNull(runner.getReport()); + + TestReport.Failure failure = new TestReport.Failure("test", null); + TestReport report = TestReport.builder() + .withLogs(List.of(record1)) + .withFailures(List.of(failure)) + .withTotalCount(15) + .withSuccessCount(8) + .withIgnoredCount(4) + .withFailedCount(2) + .withAbortedCount(1) + .build(); + first.report = report; + assertSame(report, runner.getReport()); + + second.report = report; + TestReport merged = runner.getReport(); + assertEquals(List.of(record1, record1), merged.logLines); + assertEquals(List.of(failure, failure), merged.failures); + assertEquals(30, merged.totalCount); + assertEquals(16, merged.successCount); + assertEquals(8, merged.ignoredCount); + assertEquals(4, merged.failedCount); + assertEquals(2, merged.abortedCount); + } + + static class MockTestRunner implements TestRunner { + + final List<LogRecord> log = new ArrayList<>(); + final boolean supported; + CompletableFuture<?> future; + Status status = NOT_STARTED; + TestReport report; + + public MockTestRunner(boolean supported) { + this.supported = supported; + } + + @Override + public Collection<LogRecord> getLog(long after) { + return log.stream().filter(record -> record.getSequenceNumber() > after).collect(toList()); + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public CompletableFuture<?> test(Suite suite, byte[] config) { + return future = new CompletableFuture<>(); + } + + @Override + public boolean isSupported() { + return supported; + } + + @Override + public TestReport getReport() { + return report; + } + + } + +} diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index a78dc077446..37ab8550357 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.test.json.JsonTestHelper; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; @@ -12,6 +14,7 @@ import java.io.IOException; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -27,13 +30,15 @@ import static org.mockito.Mockito.when; /** * @author mortent */ -public class TestRunnerHandlerTest { +class TestRunnerHandlerTest { private static final Instant testInstant = Instant.ofEpochMilli(1598432151660L); - private static TestRunnerHandler testRunnerHandler; - @BeforeAll - public static void setup() { + private TestRunnerHandler testRunnerHandler; + private TestRunner aggregateRunner; + + @BeforeEach + void setup() { List<LogRecord> logRecords = List.of(logRecord("Tests started")); Throwable exception = new RuntimeException("org.junit.ComparisonFailure: expected:<foo> but was:<bar>"); exception.setStackTrace(new StackTraceElement[]{new StackTraceElement("Foo", "bar", "Foo.java", 1123)}); @@ -46,10 +51,8 @@ public class TestRunnerHandlerTest { .withFailures(List.of(new TestReport.Failure("Foo.bar()", exception))) .withLogs(logRecords).build(); - testRunnerHandler = new TestRunnerHandler( - Executors.newSingleThreadExecutor(), - new MockJunitRunner(TestRunner.Status.SUCCESS, testReport), - null); + aggregateRunner = AggregateTestRunner.of(List.of(new MockJunitRunner(TestRunner.Status.SUCCESS, testReport))); + testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner); } @Test @@ -58,11 +61,13 @@ public class TestRunnerHandlerTest { ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}"); - } @Test public void returnsCorrectLog() throws IOException { + // Prime the aggregate runner to actually consider the wrapped runner for logs. + aggregateRunner.test(TestRunner.Suite.SYSTEM_TEST, new byte[0]); + HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); @@ -73,7 +78,6 @@ public class TestRunnerHandlerTest { out = new ByteArrayOutputStream(); response.render(out); assertEquals("{\"logRecords\":[]}", out.toString(UTF_8)); - } @Test @@ -83,7 +87,7 @@ public class TestRunnerHandlerTest { when(testRunner.getReport()).thenReturn(null); testRunnerHandler = new TestRunnerHandler( Executors.newSingleThreadExecutor(), - testRunner, null); + ComponentRegistry.singleton(new ComponentId("runner"), testRunner)); { HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); @@ -105,19 +109,20 @@ public class TestRunnerHandlerTest { TestRunner testRunner = mock(TestRunner.class); when(testRunner.isSupported()).thenReturn(false); TestRunner legacyTestRunner = mock(TestRunner.class); + when(legacyTestRunner.isSupported()).thenReturn(true); when(legacyTestRunner.getLog(anyLong())).thenReturn(List.of(logRecord("Legacy log message"))); + TestRunner aggregate = AggregateTestRunner.of(List.of(testRunner, legacyTestRunner)); + testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregate); - testRunnerHandler = new TestRunnerHandler( - Executors.newSingleThreadExecutor(), - testRunner, legacyTestRunner); - + // Prime the aggregate to check for logs in the wrapped runners. + aggregate.test(TestRunner.Suite.PRODUCTION_TEST, new byte[0]); HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}"); } - /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization format */ + /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */ private static LogRecord logRecord(String logMessage) { LogRecord logRecord = new LogRecord(Level.INFO, logMessage); logRecord.setInstant(testInstant); @@ -137,7 +142,9 @@ public class TestRunnerHandlerTest { } @Override - public void test(Suite suite, byte[] testConfig) { } + public CompletableFuture<?> test(Suite suite, byte[] testConfig) { + return CompletableFuture.completedFuture(null); + } @Override public Collection<LogRecord> getLog(long after) { |