Have registry injected, and wrap runners in an aggregate

8 files changed, 342 insertions, 37 deletions

diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java
new file mode 100644
index 00000000000..82c1f7194d0
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java
@@ -0,0 +1,119 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static java.util.stream.Collectors.toUnmodifiableList;
+
+/**
+ * @author jonmv
+ */
+public class AggregateTestRunner implements TestRunner {
+
+    static final TestRunner noRunner = new TestRunner() {
+        final LogRecord record = new LogRecord(Level.WARNING, "No tests were found");
+        @Override public Collection<LogRecord> getLog(long after) { return List.of(record); }
+        @Override public Status getStatus() { return Status.FAILURE; }
+        @Override public CompletableFuture<?> test(Suite suite, byte[] config) { return CompletableFuture.completedFuture(null); }
+        @Override public boolean isSupported() { return true; }
+    };
+
+    private final List<TestRunner> wrapped;
+    private final AtomicInteger current = new AtomicInteger(-1);
+
+    private AggregateTestRunner(List<TestRunner> testRunners) {
+        this.wrapped = testRunners;
+    }
+
+    public static TestRunner of(Collection<TestRunner> testRunners) {
+        List<TestRunner> supported = testRunners.stream().filter(TestRunner::isSupported).collect(toUnmodifiableList());
+        return supported.isEmpty() ? noRunner : new AggregateTestRunner(supported);
+    }
+
+    @Override
+    public Collection<LogRecord> getLog(long after) {
+        ArrayList<LogRecord> records = new ArrayList<>();
+        for (int i = 0; i <= current.get() && i < wrapped.size(); i++)
+            records.addAll(wrapped.get(i).getLog(after));
+
+        return records;
+    }
+
+    @Override
+    public Status getStatus() {
+        if (current.get() == -1)
+            return Status.NOT_STARTED;
+
+        boolean failed = false;
+        for (int i = 0; i <= current.get(); i++) {
+            if (i == wrapped.size())
+                return failed ? Status.FAILURE : Status.SUCCESS;
+
+            switch (wrapped.get(i).getStatus()) {
+                case ERROR: return Status.ERROR;
+                case FAILURE: failed = true;
+            }
+        }
+        return Status.RUNNING;
+    }
+
+    @Override
+    public CompletableFuture<?> test(Suite suite, byte[] config) {
+        if (0 <= current.get() && current.get() < wrapped.size())
+            throw new IllegalStateException("Tests already running, should not attempt to start now");
+
+        current.set(-1);
+        CompletableFuture<?> aggregate = new CompletableFuture<>();
+        CompletableFuture<?> vessel = CompletableFuture.completedFuture(null);
+        runNext(suite, config, vessel, aggregate);
+        return aggregate;
+    }
+
+    private void runNext(Suite suite, byte[] config, CompletableFuture<?> vessel, CompletableFuture<?> aggregate) {
+        vessel.whenComplete((__, ___) -> {
+            int next = current.incrementAndGet();
+            if (next == wrapped.size())
+                aggregate.complete(null);
+            else
+                runNext(suite, config, wrapped.get(next).test(suite, config), aggregate);
+        });
+    }
+
+    @Override
+    public boolean isSupported() {
+        return wrapped.stream().anyMatch(TestRunner::isSupported);
+    }
+
+    @Override
+    public TestReport getReport() {
+        return wrapped.stream().map(TestRunner::getReport).filter(Objects::nonNull)
+                      .reduce(AggregateTestRunner::merge).orElse(null);
+    }
+
+    static TestReport merge(TestReport first, TestReport second) {
+        return TestReport.builder()
+                         .withAbortedCount(first.abortedCount + second.abortedCount)
+                         .withFailedCount(first.failedCount + second.failedCount)
+                         .withIgnoredCount(first.ignoredCount + second.ignoredCount)
+                         .withSuccessCount(first.successCount + second.successCount)
+                         .withTotalCount(first.totalCount + second.totalCount)
+                         .withFailures(merged(first.failures, second.failures))
+                         .withLogs(merged(first.logLines, second.logLines))
+                         .build();
+    }
+
+    static <T> List<T> merged(List<T> first, List<T> second) {
+        ArrayList<T> merged = new ArrayList<>();
+        merged.addAll(first);
+        merged.addAll(second);
+        return merged;
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index d0f3b879fec..6aa36c62416 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -32,7 +32,6 @@ import java.util.SortedMap;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import java.util.logging.Logger;
@@ -48,7 +47,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
     private final SortedMap<Long, LogRecord> logRecords = new ConcurrentSkipListMap<>();
     private final BundleContext bundleContext;
     private final TestRuntimeProvider testRuntimeProvider;
-    private volatile Future<TestReport> execution;
+    private volatile CompletableFuture<TestReport> execution;
 
     @Inject
     public JunitRunner(OsgiFramework osgiFramework,
@@ -97,7 +96,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
     }
 
     @Override
-    public void test(TestRunner.Suite suite, byte[] testConfig) {
+    public CompletableFuture<?> test(Suite suite, byte[] testConfig) {
         if (execution != null && ! execution.isDone()) {
             throw new IllegalStateException("Test execution already in progress");
         }
@@ -117,6 +116,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         } catch (Exception e) {
             execution = CompletableFuture.completedFuture(createReportWithFailedInitialization(e));
         }
+        return execution;
     }
 
     @Override
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
index db489f5aa3d..d70a3f60c7d 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.testrunner;
 
 import java.util.Collection;
+import java.util.concurrent.CompletableFuture;
 import java.util.logging.LogRecord;
 
 /**
@@ -14,9 +15,9 @@ public interface TestRunner {
 
     Status getStatus();
 
-    void test(Suite suite, byte[] config);
+    CompletableFuture<?> test(Suite suite, byte[] config);
 
-    default boolean isSupported() { return true; }
+    boolean isSupported();
 
     default TestReport getReport() { return null; }
 
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
index da2db3798c2..62601e4dfa0 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.testrunner;
 
 import com.google.inject.Inject;
+import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.jdisc.EmptyResponse;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
@@ -34,9 +35,13 @@ public class TestRunnerHandler extends LoggingRequestHandler {
     private final TestRunner testRunner;
 
     @Inject
-    public TestRunnerHandler(Executor executor, TestRunner junitRunner, TestRunner testRunner) {
+    public TestRunnerHandler(Executor executor, ComponentRegistry<TestRunner> testRunners) {
+        this(executor, AggregateTestRunner.of(testRunners.allComponents()));
+    }
+
+    TestRunnerHandler(Executor executor, TestRunner testRunner) {
         super(executor);
-        this.testRunner = junitRunner.isSupported() ? junitRunner : testRunner;
+        this.testRunner = testRunner;
     }
 
     @Override
@@ -152,7 +157,6 @@ public class TestRunnerHandler extends LoggingRequestHandler {
     }
 
     private static void serializeFailure(TestReport.Failure failure, Cursor slime) {
-        var testIdentifier = failure.testId();
         slime.setString("testName", failure.testId());
         slime.setString("testError",failure.exception().getMessage());
         slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.exception()));
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java
deleted file mode 100644
index 6f6a8c819a6..00000000000
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * @author mortent
- */
-@ExportPackage
-package com.yahoo.vespa.testrunner.legacy;
-
-import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
new file mode 100644
index 00000000000..64f9079643a
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java
@@ -0,0 +1,174 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
+import static java.util.stream.Collectors.toList;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author jonmv
+ */
+class AggregateTestRunnerTest {
+
+    @Test
+    void onlySupportedRunnersAreUsed() {
+        MockTestRunner unsupported = new MockTestRunner(false);
+        MockTestRunner suppported = new MockTestRunner(true);
+        TestRunner runner = AggregateTestRunner.of(List.of(unsupported, suppported));
+        CompletableFuture<?> future = runner.test(null, null);
+        assertFalse(future.isDone());
+        assertNull(unsupported.future);
+        assertNotNull(suppported.future);
+        suppported.future.complete(null);
+        assertTrue(future.isDone());
+    }
+
+    @Test
+    void noTestsResultInFailure() {
+        TestRunner runner = AggregateTestRunner.of(List.of(new MockTestRunner(false)));
+        assertEquals("No tests were found", runner.getLog(-1).iterator().next().getMessage());
+        assertSame(FAILURE, runner.getStatus());
+    }
+
+    @Test
+    void chainedRunners() {
+        LogRecord record1 = new LogRecord(Level.INFO, "one");
+        LogRecord record2 = new LogRecord(Level.INFO, "two");
+        MockTestRunner first = new MockTestRunner(true);
+        MockTestRunner second = new MockTestRunner(true);
+        TestRunner runner = AggregateTestRunner.of(List.of(first, second));
+        assertSame(NOT_STARTED, runner.getStatus());
+        assertEquals(List.of(), runner.getLog(-1));
+
+        // First wrapped runner is started.
+        CompletableFuture<?> future = runner.test(null, null);
+        assertNotNull(first.future);
+        assertNull(second.future);
+        assertEquals(RUNNING, runner.getStatus());
+
+        // Logs from first wrapped runner are returned.
+        assertEquals(List.of(), runner.getLog(-1));
+        first.log.add(record1);
+        assertEquals(List.of(record1), runner.getLog(-1));
+        assertEquals(List.of(), runner.getLog(record1.getSequenceNumber()));
+
+        // First wrapped runner completes, second is started.
+        first.status = SUCCESS;
+        first.future.complete(null);
+        assertNotNull(second.future);
+        assertFalse(future.isDone());
+        assertEquals(RUNNING, runner.getStatus());
+
+        // Logs from second runner are available.
+        second.log.add(record2);
+        assertEquals(List.of(record1, record2), runner.getLog(-1));
+
+        // No failures means success.
+        second.future.complete(null);
+        assertEquals(SUCCESS, runner.getStatus());
+
+        // A failure means failure.
+        second.status = FAILURE;
+        assertEquals(FAILURE, runner.getStatus());
+
+        // An error means error.
+        first.status = ERROR;
+        assertEquals(ERROR, runner.getStatus());
+
+        // Runner is re-used. Ensure nothing from the second wrapped runner is visible.
+        runner.test(null, null);
+        assertFalse(first.future.isDone());
+        assertTrue(second.future.isDone());
+        assertEquals(List.of(record1), runner.getLog(-1));
+        assertEquals(ERROR, runner.getStatus());
+
+        // First wrapped runner completes exceptionally, but the second should be started as usual.
+        first.future.completeExceptionally(new RuntimeException("error"));
+        assertFalse(second.future.isDone());
+        assertEquals(List.of(record1, record2), runner.getLog(-1));
+
+        // Verify reports are merged.
+        assertNull(runner.getReport());
+
+        TestReport.Failure failure = new TestReport.Failure("test", null);
+        TestReport report = TestReport.builder()
+                                      .withLogs(List.of(record1))
+                                      .withFailures(List.of(failure))
+                                      .withTotalCount(15)
+                                      .withSuccessCount(8)
+                                      .withIgnoredCount(4)
+                                      .withFailedCount(2)
+                                      .withAbortedCount(1)
+                                      .build();
+        first.report = report;
+        assertSame(report, runner.getReport());
+
+        second.report = report;
+        TestReport merged = runner.getReport();
+        assertEquals(List.of(record1, record1), merged.logLines);
+        assertEquals(List.of(failure, failure), merged.failures);
+        assertEquals(30, merged.totalCount);
+        assertEquals(16, merged.successCount);
+        assertEquals(8, merged.ignoredCount);
+        assertEquals(4, merged.failedCount);
+        assertEquals(2, merged.abortedCount);
+    }
+
+    static class MockTestRunner implements TestRunner {
+
+        final List<LogRecord> log = new ArrayList<>();
+        final boolean supported;
+        CompletableFuture<?> future;
+        Status status = NOT_STARTED;
+        TestReport report;
+
+        public MockTestRunner(boolean supported) {
+            this.supported = supported;
+        }
+
+        @Override
+        public Collection<LogRecord> getLog(long after) {
+            return log.stream().filter(record -> record.getSequenceNumber() > after).collect(toList());
+        }
+
+        @Override
+        public Status getStatus() {
+            return status;
+        }
+
+        @Override
+        public CompletableFuture<?> test(Suite suite, byte[] config) {
+            return future = new CompletableFuture<>();
+        }
+
+        @Override
+        public boolean isSupported() {
+            return supported;
+        }
+
+        @Override
+        public TestReport getReport() {
+            return report;
+        }
+
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
index a78dc077446..37ab8550357 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
@@ -1,10 +1,12 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.testrunner;
 
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.test.json.JsonTestHelper;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayOutputStream;
@@ -12,6 +14,7 @@ import java.io.IOException;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
@@ -27,13 +30,15 @@ import static org.mockito.Mockito.when;
 /**
  * @author mortent
  */
-public class TestRunnerHandlerTest {
+class TestRunnerHandlerTest {
 
     private static final Instant testInstant = Instant.ofEpochMilli(1598432151660L);
-    private static TestRunnerHandler testRunnerHandler;
 
-    @BeforeAll
-    public static void setup() {
+    private TestRunnerHandler testRunnerHandler;
+    private TestRunner aggregateRunner;
+
+    @BeforeEach
+    void setup() {
         List<LogRecord> logRecords = List.of(logRecord("Tests started"));
         Throwable exception = new RuntimeException("org.junit.ComparisonFailure: expected:<foo> but was:<bar>");
         exception.setStackTrace(new StackTraceElement[]{new StackTraceElement("Foo", "bar", "Foo.java", 1123)});
@@ -46,10 +51,8 @@ public class TestRunnerHandlerTest {
                 .withFailures(List.of(new TestReport.Failure("Foo.bar()", exception)))
                 .withLogs(logRecords).build();
 
-        testRunnerHandler = new TestRunnerHandler(
-                Executors.newSingleThreadExecutor(),
-                new MockJunitRunner(TestRunner.Status.SUCCESS, testReport),
-                null);
+        aggregateRunner = AggregateTestRunner.of(List.of(new MockJunitRunner(TestRunner.Status.SUCCESS, testReport)));
+        testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner);
     }
 
     @Test
@@ -58,11 +61,13 @@ public class TestRunnerHandlerTest {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
         JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}");
-
     }
 
     @Test
     public void returnsCorrectLog() throws IOException {
+        // Prime the aggregate runner to actually consider the wrapped runner for logs.
+        aggregateRunner.test(TestRunner.Suite.SYSTEM_TEST, new byte[0]);
+
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
@@ -73,7 +78,6 @@ public class TestRunnerHandlerTest {
         out = new ByteArrayOutputStream();
         response.render(out);
         assertEquals("{\"logRecords\":[]}", out.toString(UTF_8));
-
     }
 
     @Test
@@ -83,7 +87,7 @@ public class TestRunnerHandlerTest {
         when(testRunner.getReport()).thenReturn(null);
         testRunnerHandler = new TestRunnerHandler(
                 Executors.newSingleThreadExecutor(),
-                testRunner, null);
+                ComponentRegistry.singleton(new ComponentId("runner"), testRunner));
 
         {
             HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
@@ -105,19 +109,20 @@ public class TestRunnerHandlerTest {
         TestRunner testRunner = mock(TestRunner.class);
         when(testRunner.isSupported()).thenReturn(false);
         TestRunner legacyTestRunner = mock(TestRunner.class);
+        when(legacyTestRunner.isSupported()).thenReturn(true);
         when(legacyTestRunner.getLog(anyLong())).thenReturn(List.of(logRecord("Legacy log message")));
+        TestRunner aggregate = AggregateTestRunner.of(List.of(testRunner, legacyTestRunner));
+        testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregate);
 
-        testRunnerHandler = new TestRunnerHandler(
-                Executors.newSingleThreadExecutor(),
-                testRunner, legacyTestRunner);
-
+        // Prime the aggregate to check for logs in the wrapped runners.
+        aggregate.test(TestRunner.Suite.PRODUCTION_TEST, new byte[0]);
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
         JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}");
     }
 
-    /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization format */
+    /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */
     private static LogRecord logRecord(String logMessage) {
         LogRecord logRecord = new LogRecord(Level.INFO, logMessage);
         logRecord.setInstant(testInstant);
@@ -137,7 +142,9 @@ public class TestRunnerHandlerTest {
         }
 
         @Override
-        public void test(Suite suite, byte[] testConfig) { }
+        public CompletableFuture<?> test(Suite suite, byte[] testConfig) {
+            return CompletableFuture.completedFuture(null);
+        }
 
         @Override
         public Collection<LogRecord> getLog(long after) {
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
index 8aa2cd7bc72..06f7d317b0e 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.SortedMap;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.function.Function;
 import java.util.logging.Level;
@@ -112,24 +113,32 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
         return builder;
     }
 
-    public synchronized void test(Suite suite, byte[] testConfig) {
+    @Override
+    public synchronized CompletableFuture<?> test(Suite suite, byte[] testConfig) {
         if (status == Status.RUNNING)
             throw new IllegalArgumentException("Tests are already running; should not receive this request now.");
 
         log.clear();
         status = Status.RUNNING;
 
-        new Thread(() -> runTests(toProfile(suite), testConfig)).start();
+        return CompletableFuture.runAsync(() -> runTests(toProfile(suite), testConfig));
     }
 
+    @Override
     public Collection<LogRecord> getLog(long after) {
         return log.tailMap(after + 1).values();
     }
 
+    @Override
     public synchronized Status getStatus() {
         return status;
     }
 
+    @Override
+    public boolean isSupported() {
+        return listFiles(artifactsPath).stream().anyMatch(file -> file.toString().endsWith("tests.jar"));
+    }
+
     private void runTests(TestProfile testProfile, byte[] testConfig) {
         ProcessBuilder builder = testBuilder.apply(testProfile);
         {