Merge pull request #13788 from vespa-engine/mortent/delegate-testrunner

Run tests with osgi testrunner if supported

18 files changed, 434 insertions, 152 deletions

diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/TestDescriptor.java b/hosted-api/src/main/java/ai/vespa/hosted/api/TestDescriptor.java
index 08cd3932ae7..6074bd73a20 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/TestDescriptor.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/TestDescriptor.java
@@ -26,6 +26,7 @@ public class TestDescriptor {
     private static final String JSON_FIELD_CONFIGURED_TESTS = "configuredTests";
     private static final String JSON_FIELD_SYSTEM_TESTS = "systemTests";
     private static final String JSON_FIELD_STAGING_TESTS = "stagingTests";
+    private static final String JSON_FIELD_STAGING_SETUP_TESTS = "stagingSetupTests";
     private static final String JSON_FIELD_PRODUCTION_TESTS = "productionTests";
 
     private final Map<TestCategory, List<String>> configuredTestClasses;
@@ -43,20 +44,22 @@ public class TestDescriptor {
         var testRoot = root.field(JSON_FIELD_CONFIGURED_TESTS);
         var systemTests = getJsonArray(testRoot, JSON_FIELD_SYSTEM_TESTS);
         var stagingTests = getJsonArray(testRoot, JSON_FIELD_STAGING_TESTS);
+        var stagingSetupTests = getJsonArray(testRoot, JSON_FIELD_STAGING_SETUP_TESTS);
         var productionTests = getJsonArray(testRoot, JSON_FIELD_PRODUCTION_TESTS);
-        return new TestDescriptor(version, toMap(systemTests, stagingTests, productionTests));
+        return new TestDescriptor(version, toMap(systemTests, stagingTests, stagingSetupTests, productionTests));
     }
 
     public static TestDescriptor from(
-            String version, List<String> systemTests, List<String> stagingTests, List<String> productionTests) {
-        return new TestDescriptor(version, toMap(systemTests, stagingTests, productionTests));
+            String version, List<String> systemTests, List<String> stagingTests, List<String> stagingSetupTests, List<String> productionTests) {
+        return new TestDescriptor(version, toMap(systemTests, stagingTests, stagingSetupTests, productionTests));
     }
 
     private static Map<TestCategory, List<String>> toMap(
-            List<String> systemTests, List<String> stagingTests, List<String> productionTests) {
+            List<String> systemTests, List<String> stagingTests, List<String> stagingSetupTests, List<String> productionTests) {
         return Map.of(
                 TestCategory.systemtest, systemTests,
                 TestCategory.stagingtest, stagingTests,
+                TestCategory.stagingsetuptest, stagingSetupTests,
                 TestCategory.productiontest, productionTests
         );
     }
@@ -81,6 +84,7 @@ public class TestDescriptor {
         addJsonArrayForTests(tests, JSON_FIELD_SYSTEM_TESTS, TestCategory.systemtest);
         addJsonArrayForTests(tests, JSON_FIELD_STAGING_TESTS, TestCategory.stagingtest);
         addJsonArrayForTests(tests, JSON_FIELD_PRODUCTION_TESTS, TestCategory.productiontest);
+        addJsonArrayForTests(tests, JSON_FIELD_STAGING_SETUP_TESTS, TestCategory.stagingsetuptest);
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         uncheck(() -> new JsonFormat(/*compact*/false).encode(out, slime));
         return out.toString();
@@ -100,5 +104,5 @@ public class TestDescriptor {
                '}';
     }
 
-    public enum TestCategory {systemtest, stagingtest, productiontest}
+    public enum TestCategory {systemtest, stagingsetuptest, stagingtest, productiontest}
 }
diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java
index 7e59af9ced8..d78526c500b 100644
--- a/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java
+++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java
@@ -33,6 +33,9 @@ public class TestDescriptorTest {
         var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest);
         Assertions.assertIterableEquals(Collections.emptyList(), stagingTests);
 
+        var stagingSetupTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest);
+        Assertions.assertIterableEquals(Collections.emptyList(), stagingSetupTests);
+
         var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest);
         Assertions.assertIterableEquals(Collections.emptyList(), productionTests);
     }
@@ -40,7 +43,8 @@ public class TestDescriptorTest {
     @Test
     public void parsesDescriptorFile() {
         String testDescriptor = "{\n" +
-                                "  \"version\": \"1.0\",\n" +
+                                "  \"" +
+                                "version\": \"1.0\",\n" +
                                 "  \"configuredTests\": {\n" +
                                 "    \"systemTests\": [\n" +
                                 "      \"ai.vespa.test.SystemTest1\",\n" +
@@ -50,6 +54,10 @@ public class TestDescriptorTest {
                                 "      \"ai.vespa.test.StagingTest1\",\n" +
                                 "      \"ai.vespa.test.StagingTest2\"\n" +
                                 "    ],\n" +
+                                "    \"stagingSetupTests\": [\n" +
+                                "      \"ai.vespa.test.StagingSetupTest1\",\n" +
+                                "      \"ai.vespa.test.StagingSetupTest2\"\n" +
+                                "    ],\n" +
                                 "    \"productionTests\": [\n" +
                                 "      \"ai.vespa.test.ProductionTest1\",\n" +
                                 "      \"ai.vespa.test.ProductionTest2\"\n" +
@@ -65,8 +73,13 @@ public class TestDescriptorTest {
         var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest);
         Assertions.assertIterableEquals(List.of("ai.vespa.test.StagingTest1", "ai.vespa.test.StagingTest2"), stagingTests);
 
+        var stagingSetupTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingsetuptest);
+        Assertions.assertIterableEquals(List.of("ai.vespa.test.StagingSetupTest1", "ai.vespa.test.StagingSetupTest2"), stagingSetupTests);
+
         var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest);
         Assertions.assertIterableEquals(List.of("ai.vespa.test.ProductionTest1", "ai.vespa.test.ProductionTest2"), productionTests);
+
+        JsonTestHelper.assertJsonEquals(testClassDescriptor.toJson(), testDescriptor);
     }
 
     @Test
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
index 8309b7a8124..259ae2602c4 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
@@ -33,6 +33,7 @@ public class GenerateTestDescriptorMojo extends AbstractMojo {
                 TestDescriptor.CURRENT_VERSION,
                 analyzer.systemTests(),
                 analyzer.stagingTests(),
+                analyzer.stagingSetupTests(),
                 analyzer.productionTests());
         writeDescriptorFile(descriptor);
     }
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/TestAnnotationAnalyzer.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/TestAnnotationAnalyzer.java
index c45ef21bc31..e8b29b2b0f7 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/TestAnnotationAnalyzer.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/TestAnnotationAnalyzer.java
@@ -3,6 +3,7 @@ package ai.vespa.hosted.plugin;
 
 
 import ai.vespa.hosted.cd.ProductionTest;
+import ai.vespa.hosted.cd.StagingSetup;
 import ai.vespa.hosted.cd.StagingTest;
 import ai.vespa.hosted.cd.SystemTest;
 import org.objectweb.asm.AnnotationVisitor;
@@ -28,10 +29,12 @@ class TestAnnotationAnalyzer {
 
     private final List<String> systemTests = new ArrayList<>();
     private final List<String> stagingTests = new ArrayList<>();
+    private final List<String> stagingSetupTests = new ArrayList<>();
     private final List<String> productionTests = new ArrayList<>();
 
     List<String> systemTests() { return systemTests; }
     List<String> stagingTests() { return stagingTests; }
+    List<String> stagingSetupTests() { return stagingSetupTests; }
     List<String> productionTests() { return productionTests; }
 
     void analyzeClass(Path classFile) {
@@ -65,6 +68,8 @@ class TestAnnotationAnalyzer {
                 productionTests.add(className);
             } else if (StagingTest.class.getName().equals(annotationClassName)) {
                 stagingTests.add(className);
+            } else if (StagingSetup.class.getName().equals(annotationClassName)) {
+                stagingTests.add(className);
             } else if (SystemTest.class.getName().equals(annotationClassName)) {
                 systemTests.add(className);
             }
diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml
index 62ea578f14f..db0dba89b8a 100644
--- a/vespa-osgi-testrunner/pom.xml
+++ b/vespa-osgi-testrunner/pom.xml
@@ -22,7 +22,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <!-- Verify that we need all junit deps -->
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-engine</artifactId>
@@ -45,18 +44,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>5.6.2</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.junit.jupiter</groupId>
-                    <artifactId>junit-jupiter-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        
+
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
             <artifactId>tenant-cd-api</artifactId>
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java
deleted file mode 100644
index cb7d5b8df6b..00000000000
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.testrunner;
-
-import ai.vespa.hosted.api.TestDescriptor;
-import ai.vespa.hosted.cd.internal.TestRuntimeProvider;
-import com.google.inject.Inject;
-import com.yahoo.container.handler.metrics.JsonResponse;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.LoggingRequestHandler;
-import com.yahoo.container.logging.AccessLog;
-import com.yahoo.restapi.ErrorResponse;
-import com.yahoo.restapi.MessageResponse;
-import org.osgi.framework.Bundle;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.Executor;
-import java.util.function.Function;
-
-/**
- * @author mortent
- */
-public class JunitHandler extends LoggingRequestHandler {
-
-    private final JunitRunner junitRunner;
-    private final TestRuntimeProvider testRuntimeProvider;
-
-    @Inject
-    public JunitHandler(Executor executor, AccessLog accessLog, JunitRunner junitRunner, TestRuntimeProvider testRuntimeProvider) {
-        super(executor, accessLog);
-        this.junitRunner = junitRunner;
-        this.testRuntimeProvider = testRuntimeProvider;
-    }
-
-    @Override
-    public HttpResponse handle(HttpRequest httpRequest) {
-        String mode = property("mode", "help", httpRequest, String::valueOf);
-        TestDescriptor.TestCategory category = property("category", TestDescriptor.TestCategory.systemtest, httpRequest, TestDescriptor.TestCategory::valueOf);
-
-        try {
-            testRuntimeProvider.initialize(httpRequest.getData().readAllBytes());
-        } catch (IOException e) {
-            return new ErrorResponse(500, "testruntime-initialization", "Exception reading test config");
-        }
-
-        if ("help".equalsIgnoreCase(mode)) {
-            return new MessageResponse("Accepted modes: \n help \n list \n execute");
-        }
-
-        if (!"list".equalsIgnoreCase(mode) && !"execute".equalsIgnoreCase(mode)) {
-            return new ErrorResponse(400, "client error", "Unknown mode \"" + mode + "\"");
-        }
-
-        Bundle testBundle = junitRunner.findTestBundle("-tests");
-        TestDescriptor testDescriptor = junitRunner.loadTestDescriptor(testBundle);
-        List<Class<?>> testClasses = junitRunner.loadClasses(testBundle, testDescriptor, category);
-
-        String jsonResponse = junitRunner.executeTests(testClasses);
-
-        return new JsonResponse(200, jsonResponse);
-    }
-
-    private static <VAL> VAL property(String name, VAL defaultValue, HttpRequest request, Function<String, VAL> converter) {
-        final String propertyString = request.getProperty(name);
-        if (propertyString != null) {
-            return converter.apply(propertyString);
-        }
-        return defaultValue;
-    }
-}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index 69134f86be0..3fc85365084 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -2,15 +2,12 @@
 package com.yahoo.vespa.testrunner;
 
 import ai.vespa.hosted.api.TestDescriptor;
+import ai.vespa.hosted.cd.internal.TestRuntimeProvider;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
-import com.yahoo.exception.ExceptionUtils;
 import com.yahoo.io.IOUtils;
 import com.yahoo.jdisc.application.OsgiFramework;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.slime.SlimeUtils;
-import com.yahoo.yolean.Exceptions;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
 import org.junit.jupiter.engine.JupiterTestEngine;
 import org.junit.platform.engine.discovery.DiscoverySelectors;
 import org.junit.platform.launcher.Launcher;
@@ -20,16 +17,19 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
 import org.junit.platform.launcher.core.LauncherFactory;
 import org.junit.platform.launcher.listeners.LoggingListener;
 import org.junit.platform.launcher.listeners.SummaryGeneratingListener;
-import org.junit.platform.launcher.listeners.TestExecutionSummary;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 
 import java.io.IOException;
 import java.net.URL;
 import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -41,10 +41,12 @@ public class JunitRunner extends AbstractComponent {
     private static final Logger logger = Logger.getLogger(JunitRunner.class.getName());
 
     private final BundleContext bundleContext;
+    private final TestRuntimeProvider testRuntimeProvider;
+    private Future<TestReport> execution;
 
     @Inject
-    public JunitRunner(OsgiFramework osgiFramework) {
-        // TODO mortent: Find a way to workaround this hack
+    public JunitRunner(OsgiFramework osgiFramework, TestRuntimeProvider testRuntimeProvider) {
+        this.testRuntimeProvider = testRuntimeProvider;
         var tmp = osgiFramework.bundleContext();
         try {
             var field = tmp.getClass().getDeclaredField("wrapped");
@@ -55,27 +57,54 @@ public class JunitRunner extends AbstractComponent {
         }
     }
 
-    public Bundle findTestBundle(String bundleNameSuffix) {
+    public void executeTests(TestDescriptor.TestCategory category, byte[] testConfig) {
+        if (execution != null) {
+            throw new RuntimeException("Test execution already in progress");
+        }
+        testRuntimeProvider.initialize(testConfig);
+        Optional<Bundle> testBundle = findTestBundle();
+        if (testBundle.isEmpty()) {
+            throw new RuntimeException("No test bundle available");
+        }
+
+        Optional<TestDescriptor> testDescriptor = loadTestDescriptor(testBundle.get());
+        if (testDescriptor.isEmpty()) {
+            throw new RuntimeException("Could not find test descriptor");
+        }
+        List<Class<?>> testClasses = loadClasses(testBundle.get(), testDescriptor.get(), category);
+
+        execution =  CompletableFuture.supplyAsync(() -> launchJunit(testClasses));
+    }
+
+    public boolean isSupported() {
+        return findTestBundle().isPresent();
+    }
+
+    private Optional<Bundle> findTestBundle() {
         return Stream.of(bundleContext.getBundles())
-                .filter(bundle -> bundle.getSymbolicName().endsWith(bundleNameSuffix))
-                .findAny()
-                .orElseThrow(() -> new RuntimeException("No bundle on classpath with name ending on " + bundleNameSuffix));
+                .filter(this::isTestBundle)
+                .findAny();
     }
 
-    public TestDescriptor loadTestDescriptor(Bundle bundle) {
+    private boolean isTestBundle(Bundle bundle) {
+        var testBundleHeader = bundle.getHeaders().get("X-JDisc-Test-Bundle-Version");
+        return testBundleHeader != null && !testBundleHeader.isBlank();
+    }
+
+    private Optional<TestDescriptor> loadTestDescriptor(Bundle bundle) {
         URL resource = bundle.getEntry(TestDescriptor.DEFAULT_FILENAME);
         TestDescriptor testDescriptor;
         try {
             var jsonDescriptor = IOUtils.readAll(resource.openStream(), Charset.defaultCharset()).trim();
             testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor);
             logger.info( "Test classes in bundle :" + testDescriptor.toString());
-            return testDescriptor;
+            return Optional.of(testDescriptor);
         } catch (IOException e) {
-            throw new RuntimeException("Could not load " + TestDescriptor.DEFAULT_FILENAME + " [" + e.getMessage() + "]");
+            return Optional.empty();
         }
     }
 
-    public List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) {
+    private List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) {
         List<Class<?>> testClasses = testDescriptor.getConfiguredTests(testCategory).stream()
                 .map(className -> loadClass(bundle, className))
                 .collect(Collectors.toList());
@@ -94,7 +123,7 @@ public class JunitRunner extends AbstractComponent {
         }
     }
 
-    public String executeTests(List<Class<?>> testClasses) {
+    private TestReport launchJunit(List<Class<?>> testClasses) {
         LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request()
                 .selectors(
                         testClasses.stream().map(DiscoverySelectors::selectClass).collect(Collectors.toList())
@@ -116,36 +145,8 @@ public class JunitRunner extends AbstractComponent {
 
         // Execute request
         launcher.execute(discoveryRequest);
-
         var report = summaryListener.getSummary();
-
-        return createJsonTestReport(report, logLines);
-    }
-
-    private String createJsonTestReport(TestExecutionSummary report, List<String> logLines) {
-        var slime = new Slime();
-        var root = slime.setObject();
-        var summary = root.setObject("summary");
-        summary.setLong("Total tests", report.getTestsFoundCount());
-        summary.setLong("Test success", report.getTestsSucceededCount());
-        summary.setLong("Test failed", report.getTestsFailedCount());
-        summary.setLong("Test ignored", report.getTestsSkippedCount());
-        summary.setLong("Test success", report.getTestsAbortedCount());
-        summary.setLong("Test started", report.getTestsStartedCount());
-        var failures = summary.setArray("failures");
-        report.getFailures().forEach(failure -> serializeFailure(failure, failures.addObject()));
-
-        var output = root.setArray("output");
-        logLines.forEach(output::addString);
-
-        return Exceptions.uncheck(() -> new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8));
-    }
-
-    private void serializeFailure(TestExecutionSummary.Failure failure, Cursor slime) {
-        var testIdentifier = failure.getTestIdentifier();
-        slime.setString("testName", testIdentifier.getUniqueId());
-        slime.setString("testError",failure.getException().getMessage());
-        slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.getException()));
+        return new TestReport(report, logLines);
     }
 
     private void log(List<String> logs, String message, Throwable t) {
@@ -162,4 +163,33 @@ public class JunitRunner extends AbstractComponent {
     public void deconstruct() {
         super.deconstruct();
     }
+
+    public LegacyTestRunner.Status getStatus() {
+        if (execution == null) return LegacyTestRunner.Status.NOT_STARTED;
+        if (!execution.isDone()) return LegacyTestRunner.Status.RUNNING;
+        try {
+            TestReport report = execution.get();
+            if (report.isSuccess()) {
+                return LegacyTestRunner.Status.SUCCESS;
+            } else {
+                return LegacyTestRunner.Status.FAILURE;
+            }
+        } catch (InterruptedException|ExecutionException e) {
+            logger.log(Level.WARNING, "Error while getting test report", e);
+            return LegacyTestRunner.Status.ERROR;
+        }
+    }
+
+    public String getReportAsJson() {
+        if (execution.isDone()) {
+            try {
+                return execution.get().toJson();
+            } catch (Exception e) {
+                logger.log(Level.WARNING, "Error getting test report", e);
+                return "";
+            }
+        } else {
+            return "";
+        }
+    }
 }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
new file mode 100644
index 00000000000..2e45ba96486
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
@@ -0,0 +1,55 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import com.yahoo.exception.ExceptionUtils;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.yolean.Exceptions;
+import org.junit.platform.launcher.listeners.TestExecutionSummary;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+public class TestReport {
+    private final TestExecutionSummary junitReport;
+    private final List<String> logLines;
+
+    public TestReport(TestExecutionSummary junitReport, List<String> logLines) {
+        this.junitReport = junitReport;
+        this.logLines = List.copyOf(logLines);
+    }
+
+    private void serializeFailure(TestExecutionSummary.Failure failure, Cursor slime) {
+        var testIdentifier = failure.getTestIdentifier();
+        slime.setString("testName", testIdentifier.getUniqueId());
+        slime.setString("testError",failure.getException().getMessage());
+        slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.getException()));
+    }
+
+    public String toJson() {
+        var slime = new Slime();
+        var root = slime.setObject();
+        var summary = root.setObject("summary");
+        summary.setLong("Total tests", junitReport.getTestsFoundCount());
+        summary.setLong("Test success", junitReport.getTestsSucceededCount());
+        summary.setLong("Test failed", junitReport.getTestsFailedCount());
+        summary.setLong("Test ignored", junitReport.getTestsSkippedCount());
+        summary.setLong("Test aborted", junitReport.getTestsAbortedCount());
+        summary.setLong("Test started", junitReport.getTestsStartedCount());
+        var failures = summary.setArray("failures");
+        junitReport.getFailures().forEach(failure -> serializeFailure(failure, failures.addObject()));
+
+        var output = root.setArray("output");
+        logLines.forEach(output::addString);
+
+        return Exceptions.uncheck(() -> new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8));
+    }
+
+    public boolean isSuccess() {
+        return (junitReport.getTestsFailedCount() + junitReport.getTestsAbortedCount()) == 0;
+    }
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
new file mode 100644
index 00000000000..cb337a0c176
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
@@ -0,0 +1,211 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import ai.vespa.hosted.api.TestDescriptor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static com.yahoo.jdisc.Response.Status;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ * @author mortent
+ */
+public class TestRunnerHandler extends LoggingRequestHandler {
+
+    private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json";
+
+    private final JunitRunner junitRunner;
+    private final LegacyTestRunner testRunner;
+    private final boolean useOsgiMode;
+
+    @Inject
+    public TestRunnerHandler(Executor executor, AccessLog accessLog, JunitRunner junitRunner, LegacyTestRunner testRunner) {
+        super(executor, accessLog);
+        this.junitRunner = junitRunner;
+        this.testRunner = testRunner;
+        this.useOsgiMode = junitRunner.isSupported();
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        try {
+            switch (request.getMethod()) {
+                case GET: return handleGET(request);
+                case POST: return handlePOST(request);
+
+                default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported");
+            }
+        } catch (IllegalArgumentException e) {
+            return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e));
+        } catch (Exception e) {
+            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+            return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e));
+        }
+    }
+
+    private HttpResponse handleGET(HttpRequest request) {
+        String path = request.getUri().getPath();
+        if (path.equals("/tester/v1/log")) {
+            if (useOsgiMode) {
+                // TODO (mortent): Handle case where log is returned multiple times
+                String report = junitRunner.getReportAsJson();
+                List<LogRecord> logRecords = new ArrayList<>();
+                if (!report.isBlank()) {
+                    logRecords.add(new LogRecord(Level.INFO, report));
+                }
+                return new SlimeJsonResponse(logToSlime(logRecords));
+            } else {
+                return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after")
+                        ? Long.parseLong(request.getProperty("after"))
+                        : -1)));
+            }
+        } else if (path.equals("/tester/v1/status")) {
+            if (useOsgiMode) {
+                log.info("Responding with status " + junitRunner.getStatus());
+                return new Response(junitRunner.getStatus().name());
+            } else {
+                log.info("Responding with status " + testRunner.getStatus());
+                return new Response(testRunner.getStatus().name());
+            }
+        }
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+    }
+
+    private HttpResponse handlePOST(HttpRequest request) throws IOException {
+        final String path = request.getUri().getPath();
+        if (path.startsWith("/tester/v1/run/")) {
+            String type = lastElement(path);
+            TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST");
+            byte[] config = request.getData().readAllBytes();
+            if (useOsgiMode) {
+                junitRunner.executeTests(categoryFromProfile(testProfile), config);
+                log.info("Started tests of type " + type + " and status is " + junitRunner.getStatus());
+                return new Response("Successfully started " + type + " tests");
+            } else {
+                testRunner.test(testProfile, config);
+                log.info("Started tests of type " + type + " and status is " + testRunner.getStatus());
+                return new Response("Successfully started " + type + " tests");
+            }
+        }
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+    }
+
+    TestDescriptor.TestCategory categoryFromProfile(TestProfile testProfile) {
+        switch(testProfile) {
+            case SYSTEM_TEST: return TestDescriptor.TestCategory.systemtest;
+            case STAGING_SETUP_TEST: return TestDescriptor.TestCategory.stagingsetuptest;
+            case STAGING_TEST: return TestDescriptor.TestCategory.stagingtest;
+            case PRODUCTION_TEST: return TestDescriptor.TestCategory.productiontest;
+            default: throw new RuntimeException("Unknown test profile: " + testProfile.name());
+        }
+    }
+
+    private static String lastElement(String path) {
+        if (path.endsWith("/"))
+            path = path.substring(0, path.length() - 1);
+        int lastSlash = path.lastIndexOf("/");
+        if (lastSlash < 0) return path;
+        return path.substring(lastSlash + 1);
+    }
+
+    static Slime logToSlime(Collection<LogRecord> log) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor recordArray = root.setArray("logRecords");
+        logArrayToSlime(recordArray, log);
+        return slime;
+    }
+
+    static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) {
+        log.forEach(record -> {
+            Cursor recordObject = recordArray.addObject();
+            recordObject.setLong("id", record.getSequenceNumber());
+            recordObject.setLong("at", record.getMillis());
+            recordObject.setString("type", typeOf(record.getLevel()));
+            String message = record.getMessage();
+            if (record.getThrown() != null) {
+                ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+                record.getThrown().printStackTrace(new PrintStream(buffer));
+                message += "\n" + buffer;
+            }
+            recordObject.setString("message", message);
+        });
+    }
+
+    public static String typeOf(Level level) {
+        return    level.getName().equals("html") ? "html"
+                : level.intValue() < Level.INFO.intValue() ? "debug"
+                : level.intValue() < Level.WARNING.intValue() ? "info"
+                : level.intValue() < Level.SEVERE.intValue() ? "warning"
+                : "error";
+    }
+
+    private static class SlimeJsonResponse extends HttpResponse {
+        private final Slime slime;
+
+        private SlimeJsonResponse(Slime slime) {
+            super(200);
+            this.slime = slime;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            new JsonFormat(true).encode(outputStream, slime);
+        }
+
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
+    }
+
+    private static class Response extends HttpResponse {
+        private static final ObjectMapper objectMapper = new ObjectMapper();
+        private final String message;
+
+        private Response(String response) {
+            this(200, response);
+        }
+
+        private Response(int statusCode, String message) {
+            super(statusCode);
+            this.message = message;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            ObjectNode objectNode = objectMapper.createObjectNode();
+            objectNode.put("message", message);
+            objectMapper.writeValue(outputStream, objectNode);
+        }
+
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
+    }
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java
new file mode 100644
index 00000000000..9f1a68218f0
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java
@@ -0,0 +1,22 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner.legacy;
+
+import java.util.Collection;
+import java.util.logging.LogRecord;
+
+/**
+ * @author mortent
+ */
+public interface LegacyTestRunner {
+
+    Collection<LogRecord> getLog(long after);
+
+    Status getStatus();
+
+    void test(TestProfile testProfile, byte[] config);
+
+    // TODO (mortent) : This seems to be duplicated in TesterCloud.Status and expects to have the same values
+    enum Status {
+        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
+    }
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/TestProfile.java
index d568b549f9b..59576209043 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/TestProfile.java
@@ -1,11 +1,11 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.testrunner;
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner.legacy;
 
 /**
  * @author valerijf
  * @author jvenstad
  */
-enum TestProfile {
+public enum TestProfile {
 
     SYSTEM_TEST("system, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true),
     STAGING_SETUP_TEST("staging-setup", false),
@@ -20,12 +20,11 @@ enum TestProfile {
         this.failIfNoTests = failIfNoTests;
     }
 
-    String group() {
+    public String group() {
         return group;
     }
 
-    boolean failIfNoTests() {
+    public boolean failIfNoTests() {
         return failIfNoTests;
     }
-
 }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java
new file mode 100644
index 00000000000..49f6cef0c22
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/package-info.java
@@ -0,0 +1,9 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @author mortent
+ */
+@ExportPackage
+package com.yahoo.vespa.testrunner.legacy;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml
index 31568d01fb5..e780da726a1 100644
--- a/vespa-testrunner-components/pom.xml
+++ b/vespa-testrunner-components/pom.xml
@@ -24,6 +24,24 @@
         </dependency>
 
         <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespa-osgi-testrunner</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+            <!-- junit must be excluded to keep maven-surefire-plugin to be confused -->
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.jupiter</groupId>
+                    <artifactId>junit-jupiter-engine</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.junit.platform</groupId>
+                    <artifactId>junit-platform-launcher</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
             <groupId>org.fusesource.jansi</groupId>
             <artifactId>jansi</artifactId>
             <version>1.11</version>
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
index e6f402ba563..dd424de5471 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.testrunner;
 
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 
 import java.nio.file.Path;
 import java.util.List;
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
index cdf320a6304..4308b0bba4c 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.testrunner;
 
 import com.google.inject.Inject;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.fusesource.jansi.AnsiOutputStream;
 import org.fusesource.jansi.HtmlAnsiOutputStream;
 
@@ -30,14 +32,13 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.SEVERE;
 
 /**
  * @author valerijf
  * @author jvenstad
  */
-public class TestRunner {
+public class TestRunner implements LegacyTestRunner {
 
     private static final Logger logger = Logger.getLogger(TestRunner.class.getName());
     private static final Level HTML = new Level("html", 1) { };
@@ -203,9 +204,4 @@ public class TestRunner {
         }
     }
 
-
-    public enum Status {
-        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
-    }
-
 }
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
index e92dbcede5a..8f9966a898f 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
@@ -9,10 +9,10 @@ import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.io.IOUtils;
-import java.util.logging.Level;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayOutputStream;
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
index c7799bff116..823dca4a7a2 100644
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
@@ -1,6 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.testrunner;
 
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.junit.Test;
 
 import java.io.IOException;
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
index 22fd7fddf31..b2c7a77240b 100644
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.testrunner;
 
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.fusesource.jansi.Ansi;
 import org.junit.Before;
 import org.junit.Rule;
@@ -16,7 +17,6 @@ import java.util.logging.LogRecord;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * Unit tests relying on a UNIX shell >_<