Revert "Jonmv/vespa cli test runner"

7 files changed, 260 insertions, 160 deletions

diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
index d1acc4faf0f..87b98c8efc1 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -11,6 +11,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.io.IOUtils;
 import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
 import org.junit.jupiter.engine.JupiterTestEngine;
 import org.junit.platform.engine.discovery.DiscoverySelectors;
 import org.junit.platform.launcher.Launcher;
@@ -25,12 +26,10 @@ import org.osgi.framework.BundleContext;
 import java.io.IOException;
 import java.net.URL;
 import java.nio.charset.Charset;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
-import java.util.SortedMap;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.logging.Level;
@@ -45,7 +44,6 @@ import java.util.stream.Stream;
 public class JunitRunner extends AbstractComponent implements TestRunner {
     private static final Logger logger = Logger.getLogger(JunitRunner.class.getName());
 
-    private final SortedMap<Long, LogRecord> logRecords = new ConcurrentSkipListMap<>();
     private final BundleContext bundleContext;
     private final TestRuntimeProvider testRuntimeProvider;
     private volatile Future<TestReport> execution;
@@ -86,23 +84,12 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         credentialsRoot.ifPresent(root -> System.setProperty("vespa.test.credentials.root", root));
     }
 
-    private static TestDescriptor.TestCategory toCategory(TestRunner.Suite testProfile) {
-        switch(testProfile) {
-            case SYSTEM_TEST: return TestDescriptor.TestCategory.systemtest;
-            case STAGING_SETUP_TEST: return TestDescriptor.TestCategory.stagingsetuptest;
-            case STAGING_TEST: return TestDescriptor.TestCategory.stagingtest;
-            case PRODUCTION_TEST: return TestDescriptor.TestCategory.productiontest;
-            default: throw new RuntimeException("Unknown test profile: " + testProfile.name());
-        }
-    }
-
     @Override
-    public void test(TestRunner.Suite suite, byte[] testConfig) {
-        if (execution != null && ! execution.isDone()) {
+    public void executeTests(TestDescriptor.TestCategory category, byte[] testConfig) {
+        if (execution != null && !execution.isDone()) {
             throw new IllegalStateException("Test execution already in progress");
         }
         try {
-            logRecords.clear();
             testRuntimeProvider.initialize(testConfig);
             Optional<Bundle> testBundle = findTestBundle();
             if (testBundle.isEmpty()) {
@@ -113,17 +100,12 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
             if (testDescriptor.isEmpty()) {
                 throw new RuntimeException("Could not find test descriptor");
             }
-            execution = CompletableFuture.supplyAsync(() -> launchJunit(loadClasses(testBundle.get(), testDescriptor.get(), toCategory(suite))));
+            execution =  CompletableFuture.supplyAsync(() -> launchJunit(loadClasses(testBundle.get(), testDescriptor.get(), category)));
         } catch (Exception e) {
             execution = CompletableFuture.completedFuture(createReportWithFailedInitialization(e));
         }
     }
 
-    @Override
-    public Collection<LogRecord> getLog(long after) {
-        return logRecords.tailMap(after + 1).values();
-    }
-
     private static TestReport createReportWithFailedInitialization(Exception exception) {
         TestReport.Failure failure = new TestReport.Failure("init", exception);
         return new TestReport.Builder()
@@ -193,7 +175,8 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         Launcher launcher = LauncherFactory.create(launcherConfig);
 
         // Create log listener:
-        var logListener = VespaJunitLogListener.forBiConsumer((t, m) -> log(logRecords, m.get(), t));
+        var logLines = new ArrayList<LogRecord>();
+        var logListener = VespaJunitLogListener.forBiConsumer((t, m) -> log(logLines, m.get(), t));
         // Create a summary listener:
         var summaryListener = new SummaryGeneratingListener();
         launcher.registerTestExecutionListeners(logListener, summaryListener);
@@ -210,14 +193,14 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
                 .withIgnoredCount(report.getTestsSkippedCount())
                 .withFailedCount(report.getTestsFailedCount())
                 .withFailures(failures)
-                .withLogs(logRecords.values())
+                .withLogs(logLines)
                 .build();
     }
 
-    private void log(SortedMap<Long, LogRecord> logs, String message, Throwable t) {
+    private void log(List<LogRecord> logs, String message, Throwable t) {
         LogRecord logRecord = new LogRecord(Level.INFO, message);
         Optional.ofNullable(t).ifPresent(logRecord::setThrown);
-        logs.put(logRecord.getSequenceNumber(), logRecord);
+        logs.add(logRecord);
     }
 
     @Override
@@ -226,20 +209,20 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
     }
 
     @Override
-    public TestRunner.Status getStatus() {
-        if (execution == null) return TestRunner.Status.NOT_STARTED;
-        if (!execution.isDone()) return TestRunner.Status.RUNNING;
+    public LegacyTestRunner.Status getStatus() {
+        if (execution == null) return LegacyTestRunner.Status.NOT_STARTED;
+        if (!execution.isDone()) return LegacyTestRunner.Status.RUNNING;
         try {
             TestReport report = execution.get();
             if (report.isSuccess()) {
-                return TestRunner.Status.SUCCESS;
+                return LegacyTestRunner.Status.SUCCESS;
             } else {
-                return TestRunner.Status.FAILURE;
+                return LegacyTestRunner.Status.FAILURE;
             }
         } catch (InterruptedException|ExecutionException e) {
             logger.log(Level.WARNING, "Error while getting test report", e);
             // Return FAILURE to enforce getting the test report from the caller.
-            return TestRunner.Status.FAILURE;
+            return LegacyTestRunner.Status.FAILURE;
         }
     }
 
@@ -259,4 +242,10 @@ public class JunitRunner extends AbstractComponent implements TestRunner {
         }
     }
 
+    @Override
+    public String getReportAsJson() {
+        return Optional.ofNullable(getReport())
+                .map(TestReport::toJson)
+                .orElse("");
+    }
 }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
index b3ca47e2480..9a1200d0bf3 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java
@@ -1,7 +1,13 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.testrunner;
 
-import java.util.Collection;
+import com.yahoo.exception.ExceptionUtils;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.yolean.Exceptions;
+
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.logging.LogRecord;
@@ -10,16 +16,15 @@ import java.util.logging.LogRecord;
  * @author mortent
  */
 public class TestReport {
-
-    final long totalCount;
-    final long successCount;
-    final long failedCount;
-    final long ignoredCount;
-    final long abortedCount;
-    final List<Failure> failures;
-    final List<LogRecord> logLines;
-
-    private TestReport(long totalCount, long successCount, long failedCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) {
+    private final long totalCount;
+    private final long successCount;
+    private final long failedCount;
+    private final long ignoredCount;
+    private final long abortedCount;
+    private final List<Failure> failures;
+    private final List<LogRecord> logLines;
+
+    public TestReport(long totalCount, long successCount, long failedCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) {
         this.totalCount = totalCount;
         this.successCount = successCount;
         this.failedCount = failedCount;
@@ -29,6 +34,31 @@ public class TestReport {
         this.logLines = logLines;
     }
 
+    private void serializeFailure(Failure failure, Cursor slime) {
+        var testIdentifier = failure.testId();
+        slime.setString("testName", failure.testId());
+        slime.setString("testError",failure.exception().getMessage());
+        slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.exception()));
+    }
+
+    public String toJson() {
+        var slime = new Slime();
+        var root = slime.setObject();
+        var summary = root.setObject("summary");
+        summary.setLong("total", totalCount);
+        summary.setLong("success", successCount);
+        summary.setLong("failed", failedCount);
+        summary.setLong("ignored", ignoredCount);
+        summary.setLong("aborted", abortedCount);
+        var failureRoot = summary.setArray("failures");
+        this.failures.forEach(failure -> serializeFailure(failure, failureRoot.addObject()));
+
+        var output = root.setArray("output");
+        logLines.forEach(lr -> output.addString(lr.getMessage()));
+
+        return Exceptions.uncheck(() -> new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8));
+    }
+
     public List<LogRecord> logLines() {
         return logLines;
     }
@@ -41,9 +71,7 @@ public class TestReport {
         return new Builder();
     }
 
-
     public static class Builder {
-
         private long totalCount;
         private long successCount;
         private long failedCount;
@@ -60,22 +88,18 @@ public class TestReport {
             this.totalCount = totalCount;
             return this;
         }
-
         public Builder withSuccessCount(long successCount) {
             this.successCount = successCount;
             return this;
         }
-
         public Builder withFailedCount(long failedCount) {
             this.failedCount = failedCount;
             return this;
         }
-
         public Builder withIgnoredCount(long ignoredCount) {
             this.ignoredCount = ignoredCount;
             return this;
         }
-
         public Builder withAbortedCount(long abortedCount) {
             this.abortedCount = abortedCount;
             return this;
@@ -86,14 +110,12 @@ public class TestReport {
             return this;
         }
 
-        public Builder withLogs(Collection<LogRecord> logRecords) {
-            this.logLines = List.copyOf(logRecords);
+        public Builder withLogs(List<LogRecord> logRecords) {
+            this.logLines = logRecords;
             return this;
         }
-
     }
 
-
     public static class Failure {
         private final String testId;
         private final Throwable exception;
@@ -110,7 +132,5 @@ public class TestReport {
         public Throwable exception() {
             return exception;
         }
-
     }
-
 }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
index db489f5aa3d..31474d6c348 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java
@@ -1,31 +1,20 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.testrunner;
 
-import java.util.Collection;
-import java.util.logging.LogRecord;
+import ai.vespa.hosted.api.TestDescriptor;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
 
 /**
- * @author jonmv
  * @author mortent
  */
 public interface TestRunner {
+    void executeTests(TestDescriptor.TestCategory category, byte[] testConfig);
 
-    Collection<LogRecord> getLog(long after);
+    boolean isSupported();
 
-    Status getStatus();
+    LegacyTestRunner.Status getStatus();
 
-    void test(Suite suite, byte[] config);
+    TestReport getReport();
 
-    default boolean isSupported() { return true; }
-
-    default TestReport getReport() { return null; }
-
-    enum Status {
-        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
-    }
-
-    enum Suite {
-        SYSTEM_TEST, STAGING_SETUP_TEST, STAGING_TEST, PRODUCTION_TEST
-    }
-
-}
\ No newline at end of file
+    String getReportAsJson();
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
index da2db3798c2..4c359071fc9 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java
@@ -1,26 +1,34 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.testrunner;
 
+import ai.vespa.hosted.api.TestDescriptor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.inject.Inject;
 import com.yahoo.container.jdisc.EmptyResponse;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
-import com.yahoo.exception.ExceptionUtils;
-import com.yahoo.restapi.MessageResponse;
-import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
+import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
+import java.util.stream.Collectors;
 
 import static com.yahoo.jdisc.Response.Status;
 
@@ -31,12 +39,18 @@ import static com.yahoo.jdisc.Response.Status;
  */
 public class TestRunnerHandler extends LoggingRequestHandler {
 
-    private final TestRunner testRunner;
+    private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json";
+
+    private final TestRunner junitRunner;
+    private final LegacyTestRunner testRunner;
+    private final boolean useOsgiMode;
 
     @Inject
-    public TestRunnerHandler(Executor executor, TestRunner junitRunner, TestRunner testRunner) {
+    public TestRunnerHandler(Executor executor, TestRunner junitRunner, LegacyTestRunner testRunner) {
         super(executor);
-        this.testRunner = junitRunner.isSupported() ? junitRunner : testRunner;
+        this.junitRunner = junitRunner;
+        this.testRunner = testRunner;
+        this.useOsgiMode = junitRunner.isSupported();
     }
 
     @Override
@@ -46,48 +60,81 @@ public class TestRunnerHandler extends LoggingRequestHandler {
                 case GET: return handleGET(request);
                 case POST: return handlePOST(request);
 
-                default: return new MessageResponse(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported");
+                default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported");
             }
         } catch (IllegalArgumentException e) {
-            return new MessageResponse(Status.BAD_REQUEST, Exceptions.toMessageString(e));
+            return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e));
         } catch (Exception e) {
             log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
-            return new MessageResponse(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e));
+            return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e));
         }
     }
 
     private HttpResponse handleGET(HttpRequest request) {
         String path = request.getUri().getPath();
-        switch (path) {
-            case "/tester/v1/log":
+        if (path.equals("/tester/v1/log")) {
+            if (useOsgiMode) {
                 long fetchRecordsAfter = Optional.ofNullable(request.getProperty("after"))
-                                                 .map(Long::parseLong)
-                                                 .orElse(-1L);
-                return new SlimeJsonResponse(logToSlime(testRunner.getLog(fetchRecordsAfter)));
-            case "/tester/v1/status":
+                        .map(Long::parseLong)
+                        .orElse(-1L);
+
+                List<LogRecord> logRecords = Optional.ofNullable(junitRunner.getReport())
+                        .map(TestReport::logLines)
+                        .orElse(Collections.emptyList()).stream()
+                        .filter(record -> record.getSequenceNumber()>fetchRecordsAfter)
+                        .collect(Collectors.toList());
+                return new SlimeJsonResponse(logToSlime(logRecords));
+            } else {
+                return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after")
+                        ? Long.parseLong(request.getProperty("after"))
+                        : -1)));
+            }
+        } else if (path.equals("/tester/v1/status")) {
+            if (useOsgiMode) {
+                log.info("Responding with status " + junitRunner.getStatus());
+                return new Response(junitRunner.getStatus().name());
+            } else {
                 log.info("Responding with status " + testRunner.getStatus());
-                return new MessageResponse(testRunner.getStatus().name());
-            case "/tester/v1/report":
-                TestReport report = testRunner.getReport();
-                if (report == null)
-                    return new EmptyResponse(200);
-
-                return new SlimeJsonResponse(toSlime(report));
+                return new Response(testRunner.getStatus().name());
+            }
+        } else if (path.equals("/tester/v1/report")) {
+            if (useOsgiMode) {
+                String report = junitRunner.getReportAsJson();
+                return new SlimeJsonResponse(SlimeUtils.jsonToSlime(report));
+            } else {
+                return new EmptyResponse(200);
+            }
         }
-        return new MessageResponse(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
     }
 
     private HttpResponse handlePOST(HttpRequest request) throws IOException {
         final String path = request.getUri().getPath();
         if (path.startsWith("/tester/v1/run/")) {
             String type = lastElement(path);
-            TestRunner.Suite testSuite = TestRunner.Suite.valueOf(type.toUpperCase() + "_TEST");
+            TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST");
             byte[] config = request.getData().readAllBytes();
-            testRunner.test(testSuite, config);
-            log.info("Started tests of type " + type + " and status is " + testRunner.getStatus());
-            return new MessageResponse("Successfully started " + type + " tests");
+            if (useOsgiMode) {
+                junitRunner.executeTests(categoryFromProfile(testProfile), config);
+                log.info("Started tests of type " + type + " and status is " + junitRunner.getStatus());
+                return new Response("Successfully started " + type + " tests");
+            } else {
+                testRunner.test(testProfile, config);
+                log.info("Started tests of type " + type + " and status is " + testRunner.getStatus());
+                return new Response("Successfully started " + type + " tests");
+            }
+        }
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+    }
+
+    TestDescriptor.TestCategory categoryFromProfile(TestProfile testProfile) {
+        switch(testProfile) {
+            case SYSTEM_TEST: return TestDescriptor.TestCategory.systemtest;
+            case STAGING_SETUP_TEST: return TestDescriptor.TestCategory.stagingsetuptest;
+            case STAGING_TEST: return TestDescriptor.TestCategory.stagingtest;
+            case PRODUCTION_TEST: return TestDescriptor.TestCategory.productiontest;
+            default: throw new RuntimeException("Unknown test profile: " + testProfile.name());
         }
-        return new MessageResponse(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
     }
 
     private static String lastElement(String path) {
@@ -130,32 +177,48 @@ public class TestRunnerHandler extends LoggingRequestHandler {
                 : "error";
     }
 
-    private static Slime toSlime(TestReport testReport) {
-        var slime = new Slime();
-        var root = slime.setObject();
-        if (testReport == null)
-            return slime;
+    private static class SlimeJsonResponse extends HttpResponse {
+        private final Slime slime;
 
-        var summary = root.setObject("summary");
-        summary.setLong("total", testReport.totalCount);
-        summary.setLong("success", testReport.successCount);
-        summary.setLong("failed", testReport.failedCount);
-        summary.setLong("ignored", testReport.ignoredCount);
-        summary.setLong("aborted", testReport.abortedCount);
-        var failureRoot = summary.setArray("failures");
-        testReport.failures.forEach(failure -> serializeFailure(failure, failureRoot.addObject()));
+        private SlimeJsonResponse(Slime slime) {
+            super(200);
+            this.slime = slime;
+        }
 
-        var output = root.setArray("output");
-        testReport.logLines.forEach(lr -> output.addString(lr.getMessage()));
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            new JsonFormat(true).encode(outputStream, slime);
+        }
 
-        return slime;
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
     }
 
-    private static void serializeFailure(TestReport.Failure failure, Cursor slime) {
-        var testIdentifier = failure.testId();
-        slime.setString("testName", failure.testId());
-        slime.setString("testError",failure.exception().getMessage());
-        slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.exception()));
-    }
+    private static class Response extends HttpResponse {
+        private static final ObjectMapper objectMapper = new ObjectMapper();
+        private final String message;
 
+        private Response(String response) {
+            this(200, response);
+        }
+
+        private Response(int statusCode, String message) {
+            super(statusCode);
+            this.message = message;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            ObjectNode objectNode = objectMapper.createObjectNode();
+            objectNode.put("message", message);
+            objectMapper.writeValue(outputStream, objectNode);
+        }
+
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
+    }
 }
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java
new file mode 100644
index 00000000000..418ab7fe5d0
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/LegacyTestRunner.java
@@ -0,0 +1,22 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner.legacy;
+
+import java.util.Collection;
+import java.util.logging.LogRecord;
+
+/**
+ * @author mortent
+ */
+public interface LegacyTestRunner {
+
+    Collection<LogRecord> getLog(long after);
+
+    Status getStatus();
+
+    void test(TestProfile testProfile, byte[] config);
+
+    // TODO (mortent) : This seems to be duplicated in TesterCloud.Status and expects to have the same values
+    enum Status {
+        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
+    }
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/TestProfile.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/TestProfile.java
new file mode 100644
index 00000000000..ad65d150874
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/legacy/TestProfile.java
@@ -0,0 +1,30 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner.legacy;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ */
+public enum TestProfile {
+
+    SYSTEM_TEST("system, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true),
+    STAGING_SETUP_TEST("staging-setup", false),
+    STAGING_TEST("staging, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true),
+    PRODUCTION_TEST("production, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false);
+
+    private final String group;
+    private final boolean failIfNoTests;
+
+    TestProfile(String group, boolean failIfNoTests) {
+        this.group = group;
+        this.failIfNoTests = failIfNoTests;
+    }
+
+    public String group() {
+        return group;
+    }
+
+    public boolean failIfNoTests() {
+        return failIfNoTests;
+    }
+}
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
index a78dc077446..0caac95ed34 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java
@@ -1,25 +1,24 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.testrunner;
 
+import ai.vespa.hosted.api.TestDescriptor;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.test.json.JsonTestHelper;
+import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.time.Instant;
-import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
-import java.util.stream.Collectors;
 
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -48,7 +47,7 @@ public class TestRunnerHandlerTest {
 
         testRunnerHandler = new TestRunnerHandler(
                 Executors.newSingleThreadExecutor(),
-                new MockJunitRunner(TestRunner.Status.SUCCESS, testReport),
+                new MockJunitRunner(LegacyTestRunner.Status.SUCCESS, testReport),
                 null);
     }
 
@@ -57,7 +56,7 @@ public class TestRunnerHandlerTest {
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
-        JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}");
+        JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"summary\":{\"total\":10,\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"Tests started\"]}");
 
     }
 
@@ -66,18 +65,18 @@ public class TestRunnerHandlerTest {
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
-        JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Tests started\"}]}");
+        JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Tests started\"}]}");
 
         // Should not get old log
         response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log?after=0", GET));
         out = new ByteArrayOutputStream();
         response.render(out);
-        assertEquals("{\"logRecords\":[]}", out.toString(UTF_8));
+        assertEquals("{\"logRecords\":[]}", new String(out.toByteArray()));
 
     }
 
     @Test
-    public void returnsEmptyResponsesWhenReportNotReady() throws IOException {
+    public void returnsEmptyLogWhenReportNotReady() throws IOException {
         TestRunner testRunner = mock(TestRunner.class);
         when(testRunner.isSupported()).thenReturn(true);
         when(testRunner.getReport()).thenReturn(null);
@@ -85,26 +84,17 @@ public class TestRunnerHandlerTest {
                 Executors.newSingleThreadExecutor(),
                 testRunner, null);
 
-        {
-            HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
-            response.render(out);
-            assertEquals("{\"logRecords\":[]}", out.toString(UTF_8));
-        }
-
-        {
-            HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET));
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
-            response.render(out);
-            assertEquals("", out.toString(UTF_8));
-        }
+        HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        response.render(out);
+        assertEquals("{\"logRecords\":[]}", new String(out.toByteArray()));
     }
 
     @Test
     public void usesLegacyTestRunnerWhenNotSupported() throws IOException {
         TestRunner testRunner = mock(TestRunner.class);
         when(testRunner.isSupported()).thenReturn(false);
-        TestRunner legacyTestRunner = mock(TestRunner.class);
+        LegacyTestRunner legacyTestRunner = mock(LegacyTestRunner.class);
         when(legacyTestRunner.getLog(anyLong())).thenReturn(List.of(logRecord("Legacy log message")));
 
         testRunnerHandler = new TestRunnerHandler(
@@ -114,7 +104,7 @@ public class TestRunnerHandlerTest {
         HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         response.render(out);
-        JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}");
+        JsonTestHelper.assertJsonEquals(new String(out.toByteArray()), "{\"logRecords\":[{\"id\":0,\"at\":1598432151660,\"type\":\"info\",\"message\":\"Legacy log message\"}]}");
     }
 
     /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization format */
@@ -127,23 +117,17 @@ public class TestRunnerHandlerTest {
 
     private static class MockJunitRunner implements TestRunner {
 
-        private final TestRunner.Status status;
+        private final LegacyTestRunner.Status status;
         private final TestReport testReport;
 
-        public MockJunitRunner(TestRunner.Status status, TestReport testReport) {
+        public MockJunitRunner(LegacyTestRunner.Status status, TestReport testReport) {
 
             this.status = status;
             this.testReport = testReport;
         }
 
         @Override
-        public void test(Suite suite, byte[] testConfig) { }
-
-        @Override
-        public Collection<LogRecord> getLog(long after) {
-            return getReport().logLines().stream()
-                              .filter(entry -> entry.getSequenceNumber() > after)
-                              .collect(Collectors.toList());
+        public void executeTests(TestDescriptor.TestCategory category, byte[] testConfig) {
         }
 
         @Override
@@ -152,7 +136,7 @@ public class TestRunnerHandlerTest {
         }
 
         @Override
-        public TestRunner.Status getStatus() {
+        public LegacyTestRunner.Status getStatus() {
             return status;
         }
 
@@ -161,6 +145,9 @@ public class TestRunnerHandlerTest {
             return testReport;
         }
 
+        @Override
+        public String getReportAsJson() {
+            return getReport().toJson();
+        }
     }
-
 }