Move vespa-testrunner-components here

13 files changed, 861 insertions, 0 deletions

diff --git a/vespa-testrunner-components/CMakeLists.txt b/vespa-testrunner-components/CMakeLists.txt
new file mode 100644
index 00000000000..fe2cb84b7bb
--- /dev/null
+++ b/vespa-testrunner-components/CMakeLists.txt
@@ -0,0 +1,3 @@
+install_java_artifact(vespa-testrunner-components)
+install_fat_java_artifact(vespa-testrunner-components)
+install_config_definition(src/main/resources/configdefinitions/test-runner.def test-runner.def)
diff --git a/vespa-testrunner-components/OWNERS b/vespa-testrunner-components/OWNERS
new file mode 100644
index 00000000000..134acfc20f3
--- /dev/null
+++ b/vespa-testrunner-components/OWNERS
@@ -0,0 +1 @@
+jvenstad
diff --git a/vespa-testrunner-components/README.md b/vespa-testrunner-components/README.md
new file mode 100644
index 00000000000..034ad95ac25
--- /dev/null
+++ b/vespa-testrunner-components/README.md
@@ -0,0 +1,4 @@
+# Vespa-testrunner-components
+
+Defines handler and component used by the vespa application that is deployed by the controller to
+run system/staging/production tests.
diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml
new file mode 100644
index 00000000000..80d55660bc7
--- /dev/null
+++ b/vespa-testrunner-components/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.yahoo.vespa.hosted</groupId>
+    <artifactId>vespa-testrunner-components</artifactId>
+    <packaging>container-plugin</packaging>
+
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>7-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.fusesource.jansi</groupId>
+            <artifactId>jansi</artifactId>
+            <version>1.11</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-artifacts</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>attach-artifact</goal>
+                        </goals>
+                        <configuration>
+                            <artifacts>
+                                <artifact>
+                                    <file>target/${project.artifactId}-jar-with-dependencies.jar</file>
+                                    <type>jar</type>
+                                    <classifier>deploy</classifier>
+                                </artifact>
+                            </artifacts>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
+                <version>${project.version}</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <useCommonAssemblyIds>true</useCommonAssemblyIds>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
new file mode 100644
index 00000000000..4e89e57f0ca
--- /dev/null
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
@@ -0,0 +1,108 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Generates a pom.xml file that sets up build profile to test against the provided
+ * jar artifacts.
+ *
+ * @author valerijf
+ */
+public class PomXmlGenerator {
+    private static final String PROPERTY_TEMPLATE =
+            "        <%ARTIFACT_ID%.path>%JAR_PATH%</%ARTIFACT_ID%.path>\n";
+    private static final String TEST_ARTIFACT_GROUP_ID = "com.yahoo.vespa.testrunner.test";
+    private static final String DEPENDENCY_TEMPLATE =
+            "        <dependency>\n" +
+            "            <groupId>" + TEST_ARTIFACT_GROUP_ID + "</groupId>\n" +
+            "            <artifactId>%ARTIFACT_ID%</artifactId>\n" +
+            "            <scope>system</scope>\n" +
+            "            <type>test-jar</type>\n" +
+            "            <version>test</version>\n" +
+            "            <systemPath>${%ARTIFACT_ID%.path}</systemPath>\n" +
+            "        </dependency>\n";
+    private static final String DEPENDENCY_TO_SCAN_TEMPLATE =
+            "                        <dependency>" + TEST_ARTIFACT_GROUP_ID + ":%ARTIFACT_ID%</dependency>\n";
+    private static final String POM_XML_TEMPLATE =
+            "<?xml version=\"1.0\"?>\n" +
+            "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0                              http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n" +
+            "    <modelVersion>4.0.0</modelVersion>\n" +
+            "    <groupId>com.yahoo.vespa</groupId>\n" +
+            "    <artifactId>tester-application</artifactId>\n" +
+            "    <version>1.0.0</version>\n" +
+            "\n" +
+            "    <properties>\n" +
+            "        <maven_version>4.12</maven_version>\n" +
+            "        <surefire_version>2.22.0</surefire_version>\n" +
+            "%PROPERTIES%" +
+            "    </properties>\n" +
+            "\n" +
+            "    <dependencies>\n" +
+            "        <dependency>\n" +
+            "            <groupId>junit</groupId>\n" +
+            "            <artifactId>junit</artifactId>\n" +
+            "            <version>${maven_version}</version>\n" +
+            "            <scope>test</scope>\n" +
+            "        </dependency>\n" +
+            "%DEPENDENCIES%" +
+            "    </dependencies>\n" +
+            "\n" +
+            "    <build>\n" +
+            "        <plugins>\n" +
+            "            <plugin>\n" +
+            "                <groupId>org.apache.maven.plugins</groupId>\n" +
+            "                <artifactId>maven-surefire-plugin</artifactId>\n" +
+            "                <version>${surefire_version}</version>\n" +
+            "                <configuration>\n" +
+            "                    <dependenciesToScan>\n" +
+            "%DEPENDENCIES_TO_SCAN%" +
+            "                    </dependenciesToScan>\n" +
+            "                    <groups>%GROUPS%</groups>\n" +
+            "                    <excludedGroups>com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class</excludedGroups>\n" +
+            "                    <excludes>\n" +
+            "                        <exclude>%GROUPS%</exclude>\n" +
+            "                    </excludes>\n" +
+            "                    <reportsDirectory>${env.TEST_DIR}</reportsDirectory>\n" +
+            "                    <redirectTestOutputToFile>false</redirectTestOutputToFile>\n" +
+            "                    <environmentVariables>\n" +
+            "                        <LD_LIBRARY_PATH>/home/y/lib64</LD_LIBRARY_PATH>\n" +
+            "                    </environmentVariables>\n" +
+            "                </configuration>\n" +
+            "            </plugin>\n" +
+            "            <plugin>\n" +
+            "                <groupId>org.apache.maven.plugins</groupId>\n" +
+            "                <artifactId>maven-surefire-report-plugin</artifactId>\n" +
+            "                <version>${surefire_version}</version>\n" +
+            "                <configuration>\n" +
+            "                    <reportsDirectory>${env.TEST_DIR}</reportsDirectory>\n" +
+            "                </configuration>\n" +
+            "            </plugin>\n" +
+            "        </plugins>\n" +
+            "    </build>\n" +
+            "</project>\n";
+
+    static String generatePomXml(TestProfile testProfile, List<Path> artifacts, Path testArtifact) {
+        String properties = artifacts.stream()
+                .map(path -> PROPERTY_TEMPLATE
+                        .replace("%ARTIFACT_ID%", path.getFileName().toString())
+                        .replace("%JAR_PATH%", path.toString()))
+                .collect(Collectors.joining());
+        String dependencies = artifacts.stream()
+                .map(path -> DEPENDENCY_TEMPLATE
+                        .replace("%ARTIFACT_ID%", path.getFileName().toString()))
+                .collect(Collectors.joining());
+        String dependenciesToScan =
+                DEPENDENCY_TO_SCAN_TEMPLATE
+                        .replace("%ARTIFACT_ID%", testArtifact.getFileName().toString());
+
+        return POM_XML_TEMPLATE
+                .replace("%PROPERTIES%", properties)
+                .replace("%DEPENDENCIES_TO_SCAN%", dependenciesToScan)
+                .replace("%DEPENDENCIES%", dependencies)
+                .replace("%GROUPS%", testProfile.group());
+    }
+
+    private PomXmlGenerator() {}
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java
new file mode 100644
index 00000000000..b7d3a06f30d
--- /dev/null
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java
@@ -0,0 +1,29 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ */
+enum TestProfile {
+
+    SYSTEM_TEST("com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true),
+    STAGING_TEST("com.yahoo.vespa.tenant.cd.StagingTest, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true),
+    PRODUCTION_TEST("com.yahoo.vespa.tenant.cd.ProductionTest, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false);
+
+    private final String group;
+    private final boolean failIfNoTests;
+
+    TestProfile(String group, boolean failIfNoTests) {
+        this.group = group;
+        this.failIfNoTests = failIfNoTests;
+    }
+
+    String group() {
+        return group;
+    }
+
+    boolean failIfNoTests() {
+        return failIfNoTests;
+    }
+
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
new file mode 100644
index 00000000000..fb5dccc551d
--- /dev/null
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -0,0 +1,195 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import com.google.inject.Inject;
+import com.yahoo.vespa.defaults.Defaults;
+import org.fusesource.jansi.AnsiOutputStream;
+import org.fusesource.jansi.HtmlAnsiOutputStream;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.yahoo.log.LogLevel.ERROR;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ */
+public class TestRunner {
+
+    private static final Logger logger = Logger.getLogger(TestRunner.class.getName());
+    private static final Level HTML = new Level("html", 1) { };
+    private static final Path vespaHome = Paths.get(Defaults.getDefaults().vespaHome());
+    private static final String settingsXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+                                              "<settings xmlns=\"http://maven.apache.org/SETTINGS/1.0.0\"\n" +
+                                              "          xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" +
+                                              "          xsi:schemaLocation=\"http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd\">\n" +
+                                              "    <mirrors>\n" +
+                                              "        <mirror>\n" +
+                                              "            <id>maven central</id>\n" +
+                                              "            <mirrorOf>*</mirrorOf>\n" + // Use this for everything!
+                                              "            <url>https://repo.maven.apache.org/maven2/</url>\n" +
+                                              "        </mirror>\n" +
+                                              "    </mirrors>\n" +
+                                              "</settings>";
+
+    private final Path artifactsPath;
+    private final Path testPath;
+    private final Path logFile;
+    private final Path configFile;
+    private final Path settingsFile;
+    private final Function<TestProfile, ProcessBuilder> testBuilder;
+    private final SortedMap<Long, LogRecord> log = new ConcurrentSkipListMap<>();
+
+    private volatile Status status = Status.NOT_STARTED;
+
+    @Inject
+    public TestRunner(TestRunnerConfig config) {
+        this(config.artifactsPath(),
+             vespaHome.resolve("tmp/test"),
+             vespaHome.resolve("logs/vespa/maven.log"),
+             vespaHome.resolve("tmp/config.json"),
+             vespaHome.resolve("tmp/settings.xml"),
+                     profile -> { // Anything to make this testable! >_<
+                         String[] command = new String[]{
+                                 "mvn",
+                                 "test",
+
+                                 "--batch-mode", // Run in non-interactive (batch) mode (disables output color)
+                                 "--show-version", // Display version information WITHOUT stopping build
+                                 "--settings", // Need to override repository settings in ymaven config >_<
+                                 vespaHome.resolve("tmp/settings.xml").toString(),
+
+                                 // Disable maven download progress indication
+                                 "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn",
+                                 "-Dstyle.color=always", // Enable ANSI color codes again
+                                 "-DfailIfNoTests=" + profile.failIfNoTests(),
+                                 "-Dvespa.test.config=" + vespaHome.resolve("tmp/config.json"),
+                                 "-Dvespa.test.credentials.root=" + Defaults.getDefaults().vespaHome() + "/var/vespa/sia",
+                                 String.format("-DargLine=-Xms%1$dm -Xmx%1$dm", config.surefireMemoryMb())
+                         };
+                         ProcessBuilder builder = new ProcessBuilder(command);
+                         builder.environment().merge("MAVEN_OPTS", " -Djansi.force=true", String::concat);
+                         builder.directory(vespaHome.resolve("tmp/test").toFile());
+                         builder.redirectErrorStream(true);
+                         return builder;
+                     });
+    }
+
+    TestRunner(Path artifactsPath, Path testPath, Path logFile, Path configFile, Path settingsFile, Function<TestProfile, ProcessBuilder> testBuilder) {
+        this.artifactsPath = artifactsPath;
+        this.testPath = testPath;
+        this.logFile = logFile;
+        this.configFile = configFile;
+        this.settingsFile = settingsFile;
+        this.testBuilder = testBuilder;
+    }
+
+    public synchronized void test(TestProfile testProfile, byte[] testConfig) {
+        if (status == Status.RUNNING)
+            throw new IllegalArgumentException("Tests are already running; should not receive this request now.");
+
+        log.clear();
+        status = Status.RUNNING;
+
+        new Thread(() -> runTests(testProfile, testConfig)).start();
+    }
+
+    public Collection<LogRecord> getLog(long after) {
+        return log.tailMap(after + 1).values();
+    }
+
+    public synchronized Status getStatus() {
+        return status;
+    }
+
+    private void runTests(TestProfile testProfile, byte[] testConfig) {
+        ProcessBuilder builder = testBuilder.apply(testProfile);
+        {
+            LogRecord record = new LogRecord(Level.INFO,
+                                             String.format("Starting %s. Artifacts directory: %s Config file: %s\nCommand to run: %s",
+                                                           testProfile.name(), artifactsPath, configFile, String.join(" ", builder.command())));
+            log.put(record.getSequenceNumber(), record);
+            logger.log(record);
+        }
+
+        boolean success;
+        // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure.
+        try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile()))));
+             ByteArrayOutputStream logBuffer = new ByteArrayOutputStream();
+             PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){
+            writeTestApplicationPom(testProfile);
+            Files.write(configFile, testConfig);
+            Files.write(settingsFile, settingsXml.getBytes());
+
+            Process mavenProcess = builder.start();
+            BufferedReader in = new BufferedReader(new InputStreamReader(mavenProcess.getInputStream()));
+            in.lines().forEach(line -> {
+                fileStream.println(line);
+                logFormatter.print(line);
+                LogRecord record = new LogRecord(HTML, logBuffer.toString());
+                log.put(record.getSequenceNumber(), record);
+                logBuffer.reset();
+            });
+            success = mavenProcess.waitFor() == 0;
+        }
+        catch (Exception exception) {
+            LogRecord record = new LogRecord(ERROR, "Failed to execute maven command: " + String.join(" ", builder.command()));
+            record.setThrown(exception);
+            logger.log(record);
+            log.put(record.getSequenceNumber(), record);
+            try (PrintStream file = new PrintStream(new FileOutputStream(logFile.toFile(), true))) {
+                file.println(record.getMessage());
+                exception.printStackTrace(file);
+            }
+            catch (IOException ignored) { }
+            status = Status.ERROR;
+            return;
+        }
+        status = success ? Status.SUCCESS : Status.FAILURE;
+    }
+
+    private void writeTestApplicationPom(TestProfile testProfile) throws IOException {
+        List<Path> files = listFiles(artifactsPath);
+        Path testJar = files.stream().filter(file -> file.toString().endsWith("tests.jar")).findFirst()
+                       .orElseThrow(() -> new IllegalStateException("No file ending with 'tests.jar' found under '" + artifactsPath + "'!"));
+        String pomXml = PomXmlGenerator.generatePomXml(testProfile, files, testJar);
+        testPath.toFile().mkdirs();
+        Files.write(testPath.resolve("pom.xml"), pomXml.getBytes());
+    }
+
+    private static List<Path> listFiles(Path directory) {
+        try (Stream<Path> element = Files.walk(directory)) {
+            return element
+                    .filter(Files::isRegularFile)
+                    .filter(path -> path.toString().endsWith(".jar"))
+                    .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to list files under " + directory, e);
+        }
+    }
+
+
+    public enum Status {
+        NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS
+    }
+
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
new file mode 100644
index 00000000000..d3393ce8dbe
--- /dev/null
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
@@ -0,0 +1,166 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.log.LogLevel;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static com.yahoo.jdisc.Response.Status;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ */
+public class TestRunnerHandler extends LoggingRequestHandler {
+
+    private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json";
+
+    private final TestRunner testRunner;
+
+    @Inject
+    public TestRunnerHandler(Executor executor, AccessLog accessLog, TestRunner testRunner) {
+        super(executor, accessLog);
+        this.testRunner = testRunner;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        try {
+            switch (request.getMethod()) {
+                case GET: return handleGET(request);
+                case POST: return handlePOST(request);
+
+                default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported");
+            }
+        } catch (IllegalArgumentException e) {
+            return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e));
+        } catch (Exception e) {
+            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+            return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e));
+        }
+    }
+
+    private HttpResponse handleGET(HttpRequest request) {
+        String path = request.getUri().getPath();
+        if (path.equals("/tester/v1/log")) {
+            return new SlimeJsonResponse(toSlime(testRunner.getLog(request.hasProperty("after")
+                                                                           ? Long.parseLong(request.getProperty("after"))
+                                                                           : -1)));
+        } else if (path.equals("/tester/v1/status")) {
+            log.info("Responding with status " + testRunner.getStatus());
+            return new Response(testRunner.getStatus().name());
+        }
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+    }
+
+    private HttpResponse handlePOST(HttpRequest request) throws IOException, InterruptedException {
+        final String path = request.getUri().getPath();
+        if (path.startsWith("/tester/v1/run/")) {
+            String type = lastElement(path);
+            TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST");
+            byte[] config = IOUtils.readBytes(request.getData(), 1 << 16);
+            testRunner.test(testProfile, config);
+            log.info("Started tests of type " + type + " and status is " + testRunner.getStatus());
+            return new Response("Successfully started " + type + " tests");
+        }
+        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
+    }
+
+    private static String lastElement(String path) {
+        if (path.endsWith("/"))
+            path = path.substring(0, path.length()-1);
+        int lastSlash = path.lastIndexOf("/");
+        if (lastSlash < 0) return path;
+        return path.substring(lastSlash + 1, path.length());
+    }
+
+    static Slime toSlime(Collection<LogRecord> log) {
+        Slime root = new Slime();
+        Cursor recordArray = root.setArray();
+        log.forEach(record -> {
+            Cursor recordObject = recordArray.addObject();
+            recordObject.setLong("id", record.getSequenceNumber());
+            recordObject.setLong("at", record.getMillis());
+            recordObject.setString("type", typeOf(record.getLevel()));
+            String message = record.getMessage();
+            if (record.getThrown() != null) {
+                ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+                record.getThrown().printStackTrace(new PrintStream(buffer));
+                message += "\n" + buffer;
+            }
+            recordObject.setString("message", message);
+        });
+        return root;
+    }
+
+    public static String typeOf(Level level) {
+        return    level.getName().equals("html") ? "html"
+                : level.intValue() < LogLevel.INFO.intValue() ? "debug"
+                : level.intValue() < LogLevel.WARNING.intValue() ? "info"
+                : level.intValue() < LogLevel.ERROR.intValue() ? "warning"
+                : "error";
+    }
+
+    private class SlimeJsonResponse extends HttpResponse {
+        private final Slime slime;
+
+        private SlimeJsonResponse(Slime slime) {
+            super(200);
+            this.slime = slime;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            new JsonFormat(true).encode(outputStream, slime);
+        }
+
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
+    }
+
+    private static class Response extends HttpResponse {
+        private static final ObjectMapper objectMapper = new ObjectMapper();
+        private final String message;
+
+        private Response(String response) {
+            this(200, response);
+        }
+
+        private Response(int statusCode, String message) {
+            super(statusCode);
+            this.message = message;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            ObjectNode objectNode = objectMapper.createObjectNode();
+            objectNode.put("message", message);
+            objectMapper.writeValue(outputStream, objectNode);
+        }
+
+        @Override
+        public String getContentType() {
+            return CONTENT_TYPE_APPLICATION_JSON;
+        }
+    }
+}
diff --git a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def
new file mode 100644
index 00000000000..a2d0eacd9be
--- /dev/null
+++ b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def
@@ -0,0 +1,4 @@
+package=com.yahoo.vespa.hosted.testrunner
+
+artifactsPath       path
+surefireMemoryMb    int
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
new file mode 100644
index 00000000000..dce02922c63
--- /dev/null
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
@@ -0,0 +1,33 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author valerijf
+ */
+public class PomXmlGeneratorTest {
+
+    @Test
+    public void write_system_tests_pom_xml() throws IOException {
+        List<Path> artifacts = Arrays.asList(
+                Paths.get("components/my-comp.jar"),
+                Paths.get("main.jar"));
+
+        String actual = PomXmlGenerator.generatePomXml(TestProfile.SYSTEM_TEST, artifacts, artifacts.get(1));
+        assertFile("/pom.xml_system_tests", actual);
+    }
+
+    private void assertFile(String resourceFile, String actual) throws IOException {
+        String expected = IOUtils.toString(this.getClass().getResourceAsStream(resourceFile));
+        assertEquals(resourceFile, expected, actual);
+    }
+}
\ No newline at end of file
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
new file mode 100644
index 00000000000..a91b1308080
--- /dev/null
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
@@ -0,0 +1,37 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import com.yahoo.vespa.config.SlimeUtils;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jvenstad
+ */
+public class TestRunnerHandlerTest {
+
+    @Test
+    public void logSerialization() throws IOException {
+        LogRecord record = new LogRecord(Level.INFO, "Hello.");
+        record.setSequenceNumber(1);
+        record.setInstant(Instant.ofEpochMilli(2));
+        Exception exception = new RuntimeException();
+        record.setThrown(exception);
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        exception.printStackTrace(new PrintStream(buffer));
+        String trace = buffer.toString()
+                             .replaceAll("\n", "\\\\n")
+                             .replaceAll("\t", "\\\\t");
+        assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]",
+                     new String(SlimeUtils.toJsonBytes(TestRunnerHandler.toSlime(Collections.singletonList(record)))));
+    }
+
+}
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
new file mode 100644
index 00000000000..49c95fa4b6f
--- /dev/null
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
@@ -0,0 +1,127 @@
+package com.yahoo.vespa.hosted.testrunner;
+
+import org.fusesource.jansi.Ansi;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.logging.LogRecord;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Unit tests relying on a UNIX shell >_<
+ *
+ * @author jvenstad
+ */
+public class TestRunnerTest {
+
+    @Rule
+    public TemporaryFolder tmp = new TemporaryFolder();
+
+    private Path artifactsPath;
+    private Path testPath;
+    private Path logFile;
+    private Path configFile;
+    private Path settingsFile;
+
+    @Before
+    public void setup() throws IOException {
+        artifactsPath = tmp.newFolder("artifacts").toPath();
+        Files.createFile(artifactsPath.resolve("my-tests.jar"));
+        Files.createFile(artifactsPath.resolve("my-fat-test.jar"));
+        testPath = tmp.newFolder("testData").toPath();
+        logFile = tmp.newFile("maven.log").toPath();
+        configFile = tmp.newFile("testConfig.json").toPath();
+        settingsFile = tmp.newFile("settings.xml").toPath();
+    }
+
+    @Test
+    public void ansiCodesAreConvertedToHtml() throws InterruptedException {
+        TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
+                                           __ -> new ProcessBuilder("echo", Ansi.ansi().fg(Ansi.Color.RED).a("Hello!").reset().toString()));
+        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+        Iterator<LogRecord> log = runner.getLog(-1).iterator();
+        log.next();
+        LogRecord record = log.next();
+        assertEquals("<span style=\"color: red;\">Hello!</span>", record.getMessage());
+        assertEquals(0, runner.getLog(record.getSequenceNumber()).size());
+        assertEquals(TestRunner.Status.SUCCESS, runner.getStatus());
+    }
+
+    @Test
+    public void errorLeadsToError() throws InterruptedException {
+        TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
+                                           __ -> new ProcessBuilder("This is a command that doesn't exist, for sure!"));
+        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+        Iterator<LogRecord> log = runner.getLog(-1).iterator();
+        log.next();
+        LogRecord record = log.next();
+        assertEquals("Failed to execute maven command: This is a command that doesn't exist, for sure!", record.getMessage());
+        assertNotNull(record.getThrown());
+        assertEquals(TestRunner.Status.ERROR, runner.getStatus());
+    }
+
+    @Test
+    public void failureLeadsToFailure() throws InterruptedException {
+        TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
+                                           __ -> new ProcessBuilder("false"));
+        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+        assertEquals(1, runner.getLog(-1).size());
+        assertEquals(TestRunner.Status.FAILURE, runner.getStatus());
+    }
+
+    @Test
+    public void filesAreGenerated() throws InterruptedException, IOException {
+        TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
+                                           __ -> new ProcessBuilder("echo", "Hello!"));
+        runner.test(TestProfile.SYSTEM_TEST, "config".getBytes());
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+        assertEquals("config", new String(Files.readAllBytes(configFile)));
+        assertTrue(Files.exists(testPath.resolve("pom.xml")));
+        assertTrue(Files.exists(settingsFile));
+        assertEquals("Hello!\n", new String(Files.readAllBytes(logFile)));
+    }
+
+    @Test
+    public void runnerCanBeReused() throws InterruptedException, IOException {
+        TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
+                                           __ -> new ProcessBuilder("sleep", "0.1"));
+        runner.test(TestProfile.SYSTEM_TEST, "config".getBytes());
+        assertEquals(TestRunner.Status.RUNNING, runner.getStatus());
+
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+        assertEquals(1, runner.getLog(-1).size());
+        assertEquals(TestRunner.Status.SUCCESS, runner.getStatus());
+
+        runner.test(TestProfile.STAGING_TEST, "newConfig".getBytes());
+        while (runner.getStatus() == TestRunner.Status.RUNNING) {
+            Thread.sleep(10);
+        }
+
+        assertEquals("newConfig", new String(Files.readAllBytes(configFile)));
+        assertEquals(1, runner.getLog(-1).size());
+    }
+
+}
diff --git a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests
new file mode 100644
index 00000000000..4f7565e7449
--- /dev/null
+++ b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests
@@ -0,0 +1,72 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0                              http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>tester-application</artifactId>
+    <version>1.0.0</version>
+
+    <properties>
+        <maven_version>4.12</maven_version>
+        <surefire_version>2.22.0</surefire_version>
+        <my-comp.jar.path>components/my-comp.jar</my-comp.jar.path>
+        <main.jar.path>main.jar</main.jar.path>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${maven_version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa.testrunner.test</groupId>
+            <artifactId>my-comp.jar</artifactId>
+            <scope>system</scope>
+            <type>test-jar</type>
+            <version>test</version>
+            <systemPath>${my-comp.jar.path}</systemPath>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa.testrunner.test</groupId>
+            <artifactId>main.jar</artifactId>
+            <scope>system</scope>
+            <type>test-jar</type>
+            <version>test</version>
+            <systemPath>${main.jar.path}</systemPath>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire_version}</version>
+                <configuration>
+                    <dependenciesToScan>
+                        <dependency>com.yahoo.vespa.testrunner.test:main.jar</dependency>
+                    </dependenciesToScan>
+                    <groups>com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</groups>
+                    <excludedGroups>com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class</excludedGroups>
+                    <excludes>
+                        <exclude>com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</exclude>
+                    </excludes>
+                    <reportsDirectory>${env.TEST_DIR}</reportsDirectory>
+                    <redirectTestOutputToFile>false</redirectTestOutputToFile>
+                    <environmentVariables>
+                        <LD_LIBRARY_PATH>/home/y/lib64</LD_LIBRARY_PATH>
+                    </environmentVariables>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-report-plugin</artifactId>
+                <version>${surefire_version}</version>
+                <configuration>
+                    <reportsDirectory>${env.TEST_DIR}</reportsDirectory>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>