Set up VespaCliTestRunner as well when using new test framework

This should be OK to merge, as the runner will only be enabled if a "tests/" direcctory is present in the artifacts directoy, which should not currently be the case for anything else than our experiments

5 files changed, 174 insertions, 2 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 8563375ab5c..4fcd6b10efa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -929,6 +929,13 @@ public class InternalStepRunner implements StepRunner {
                 "                <artifactsPath>artifacts</artifactsPath>\n" +
                 "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
                 "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+                "            <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+                "            </config>\n" +
                 "        </component>\n";
 
         String servicesXml =
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
index 01a7afb3bed..634137e3fb6 100644
--- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
@@ -24,6 +24,13 @@
             </config>
         </component>
 
+        <component id="com.yahoo.vespa.testrunner.VespaCliTestRunner" bundle="vespa-osgi-testrunner">
+            <config name="com.yahoo.vespa.testrunner.vespa-cli-test-runner">
+                <artifactsPath>artifacts</artifactsPath>
+                <useAthenzCredentials>true</useAthenzCredentials>
+            </config>
+        </component>
+
         <nodes count="1" allocated-memory="17%">
             <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/>
         </nodes>
diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml
index 845d0d31af4..6ec70b08d39 100644
--- a/vespa-osgi-testrunner/pom.xml
+++ b/vespa-osgi-testrunner/pom.xml
@@ -59,6 +59,12 @@
         </dependency>
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
+            <artifactId>config-provisioning</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
             <artifactId>testutil</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
new file mode 100644
index 00000000000..d8ff6334382
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
@@ -0,0 +1,148 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import ai.vespa.hosted.api.TestConfig;
+import com.google.inject.Inject;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import java.util.stream.Stream;
+
+import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * @author jonmv
+ */
+public class VespaCliTestRunner implements TestRunner {
+
+    private static final Logger logger = Logger.getLogger(VespaCliTestRunner.class.getName());
+
+    private final SortedMap<Long, LogRecord> log = new ConcurrentSkipListMap<>();
+    private final Path artifactsPath;
+    private AtomicReference<Status> status = new AtomicReference<>(Status.NOT_STARTED);
+
+    @Inject
+    public VespaCliTestRunner(VespaCliTestRunnerConfig config) {
+        this(config.artifactsPath());
+    }
+
+    VespaCliTestRunner(Path artifactsPath) {
+        this.artifactsPath = artifactsPath;
+    }
+
+    @Override
+    public Collection<LogRecord> getLog(long after) {
+        return log.tailMap(after + 1).values();
+    }
+
+    @Override
+    public Status getStatus() {
+        return status.get();
+    }
+
+    @Override
+    public CompletableFuture<?> test(Suite suite, byte[] config) {
+        if (status.getAndSet(RUNNING) == RUNNING)
+            throw new IllegalStateException("Tests already running, not supposed to be started now");
+
+        return CompletableFuture.runAsync(() -> runTests(suite, config));
+    }
+
+    @Override
+    public boolean isSupported() {
+        return getChildDirectory(artifactsPath, "tests").isPresent();
+    }
+
+    void runTests(Suite suite, byte[] config) {
+        Process process = null;
+        try {
+            process = testRunProcessBuilder(suite, toEndpointsConfig(config)).start();
+            BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
+            in.lines().forEach(line -> {
+                if (line.length() > 1 << 13)
+                    line = line.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+
+                log(Level.INFO, line, null);
+            });
+            status.set(process.waitFor() == 0 ? SUCCESS : process.exitValue() == 3 ? FAILURE : ERROR);
+        }
+        catch (Exception e) {
+            if (process != null)
+                process.destroyForcibly();
+
+            log(Level.SEVERE, "Failed running tests", e);
+            status.set(ERROR);
+        }
+    }
+
+    private ProcessBuilder testRunProcessBuilder(Suite suite, String endpointsConfig) {
+        Path suitePath = getChildDirectory(artifactsPath, "tests")
+                .flatMap(testsPath -> getChildDirectory(testsPath, toSuiteDirectoryName(suite)))
+                .orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'"));
+
+        ProcessBuilder builder = new ProcessBuilder("vespa", "test", "--endpoints", endpointsConfig);
+        builder.redirectErrorStream(true);
+        builder.directory(suitePath.toFile());
+        return builder;
+    }
+
+    private static String toSuiteDirectoryName(Suite suite) {
+        switch (suite) {
+            case SYSTEM_TEST: return "system-test";
+            case STAGING_SETUP_TEST: return "staging-setup";
+            case STAGING_TEST: return "staging-test";
+            default: throw new IllegalArgumentException("Unsupported test suite '" + suite + "'");
+        }
+    }
+
+    private void log(Level level, String message, Throwable thrown) {
+        LogRecord record = new LogRecord(level, message);
+        record.setThrown(thrown);
+        logger.log(record);
+        log.put(record.getSequenceNumber(), record);
+    }
+
+    private static Optional<Path> getChildDirectory(Path parent, String name) {
+        try (Stream<Path> children = Files.list(parent)) {
+            return children.filter(Files::isDirectory)
+                           .filter(path -> path.endsWith(name))
+                           .findAny();
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException("Failed to list files under " + parent, e);
+        }
+    }
+
+    static String toEndpointsConfig(byte[] testConfig) throws IOException {
+        TestConfig config = TestConfig.fromJson(testConfig);
+        Cursor root = new Slime().setObject();
+        Cursor endpointsArray = root.setArray("endpoints");
+        config.deployments().get(config.zone()).forEach((cluster, url) -> {
+            Cursor endpointObject = endpointsArray.addObject();
+            endpointObject.setString("cluster", cluster);
+            endpointObject.setString("url", url.toString());
+        });
+        return new String(SlimeUtils.toJsonBytes(root), UTF_8);
+    }
+
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
index 06f7d317b0e..6f12535c317 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -157,6 +157,7 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
         // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure.
         try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile()))));
              ByteArrayOutputStream logBuffer = new ByteArrayOutputStream();
+             PrintStream logPlainFormatter = new PrintStream(new AnsiOutputStream(logBuffer));
              PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){
             writeTestApplicationPom(testProfile);
             Files.write(configFile, testConfig);
@@ -168,8 +169,11 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
                 fileStream.println(line);
                 logFormatter.print(line);
                 String message = logBuffer.toString(UTF_8);
-                if (message.length() > 1 << 13)
-                    message = message.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+                if (message.length() > 1 << 13) {
+                    logBuffer.reset();
+                    logPlainFormatter.print(line); // Avoid HTML since we don't know what we'll strip here.
+                    message = logBuffer.toString(UTF_8).substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+                }
                 LogRecord record = new LogRecord(HTML, message);
                 log.put(record.getSequenceNumber(), record);
                 logBuffer.reset();