Merge pull request #20074 from vespa-engine/jonmv/vespa-cli-test-runner

Set up VespaCliTestRunner as well when using new test framework

7 files changed, 246 insertions, 2 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 8563375ab5c..4fcd6b10efa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -929,6 +929,13 @@ public class InternalStepRunner implements StepRunner {
                 "                <artifactsPath>artifacts</artifactsPath>\n" +
                 "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
                 "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+                "            <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "                <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+                "            </config>\n" +
                 "        </component>\n";
 
         String servicesXml =
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
index 01a7afb3bed..634137e3fb6 100644
--- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
@@ -24,6 +24,13 @@
             </config>
         </component>
 
+        <component id="com.yahoo.vespa.testrunner.VespaCliTestRunner" bundle="vespa-osgi-testrunner">
+            <config name="com.yahoo.vespa.testrunner.vespa-cli-test-runner">
+                <artifactsPath>artifacts</artifactsPath>
+                <useAthenzCredentials>true</useAthenzCredentials>
+            </config>
+        </component>
+
         <nodes count="1" allocated-memory="17%">
             <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/>
         </nodes>
diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml
index 845d0d31af4..6ec70b08d39 100644
--- a/vespa-osgi-testrunner/pom.xml
+++ b/vespa-osgi-testrunner/pom.xml
@@ -59,6 +59,12 @@
         </dependency>
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
+            <artifactId>config-provisioning</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
             <artifactId>testutil</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
new file mode 100644
index 00000000000..831ec24ac5f
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
@@ -0,0 +1,148 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import ai.vespa.hosted.api.TestConfig;
+import com.google.inject.Inject;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import java.util.stream.Stream;
+
+import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING;
+import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * @author jonmv
+ */
+public class VespaCliTestRunner implements TestRunner {
+
+    private static final Logger logger = Logger.getLogger(VespaCliTestRunner.class.getName());
+
+    private final SortedMap<Long, LogRecord> log = new ConcurrentSkipListMap<>();
+    private final Path artifactsPath;
+    private AtomicReference<Status> status = new AtomicReference<>(Status.NOT_STARTED);
+
+    @Inject
+    public VespaCliTestRunner(VespaCliTestRunnerConfig config) {
+        this(config.artifactsPath());
+    }
+
+    VespaCliTestRunner(Path artifactsPath) {
+        this.artifactsPath = artifactsPath;
+    }
+
+    @Override
+    public Collection<LogRecord> getLog(long after) {
+        return log.tailMap(after + 1).values();
+    }
+
+    @Override
+    public Status getStatus() {
+        return status.get();
+    }
+
+    @Override
+    public CompletableFuture<?> test(Suite suite, byte[] config) {
+        if (status.getAndSet(RUNNING) == RUNNING)
+            throw new IllegalStateException("Tests already running, not supposed to be started now");
+
+        return CompletableFuture.runAsync(() -> runTests(suite, config));
+    }
+
+    @Override
+    public boolean isSupported() {
+        return getChildDirectory(artifactsPath, "tests").isPresent();
+    }
+
+    void runTests(Suite suite, byte[] config) {
+        Process process = null;
+        try {
+            process = testRunProcessBuilder(suite, toEndpointsConfig(config)).start();
+            BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
+            in.lines().forEach(line -> {
+                if (line.length() > 1 << 13)
+                    line = line.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+
+                log(Level.INFO, line, null);
+            });
+            status.set(process.waitFor() == 0 ? SUCCESS : process.waitFor() == 3 ? FAILURE : ERROR);
+        }
+        catch (Exception e) {
+            if (process != null)
+                process.destroyForcibly();
+
+            log(Level.SEVERE, "Failed running tests", e);
+            status.set(ERROR);
+        }
+    }
+
+    ProcessBuilder testRunProcessBuilder(Suite suite, String endpointsConfig) {
+        Path suitePath = getChildDirectory(artifactsPath, "tests")
+                .flatMap(testsPath -> getChildDirectory(testsPath, toSuiteDirectoryName(suite)))
+                .orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'"));
+
+        ProcessBuilder builder = new ProcessBuilder("vespa", "test", "--endpoints", endpointsConfig);
+        builder.redirectErrorStream(true);
+        builder.directory(suitePath.toFile());
+        return builder;
+    }
+
+    private static String toSuiteDirectoryName(Suite suite) {
+        switch (suite) {
+            case SYSTEM_TEST: return "system-test";
+            case STAGING_SETUP_TEST: return "staging-setup";
+            case STAGING_TEST: return "staging-test";
+            default: throw new IllegalArgumentException("Unsupported test suite '" + suite + "'");
+        }
+    }
+
+    private void log(Level level, String message, Throwable thrown) {
+        LogRecord record = new LogRecord(level, message);
+        record.setThrown(thrown);
+        logger.log(record);
+        log.put(record.getSequenceNumber(), record);
+    }
+
+    private static Optional<Path> getChildDirectory(Path parent, String name) {
+        try (Stream<Path> children = Files.list(parent)) {
+            return children.filter(Files::isDirectory)
+                           .filter(path -> path.endsWith(name))
+                           .findAny();
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException("Failed to list files under " + parent, e);
+        }
+    }
+
+    static String toEndpointsConfig(byte[] testConfig) throws IOException {
+        TestConfig config = TestConfig.fromJson(testConfig);
+        Cursor root = new Slime().setObject();
+        Cursor endpointsArray = root.setArray("endpoints");
+        config.deployments().get(config.zone()).forEach((cluster, url) -> {
+            Cursor endpointObject = endpointsArray.addObject();
+            endpointObject.setString("cluster", cluster);
+            endpointObject.setString("url", url.toString());
+        });
+        return new String(SlimeUtils.toJsonBytes(root), UTF_8);
+    }
+
+}
diff --git a/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def b/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def
new file mode 100644
index 00000000000..7671096477e
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def
@@ -0,0 +1,5 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=com.yahoo.vespa.testrunner
+
+artifactsPath          path
+useAthenzCredentials   bool default=false
\ No newline at end of file
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java
new file mode 100644
index 00000000000..68d44a386f8
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java
@@ -0,0 +1,67 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author jonmv
+ */
+class VespaCliTestRunnerTest {
+
+    @Test
+    void testEndpointsConfig() throws IOException {
+        byte[] testConfig = ("{\n" +
+                             "  \"application\": \"t:a:i\",\n" +
+                             "  \"zone\": \"dev.aws-us-east-1c\",\n" +
+                             "  \"system\": \"publiccd\",\n" +
+                             "  \"isCI\": true,\n" +
+                             "  \"zoneEndpoints\": {\n" +
+                             "    \"dev.aws-us-east-1c\": {\n" +
+                             "      \"default\": \"https://dev.endpoint:443/\"\n" +
+                             "    },\n" +
+                             "    \"prod.aws-us-east-1a\": {\n" +
+                             "      \"default\": \"https://prod.endpoint:443/\"\n" +
+                             "    }\n" +
+                             "  },\n" +
+                             "  \"clusters\": {\n" +
+                             "    \"prod.aws-us-east-1c\": [\n" +
+                             "      \"documents\"\n" +
+                             "    ]\n" +
+                             "  }\n" +
+                             "}\n").getBytes(StandardCharsets.UTF_8);
+
+        assertEquals("{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}",
+                     VespaCliTestRunner.toEndpointsConfig(testConfig));
+    }
+
+    @Test
+    void testSuitePathDiscovery() throws IOException {
+        Path temp = Files.createTempDirectory("vespa-cli-test-runner-test-");
+        temp.toFile().deleteOnExit();
+        VespaCliTestRunner runner = new VespaCliTestRunner(temp);
+        assertFalse(runner.isSupported());
+
+        Path tests = Files.createDirectory(temp.resolve("tests"));
+        assertTrue(runner.isSupported());
+        IllegalStateException expected = assertThrows(IllegalStateException.class,
+                                                      () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, ""));
+        assertEquals("No tests found, for suite 'SYSTEM_TEST'", expected.getMessage());
+
+        Path systemTests = Files.createDirectory(tests.resolve("system-test"));
+        ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "config");
+        assertEquals(systemTests.toFile(), builder.directory());
+        assertEquals(List.of("vespa", "test", "--endpoints", "config"), builder.command());
+    }
+
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
index 06f7d317b0e..6f12535c317 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -157,6 +157,7 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
         // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure.
         try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile()))));
              ByteArrayOutputStream logBuffer = new ByteArrayOutputStream();
+             PrintStream logPlainFormatter = new PrintStream(new AnsiOutputStream(logBuffer));
              PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){
             writeTestApplicationPom(testProfile);
             Files.write(configFile, testConfig);
@@ -168,8 +169,11 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
                 fileStream.println(line);
                 logFormatter.print(line);
                 String message = logBuffer.toString(UTF_8);
-                if (message.length() > 1 << 13)
-                    message = message.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+                if (message.length() > 1 << 13) {
+                    logBuffer.reset();
+                    logPlainFormatter.print(line); // Avoid HTML since we don't know what we'll strip here.
+                    message = logBuffer.toString(UTF_8).substring(0, 1 << 13) + " ... (this log entry was truncated due to size)";
+                }
                 LogRecord record = new LogRecord(HTML, message);
                 log.put(record.getSequenceNumber(), record);
                 logBuffer.reset();