diff options
author | HÃ¥kon Hallingstad <hakon@verizonmedia.com> | 2021-11-18 08:48:33 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-18 08:48:33 +0100 |
commit | 8573e4d4973542c8444ec938a5e6c74a41b9f937 (patch) | |
tree | 2bdbe0b871cf48592557583d0eeba75710569cf1 | |
parent | b60f1f985d54a5a7e0c6f86cd889d04eceec0be2 (diff) | |
parent | c7a040a43aa3f043e387b6a235aeda0c484839ce (diff) |
Merge pull request #20074 from vespa-engine/jonmv/vespa-cli-test-runner
Set up VespaCliTestRunner as well when using new test framework
7 files changed, 246 insertions, 2 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 8563375ab5c..4fcd6b10efa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -929,6 +929,13 @@ public class InternalStepRunner implements StepRunner { " <artifactsPath>artifacts</artifactsPath>\n" + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + " </config>\n" + + " </component>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + " </component>\n"; String servicesXml = diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi index 01a7afb3bed..634137e3fb6 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi +++ b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi @@ -24,6 +24,13 @@ </config> </component> + <component id="com.yahoo.vespa.testrunner.VespaCliTestRunner" bundle="vespa-osgi-testrunner"> + <config name="com.yahoo.vespa.testrunner.vespa-cli-test-runner"> + <artifactsPath>artifacts</artifactsPath> + <useAthenzCredentials>true</useAthenzCredentials> + </config> + </component> + <nodes count="1" allocated-memory="17%"> <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> </nodes> diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index 845d0d31af4..6ec70b08d39 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -59,6 +59,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> <scope>test</scope> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java new file mode 100644 index 00000000000..831ec24ac5f --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -0,0 +1,148 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import ai.vespa.hosted.api.TestConfig; +import com.google.inject.Inject; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Optional; +import java.util.SortedMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; +import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; +import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * @author jonmv + */ +public class VespaCliTestRunner implements TestRunner { + + private static final Logger logger = Logger.getLogger(VespaCliTestRunner.class.getName()); + + private final SortedMap<Long, LogRecord> log = new ConcurrentSkipListMap<>(); + private final Path artifactsPath; + private AtomicReference<Status> status = new AtomicReference<>(Status.NOT_STARTED); + + @Inject + public VespaCliTestRunner(VespaCliTestRunnerConfig config) { + this(config.artifactsPath()); + } + + VespaCliTestRunner(Path artifactsPath) { + this.artifactsPath = artifactsPath; + } + + @Override + public Collection<LogRecord> getLog(long after) { + return log.tailMap(after + 1).values(); + } + + @Override + public Status getStatus() { + return status.get(); + } + + @Override + public CompletableFuture<?> test(Suite suite, byte[] config) { + if (status.getAndSet(RUNNING) == RUNNING) + throw new IllegalStateException("Tests already running, not supposed to be started now"); + + return CompletableFuture.runAsync(() -> runTests(suite, config)); + } + + @Override + public boolean isSupported() { + return getChildDirectory(artifactsPath, "tests").isPresent(); + } + + void runTests(Suite suite, byte[] config) { + Process process = null; + try { + process = testRunProcessBuilder(suite, toEndpointsConfig(config)).start(); + BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); + in.lines().forEach(line -> { + if (line.length() > 1 << 13) + line = line.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)"; + + log(Level.INFO, line, null); + }); + status.set(process.waitFor() == 0 ? SUCCESS : process.waitFor() == 3 ? FAILURE : ERROR); + } + catch (Exception e) { + if (process != null) + process.destroyForcibly(); + + log(Level.SEVERE, "Failed running tests", e); + status.set(ERROR); + } + } + + ProcessBuilder testRunProcessBuilder(Suite suite, String endpointsConfig) { + Path suitePath = getChildDirectory(artifactsPath, "tests") + .flatMap(testsPath -> getChildDirectory(testsPath, toSuiteDirectoryName(suite))) + .orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'")); + + ProcessBuilder builder = new ProcessBuilder("vespa", "test", "--endpoints", endpointsConfig); + builder.redirectErrorStream(true); + builder.directory(suitePath.toFile()); + return builder; + } + + private static String toSuiteDirectoryName(Suite suite) { + switch (suite) { + case SYSTEM_TEST: return "system-test"; + case STAGING_SETUP_TEST: return "staging-setup"; + case STAGING_TEST: return "staging-test"; + default: throw new IllegalArgumentException("Unsupported test suite '" + suite + "'"); + } + } + + private void log(Level level, String message, Throwable thrown) { + LogRecord record = new LogRecord(level, message); + record.setThrown(thrown); + logger.log(record); + log.put(record.getSequenceNumber(), record); + } + + private static Optional<Path> getChildDirectory(Path parent, String name) { + try (Stream<Path> children = Files.list(parent)) { + return children.filter(Files::isDirectory) + .filter(path -> path.endsWith(name)) + .findAny(); + } + catch (IOException e) { + throw new UncheckedIOException("Failed to list files under " + parent, e); + } + } + + static String toEndpointsConfig(byte[] testConfig) throws IOException { + TestConfig config = TestConfig.fromJson(testConfig); + Cursor root = new Slime().setObject(); + Cursor endpointsArray = root.setArray("endpoints"); + config.deployments().get(config.zone()).forEach((cluster, url) -> { + Cursor endpointObject = endpointsArray.addObject(); + endpointObject.setString("cluster", cluster); + endpointObject.setString("url", url.toString()); + }); + return new String(SlimeUtils.toJsonBytes(root), UTF_8); + } + +} diff --git a/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def b/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def new file mode 100644 index 00000000000..7671096477e --- /dev/null +++ b/vespa-osgi-testrunner/src/main/resources/configdefinitions/vespa-cli-test-runner.def @@ -0,0 +1,5 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=com.yahoo.vespa.testrunner + +artifactsPath path +useAthenzCredentials bool default=false
\ No newline at end of file diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java new file mode 100644 index 00000000000..68d44a386f8 --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author jonmv + */ +class VespaCliTestRunnerTest { + + @Test + void testEndpointsConfig() throws IOException { + byte[] testConfig = ("{\n" + + " \"application\": \"t:a:i\",\n" + + " \"zone\": \"dev.aws-us-east-1c\",\n" + + " \"system\": \"publiccd\",\n" + + " \"isCI\": true,\n" + + " \"zoneEndpoints\": {\n" + + " \"dev.aws-us-east-1c\": {\n" + + " \"default\": \"https://dev.endpoint:443/\"\n" + + " },\n" + + " \"prod.aws-us-east-1a\": {\n" + + " \"default\": \"https://prod.endpoint:443/\"\n" + + " }\n" + + " },\n" + + " \"clusters\": {\n" + + " \"prod.aws-us-east-1c\": [\n" + + " \"documents\"\n" + + " ]\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8); + + assertEquals("{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}", + VespaCliTestRunner.toEndpointsConfig(testConfig)); + } + + @Test + void testSuitePathDiscovery() throws IOException { + Path temp = Files.createTempDirectory("vespa-cli-test-runner-test-"); + temp.toFile().deleteOnExit(); + VespaCliTestRunner runner = new VespaCliTestRunner(temp); + assertFalse(runner.isSupported()); + + Path tests = Files.createDirectory(temp.resolve("tests")); + assertTrue(runner.isSupported()); + IllegalStateException expected = assertThrows(IllegalStateException.class, + () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "")); + assertEquals("No tests found, for suite 'SYSTEM_TEST'", expected.getMessage()); + + Path systemTests = Files.createDirectory(tests.resolve("system-test")); + ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "config"); + assertEquals(systemTests.toFile(), builder.directory()); + assertEquals(List.of("vespa", "test", "--endpoints", "config"), builder.command()); + } + +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index 06f7d317b0e..6f12535c317 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -157,6 +157,7 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner { // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure. try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile())))); ByteArrayOutputStream logBuffer = new ByteArrayOutputStream(); + PrintStream logPlainFormatter = new PrintStream(new AnsiOutputStream(logBuffer)); PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){ writeTestApplicationPom(testProfile); Files.write(configFile, testConfig); @@ -168,8 +169,11 @@ public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner { fileStream.println(line); logFormatter.print(line); String message = logBuffer.toString(UTF_8); - if (message.length() > 1 << 13) - message = message.substring(0, 1 << 13) + " ... (this log entry was truncated due to size)"; + if (message.length() > 1 << 13) { + logBuffer.reset(); + logPlainFormatter.print(line); // Avoid HTML since we don't know what we'll strip here. + message = logBuffer.toString(UTF_8).substring(0, 1 << 13) + " ... (this log entry was truncated due to size)"; + } LogRecord record = new LogRecord(HTML, message); log.put(record.getSequenceNumber(), record); logBuffer.reset(); |