diff options
author | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2019-03-26 13:55:35 +0100 |
---|---|---|
committer | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2019-03-26 13:55:35 +0100 |
commit | ba2b88382a0d32a585a4734c445330b8f1521602 (patch) | |
tree | 89113769ec907ef50a3b5a23655d89a16eeca012 | |
parent | 763d6a31683769f8424d16b36a6dcbac315fe3c4 (diff) |
Move vespa-testrunner-components here
15 files changed, 864 insertions, 1 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 8456b79f124..eba69f6ed02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,6 +110,7 @@ add_subdirectory(vdslib) add_subdirectory(vdstestlib) add_subdirectory(vespa-athenz) add_subdirectory(vespa-http-client) +add_subdirectory(vespa-testrunner-components) add_subdirectory(vespa_feed_perf) add_subdirectory(vespa_jersey2) add_subdirectory(vespabase) @@ -127,9 +127,10 @@ <module>vespaclient-java</module> <module>vespa-athenz</module> <module>vespa-documentgen-plugin</module> - <module>vespa_feed_perf</module> <module>vespa-hadoop</module> <module>vespa-http-client</module> + <module>vespa-testrunner-components</module> + <module>vespa_feed_perf</module> <module>vespa_jersey2</module> <module>vespajlib</module> <module>vespalog</module> diff --git a/vespa-testrunner-components/CMakeLists.txt b/vespa-testrunner-components/CMakeLists.txt new file mode 100644 index 00000000000..fe2cb84b7bb --- /dev/null +++ b/vespa-testrunner-components/CMakeLists.txt @@ -0,0 +1,3 @@ +install_java_artifact(vespa-testrunner-components) +install_fat_java_artifact(vespa-testrunner-components) +install_config_definition(src/main/resources/configdefinitions/test-runner.def test-runner.def) diff --git a/vespa-testrunner-components/OWNERS b/vespa-testrunner-components/OWNERS new file mode 100644 index 00000000000..134acfc20f3 --- /dev/null +++ b/vespa-testrunner-components/OWNERS @@ -0,0 +1 @@ +jvenstad diff --git a/vespa-testrunner-components/README.md b/vespa-testrunner-components/README.md new file mode 100644 index 00000000000..034ad95ac25 --- /dev/null +++ b/vespa-testrunner-components/README.md @@ -0,0 +1,4 @@ +# Vespa-testrunner-components + +Defines handler and component used by the vespa application that is deployed by the controller to +run system/staging/production tests. diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml new file mode 100644 index 00000000000..80d55660bc7 --- /dev/null +++ b/vespa-testrunner-components/pom.xml @@ -0,0 +1,82 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.vespa.hosted</groupId> + <artifactId>vespa-testrunner-components</artifactId> + <packaging>container-plugin</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.fusesource.jansi</groupId> + <artifactId>jansi</artifactId> + <version>1.11</version> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-jar-with-dependencies.jar</file> + <type>jar</type> + <classifier>deploy</classifier> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${project.version}</version> + <extensions>true</extensions> + <configuration> + <useCommonAssemblyIds>true</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java new file mode 100644 index 00000000000..4e89e57f0ca --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java @@ -0,0 +1,108 @@ +package com.yahoo.vespa.hosted.testrunner; + +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Generates a pom.xml file that sets up build profile to test against the provided + * jar artifacts. + * + * @author valerijf + */ +public class PomXmlGenerator { + private static final String PROPERTY_TEMPLATE = + " <%ARTIFACT_ID%.path>%JAR_PATH%</%ARTIFACT_ID%.path>\n"; + private static final String TEST_ARTIFACT_GROUP_ID = "com.yahoo.vespa.testrunner.test"; + private static final String DEPENDENCY_TEMPLATE = + " <dependency>\n" + + " <groupId>" + TEST_ARTIFACT_GROUP_ID + "</groupId>\n" + + " <artifactId>%ARTIFACT_ID%</artifactId>\n" + + " <scope>system</scope>\n" + + " <type>test-jar</type>\n" + + " <version>test</version>\n" + + " <systemPath>${%ARTIFACT_ID%.path}</systemPath>\n" + + " </dependency>\n"; + private static final String DEPENDENCY_TO_SCAN_TEMPLATE = + " <dependency>" + TEST_ARTIFACT_GROUP_ID + ":%ARTIFACT_ID%</dependency>\n"; + private static final String POM_XML_TEMPLATE = + "<?xml version=\"1.0\"?>\n" + + "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n" + + " <modelVersion>4.0.0</modelVersion>\n" + + " <groupId>com.yahoo.vespa</groupId>\n" + + " <artifactId>tester-application</artifactId>\n" + + " <version>1.0.0</version>\n" + + "\n" + + " <properties>\n" + + " <maven_version>4.12</maven_version>\n" + + " <surefire_version>2.22.0</surefire_version>\n" + + "%PROPERTIES%" + + " </properties>\n" + + "\n" + + " <dependencies>\n" + + " <dependency>\n" + + " <groupId>junit</groupId>\n" + + " <artifactId>junit</artifactId>\n" + + " <version>${maven_version}</version>\n" + + " <scope>test</scope>\n" + + " </dependency>\n" + + "%DEPENDENCIES%" + + " </dependencies>\n" + + "\n" + + " <build>\n" + + " <plugins>\n" + + " <plugin>\n" + + " <groupId>org.apache.maven.plugins</groupId>\n" + + " <artifactId>maven-surefire-plugin</artifactId>\n" + + " <version>${surefire_version}</version>\n" + + " <configuration>\n" + + " <dependenciesToScan>\n" + + "%DEPENDENCIES_TO_SCAN%" + + " </dependenciesToScan>\n" + + " <groups>%GROUPS%</groups>\n" + + " <excludedGroups>com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class</excludedGroups>\n" + + " <excludes>\n" + + " <exclude>%GROUPS%</exclude>\n" + + " </excludes>\n" + + " <reportsDirectory>${env.TEST_DIR}</reportsDirectory>\n" + + " <redirectTestOutputToFile>false</redirectTestOutputToFile>\n" + + " <environmentVariables>\n" + + " <LD_LIBRARY_PATH>/home/y/lib64</LD_LIBRARY_PATH>\n" + + " </environmentVariables>\n" + + " </configuration>\n" + + " </plugin>\n" + + " <plugin>\n" + + " <groupId>org.apache.maven.plugins</groupId>\n" + + " <artifactId>maven-surefire-report-plugin</artifactId>\n" + + " <version>${surefire_version}</version>\n" + + " <configuration>\n" + + " <reportsDirectory>${env.TEST_DIR}</reportsDirectory>\n" + + " </configuration>\n" + + " </plugin>\n" + + " </plugins>\n" + + " </build>\n" + + "</project>\n"; + + static String generatePomXml(TestProfile testProfile, List<Path> artifacts, Path testArtifact) { + String properties = artifacts.stream() + .map(path -> PROPERTY_TEMPLATE + .replace("%ARTIFACT_ID%", path.getFileName().toString()) + .replace("%JAR_PATH%", path.toString())) + .collect(Collectors.joining()); + String dependencies = artifacts.stream() + .map(path -> DEPENDENCY_TEMPLATE + .replace("%ARTIFACT_ID%", path.getFileName().toString())) + .collect(Collectors.joining()); + String dependenciesToScan = + DEPENDENCY_TO_SCAN_TEMPLATE + .replace("%ARTIFACT_ID%", testArtifact.getFileName().toString()); + + return POM_XML_TEMPLATE + .replace("%PROPERTIES%", properties) + .replace("%DEPENDENCIES_TO_SCAN%", dependenciesToScan) + .replace("%DEPENDENCIES%", dependencies) + .replace("%GROUPS%", testProfile.group()); + } + + private PomXmlGenerator() {} +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java new file mode 100644 index 00000000000..b7d3a06f30d --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.hosted.testrunner; + +/** + * @author valerijf + * @author jvenstad + */ +enum TestProfile { + + SYSTEM_TEST("com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true), + STAGING_TEST("com.yahoo.vespa.tenant.cd.StagingTest, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true), + PRODUCTION_TEST("com.yahoo.vespa.tenant.cd.ProductionTest, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false); + + private final String group; + private final boolean failIfNoTests; + + TestProfile(String group, boolean failIfNoTests) { + this.group = group; + this.failIfNoTests = failIfNoTests; + } + + String group() { + return group; + } + + boolean failIfNoTests() { + return failIfNoTests; + } + +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java new file mode 100644 index 00000000000..fb5dccc551d --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -0,0 +1,195 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.google.inject.Inject; +import com.yahoo.vespa.defaults.Defaults; +import org.fusesource.jansi.AnsiOutputStream; +import org.fusesource.jansi.HtmlAnsiOutputStream; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.List; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.log.LogLevel.ERROR; + +/** + * @author valerijf + * @author jvenstad + */ +public class TestRunner { + + private static final Logger logger = Logger.getLogger(TestRunner.class.getName()); + private static final Level HTML = new Level("html", 1) { }; + private static final Path vespaHome = Paths.get(Defaults.getDefaults().vespaHome()); + private static final String settingsXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<settings xmlns=\"http://maven.apache.org/SETTINGS/1.0.0\"\n" + + " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + + " xsi:schemaLocation=\"http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd\">\n" + + " <mirrors>\n" + + " <mirror>\n" + + " <id>maven central</id>\n" + + " <mirrorOf>*</mirrorOf>\n" + // Use this for everything! + " <url>https://repo.maven.apache.org/maven2/</url>\n" + + " </mirror>\n" + + " </mirrors>\n" + + "</settings>"; + + private final Path artifactsPath; + private final Path testPath; + private final Path logFile; + private final Path configFile; + private final Path settingsFile; + private final Function<TestProfile, ProcessBuilder> testBuilder; + private final SortedMap<Long, LogRecord> log = new ConcurrentSkipListMap<>(); + + private volatile Status status = Status.NOT_STARTED; + + @Inject + public TestRunner(TestRunnerConfig config) { + this(config.artifactsPath(), + vespaHome.resolve("tmp/test"), + vespaHome.resolve("logs/vespa/maven.log"), + vespaHome.resolve("tmp/config.json"), + vespaHome.resolve("tmp/settings.xml"), + profile -> { // Anything to make this testable! >_< + String[] command = new String[]{ + "mvn", + "test", + + "--batch-mode", // Run in non-interactive (batch) mode (disables output color) + "--show-version", // Display version information WITHOUT stopping build + "--settings", // Need to override repository settings in ymaven config >_< + vespaHome.resolve("tmp/settings.xml").toString(), + + // Disable maven download progress indication + "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", + "-Dstyle.color=always", // Enable ANSI color codes again + "-DfailIfNoTests=" + profile.failIfNoTests(), + "-Dvespa.test.config=" + vespaHome.resolve("tmp/config.json"), + "-Dvespa.test.credentials.root=" + Defaults.getDefaults().vespaHome() + "/var/vespa/sia", + String.format("-DargLine=-Xms%1$dm -Xmx%1$dm", config.surefireMemoryMb()) + }; + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().merge("MAVEN_OPTS", " -Djansi.force=true", String::concat); + builder.directory(vespaHome.resolve("tmp/test").toFile()); + builder.redirectErrorStream(true); + return builder; + }); + } + + TestRunner(Path artifactsPath, Path testPath, Path logFile, Path configFile, Path settingsFile, Function<TestProfile, ProcessBuilder> testBuilder) { + this.artifactsPath = artifactsPath; + this.testPath = testPath; + this.logFile = logFile; + this.configFile = configFile; + this.settingsFile = settingsFile; + this.testBuilder = testBuilder; + } + + public synchronized void test(TestProfile testProfile, byte[] testConfig) { + if (status == Status.RUNNING) + throw new IllegalArgumentException("Tests are already running; should not receive this request now."); + + log.clear(); + status = Status.RUNNING; + + new Thread(() -> runTests(testProfile, testConfig)).start(); + } + + public Collection<LogRecord> getLog(long after) { + return log.tailMap(after + 1).values(); + } + + public synchronized Status getStatus() { + return status; + } + + private void runTests(TestProfile testProfile, byte[] testConfig) { + ProcessBuilder builder = testBuilder.apply(testProfile); + { + LogRecord record = new LogRecord(Level.INFO, + String.format("Starting %s. Artifacts directory: %s Config file: %s\nCommand to run: %s", + testProfile.name(), artifactsPath, configFile, String.join(" ", builder.command()))); + log.put(record.getSequenceNumber(), record); + logger.log(record); + } + + boolean success; + // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure. + try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile())))); + ByteArrayOutputStream logBuffer = new ByteArrayOutputStream(); + PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){ + writeTestApplicationPom(testProfile); + Files.write(configFile, testConfig); + Files.write(settingsFile, settingsXml.getBytes()); + + Process mavenProcess = builder.start(); + BufferedReader in = new BufferedReader(new InputStreamReader(mavenProcess.getInputStream())); + in.lines().forEach(line -> { + fileStream.println(line); + logFormatter.print(line); + LogRecord record = new LogRecord(HTML, logBuffer.toString()); + log.put(record.getSequenceNumber(), record); + logBuffer.reset(); + }); + success = mavenProcess.waitFor() == 0; + } + catch (Exception exception) { + LogRecord record = new LogRecord(ERROR, "Failed to execute maven command: " + String.join(" ", builder.command())); + record.setThrown(exception); + logger.log(record); + log.put(record.getSequenceNumber(), record); + try (PrintStream file = new PrintStream(new FileOutputStream(logFile.toFile(), true))) { + file.println(record.getMessage()); + exception.printStackTrace(file); + } + catch (IOException ignored) { } + status = Status.ERROR; + return; + } + status = success ? Status.SUCCESS : Status.FAILURE; + } + + private void writeTestApplicationPom(TestProfile testProfile) throws IOException { + List<Path> files = listFiles(artifactsPath); + Path testJar = files.stream().filter(file -> file.toString().endsWith("tests.jar")).findFirst() + .orElseThrow(() -> new IllegalStateException("No file ending with 'tests.jar' found under '" + artifactsPath + "'!")); + String pomXml = PomXmlGenerator.generatePomXml(testProfile, files, testJar); + testPath.toFile().mkdirs(); + Files.write(testPath.resolve("pom.xml"), pomXml.getBytes()); + } + + private static List<Path> listFiles(Path directory) { + try (Stream<Path> element = Files.walk(directory)) { + return element + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jar")) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to list files under " + directory, e); + } + } + + + public enum Status { + NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS + } + +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java new file mode 100644 index 00000000000..d3393ce8dbe --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -0,0 +1,166 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.yolean.Exceptions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Collection; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static com.yahoo.jdisc.Response.Status; + +/** + * @author valerijf + * @author jvenstad + */ +public class TestRunnerHandler extends LoggingRequestHandler { + + private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json"; + + private final TestRunner testRunner; + + @Inject + public TestRunnerHandler(Executor executor, AccessLog accessLog, TestRunner testRunner) { + super(executor, accessLog); + this.testRunner = testRunner; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case POST: return handlePOST(request); + + default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported"); + } + } catch (IllegalArgumentException e) { + return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e)); + } catch (Exception e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.equals("/tester/v1/log")) { + return new SlimeJsonResponse(toSlime(testRunner.getLog(request.hasProperty("after") + ? Long.parseLong(request.getProperty("after")) + : -1))); + } else if (path.equals("/tester/v1/status")) { + log.info("Responding with status " + testRunner.getStatus()); + return new Response(testRunner.getStatus().name()); + } + return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); + } + + private HttpResponse handlePOST(HttpRequest request) throws IOException, InterruptedException { + final String path = request.getUri().getPath(); + if (path.startsWith("/tester/v1/run/")) { + String type = lastElement(path); + TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST"); + byte[] config = IOUtils.readBytes(request.getData(), 1 << 16); + testRunner.test(testProfile, config); + log.info("Started tests of type " + type + " and status is " + testRunner.getStatus()); + return new Response("Successfully started " + type + " tests"); + } + return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); + } + + private static String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash + 1, path.length()); + } + + static Slime toSlime(Collection<LogRecord> log) { + Slime root = new Slime(); + Cursor recordArray = root.setArray(); + log.forEach(record -> { + Cursor recordObject = recordArray.addObject(); + recordObject.setLong("id", record.getSequenceNumber()); + recordObject.setLong("at", record.getMillis()); + recordObject.setString("type", typeOf(record.getLevel())); + String message = record.getMessage(); + if (record.getThrown() != null) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + record.getThrown().printStackTrace(new PrintStream(buffer)); + message += "\n" + buffer; + } + recordObject.setString("message", message); + }); + return root; + } + + public static String typeOf(Level level) { + return level.getName().equals("html") ? "html" + : level.intValue() < LogLevel.INFO.intValue() ? "debug" + : level.intValue() < LogLevel.WARNING.intValue() ? "info" + : level.intValue() < LogLevel.ERROR.intValue() ? "warning" + : "error"; + } + + private class SlimeJsonResponse extends HttpResponse { + private final Slime slime; + + private SlimeJsonResponse(Slime slime) { + super(200); + this.slime = slime; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return CONTENT_TYPE_APPLICATION_JSON; + } + } + + private static class Response extends HttpResponse { + private static final ObjectMapper objectMapper = new ObjectMapper(); + private final String message; + + private Response(String response) { + this(200, response); + } + + private Response(int statusCode, String message) { + super(statusCode); + this.message = message; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.put("message", message); + objectMapper.writeValue(outputStream, objectNode); + } + + @Override + public String getContentType() { + return CONTENT_TYPE_APPLICATION_JSON; + } + } +} diff --git a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def new file mode 100644 index 00000000000..a2d0eacd9be --- /dev/null +++ b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def @@ -0,0 +1,4 @@ +package=com.yahoo.vespa.hosted.testrunner + +artifactsPath path +surefireMemoryMb int diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java new file mode 100644 index 00000000000..dce02922c63 --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.testrunner; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author valerijf + */ +public class PomXmlGeneratorTest { + + @Test + public void write_system_tests_pom_xml() throws IOException { + List<Path> artifacts = Arrays.asList( + Paths.get("components/my-comp.jar"), + Paths.get("main.jar")); + + String actual = PomXmlGenerator.generatePomXml(TestProfile.SYSTEM_TEST, artifacts, artifacts.get(1)); + assertFile("/pom.xml_system_tests", actual); + } + + private void assertFile(String resourceFile, String actual) throws IOException { + String expected = IOUtils.toString(this.getClass().getResourceAsStream(resourceFile)); + assertEquals(resourceFile, expected, actual); + } +}
\ No newline at end of file diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java new file mode 100644 index 00000000000..a91b1308080 --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.yahoo.vespa.config.SlimeUtils; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.time.Instant; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static org.junit.Assert.assertEquals; + +/** + * @author jvenstad + */ +public class TestRunnerHandlerTest { + + @Test + public void logSerialization() throws IOException { + LogRecord record = new LogRecord(Level.INFO, "Hello."); + record.setSequenceNumber(1); + record.setInstant(Instant.ofEpochMilli(2)); + Exception exception = new RuntimeException(); + record.setThrown(exception); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + exception.printStackTrace(new PrintStream(buffer)); + String trace = buffer.toString() + .replaceAll("\n", "\\\\n") + .replaceAll("\t", "\\\\t"); + assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]", + new String(SlimeUtils.toJsonBytes(TestRunnerHandler.toSlime(Collections.singletonList(record))))); + } + +} diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java new file mode 100644 index 00000000000..49c95fa4b6f --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java @@ -0,0 +1,127 @@ +package com.yahoo.vespa.hosted.testrunner; + +import org.fusesource.jansi.Ansi; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.logging.LogRecord; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Unit tests relying on a UNIX shell >_< + * + * @author jvenstad + */ +public class TestRunnerTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private Path artifactsPath; + private Path testPath; + private Path logFile; + private Path configFile; + private Path settingsFile; + + @Before + public void setup() throws IOException { + artifactsPath = tmp.newFolder("artifacts").toPath(); + Files.createFile(artifactsPath.resolve("my-tests.jar")); + Files.createFile(artifactsPath.resolve("my-fat-test.jar")); + testPath = tmp.newFolder("testData").toPath(); + logFile = tmp.newFile("maven.log").toPath(); + configFile = tmp.newFile("testConfig.json").toPath(); + settingsFile = tmp.newFile("settings.xml").toPath(); + } + + @Test + public void ansiCodesAreConvertedToHtml() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("echo", Ansi.ansi().fg(Ansi.Color.RED).a("Hello!").reset().toString())); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + Iterator<LogRecord> log = runner.getLog(-1).iterator(); + log.next(); + LogRecord record = log.next(); + assertEquals("<span style=\"color: red;\">Hello!</span>", record.getMessage()); + assertEquals(0, runner.getLog(record.getSequenceNumber()).size()); + assertEquals(TestRunner.Status.SUCCESS, runner.getStatus()); + } + + @Test + public void errorLeadsToError() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("This is a command that doesn't exist, for sure!")); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + Iterator<LogRecord> log = runner.getLog(-1).iterator(); + log.next(); + LogRecord record = log.next(); + assertEquals("Failed to execute maven command: This is a command that doesn't exist, for sure!", record.getMessage()); + assertNotNull(record.getThrown()); + assertEquals(TestRunner.Status.ERROR, runner.getStatus()); + } + + @Test + public void failureLeadsToFailure() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("false")); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals(1, runner.getLog(-1).size()); + assertEquals(TestRunner.Status.FAILURE, runner.getStatus()); + } + + @Test + public void filesAreGenerated() throws InterruptedException, IOException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("echo", "Hello!")); + runner.test(TestProfile.SYSTEM_TEST, "config".getBytes()); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals("config", new String(Files.readAllBytes(configFile))); + assertTrue(Files.exists(testPath.resolve("pom.xml"))); + assertTrue(Files.exists(settingsFile)); + assertEquals("Hello!\n", new String(Files.readAllBytes(logFile))); + } + + @Test + public void runnerCanBeReused() throws InterruptedException, IOException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("sleep", "0.1")); + runner.test(TestProfile.SYSTEM_TEST, "config".getBytes()); + assertEquals(TestRunner.Status.RUNNING, runner.getStatus()); + + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals(1, runner.getLog(-1).size()); + assertEquals(TestRunner.Status.SUCCESS, runner.getStatus()); + + runner.test(TestProfile.STAGING_TEST, "newConfig".getBytes()); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + + assertEquals("newConfig", new String(Files.readAllBytes(configFile))); + assertEquals(1, runner.getLog(-1).size()); + } + +} diff --git a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests new file mode 100644 index 00000000000..4f7565e7449 --- /dev/null +++ b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests @@ -0,0 +1,72 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tester-application</artifactId> + <version>1.0.0</version> + + <properties> + <maven_version>4.12</maven_version> + <surefire_version>2.22.0</surefire_version> + <my-comp.jar.path>components/my-comp.jar</my-comp.jar.path> + <main.jar.path>main.jar</main.jar.path> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${maven_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa.testrunner.test</groupId> + <artifactId>my-comp.jar</artifactId> + <scope>system</scope> + <type>test-jar</type> + <version>test</version> + <systemPath>${my-comp.jar.path}</systemPath> + </dependency> + <dependency> + <groupId>com.yahoo.vespa.testrunner.test</groupId> + <artifactId>main.jar</artifactId> + <scope>system</scope> + <type>test-jar</type> + <version>test</version> + <systemPath>${main.jar.path}</systemPath> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <dependenciesToScan> + <dependency>com.yahoo.vespa.testrunner.test:main.jar</dependency> + </dependenciesToScan> + <groups>com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</groups> + <excludedGroups>com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class</excludedGroups> + <excludes> + <exclude>com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</exclude> + </excludes> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <environmentVariables> + <LD_LIBRARY_PATH>/home/y/lib64</LD_LIBRARY_PATH> + </environmentVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </build> +</project> |