From ba2b88382a0d32a585a4734c445330b8f1521602 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 26 Mar 2019 13:55:35 +0100 Subject: Move vespa-testrunner-components here --- vespa-testrunner-components/CMakeLists.txt | 3 + vespa-testrunner-components/OWNERS | 1 + vespa-testrunner-components/README.md | 4 + vespa-testrunner-components/pom.xml | 82 +++++++++ .../vespa/hosted/testrunner/PomXmlGenerator.java | 108 ++++++++++++ .../yahoo/vespa/hosted/testrunner/TestProfile.java | 29 +++ .../yahoo/vespa/hosted/testrunner/TestRunner.java | 195 +++++++++++++++++++++ .../vespa/hosted/testrunner/TestRunnerHandler.java | 166 ++++++++++++++++++ .../resources/configdefinitions/test-runner.def | 4 + .../hosted/testrunner/PomXmlGeneratorTest.java | 33 ++++ .../hosted/testrunner/TestRunnerHandlerTest.java | 37 ++++ .../vespa/hosted/testrunner/TestRunnerTest.java | 127 ++++++++++++++ .../src/test/resources/pom.xml_system_tests | 72 ++++++++ 13 files changed, 861 insertions(+) create mode 100644 vespa-testrunner-components/CMakeLists.txt create mode 100644 vespa-testrunner-components/OWNERS create mode 100644 vespa-testrunner-components/README.md create mode 100644 vespa-testrunner-components/pom.xml create mode 100644 vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java create mode 100644 vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java create mode 100644 vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java create mode 100644 vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java create mode 100644 vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def create mode 100644 vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java create mode 100644 vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java create mode 100644 vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java create mode 100644 vespa-testrunner-components/src/test/resources/pom.xml_system_tests (limited to 'vespa-testrunner-components') diff --git a/vespa-testrunner-components/CMakeLists.txt b/vespa-testrunner-components/CMakeLists.txt new file mode 100644 index 00000000000..fe2cb84b7bb --- /dev/null +++ b/vespa-testrunner-components/CMakeLists.txt @@ -0,0 +1,3 @@ +install_java_artifact(vespa-testrunner-components) +install_fat_java_artifact(vespa-testrunner-components) +install_config_definition(src/main/resources/configdefinitions/test-runner.def test-runner.def) diff --git a/vespa-testrunner-components/OWNERS b/vespa-testrunner-components/OWNERS new file mode 100644 index 00000000000..134acfc20f3 --- /dev/null +++ b/vespa-testrunner-components/OWNERS @@ -0,0 +1 @@ +jvenstad diff --git a/vespa-testrunner-components/README.md b/vespa-testrunner-components/README.md new file mode 100644 index 00000000000..034ad95ac25 --- /dev/null +++ b/vespa-testrunner-components/README.md @@ -0,0 +1,4 @@ +# Vespa-testrunner-components + +Defines handler and component used by the vespa application that is deployed by the controller to +run system/staging/production tests. diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml new file mode 100644 index 00000000000..80d55660bc7 --- /dev/null +++ b/vespa-testrunner-components/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + com.yahoo.vespa.hosted + vespa-testrunner-components + container-plugin + + + com.yahoo.vespa + parent + 7-SNAPSHOT + ../parent/pom.xml + + + + + com.yahoo.vespa + container + ${project.version} + provided + + + + org.fusesource.jansi + jansi + 1.11 + + + + junit + junit + test + + + commons-io + commons-io + test + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-artifacts + package + + attach-artifact + + + + + target/${project.artifactId}-jar-with-dependencies.jar + jar + deploy + + + + + + + + com.yahoo.vespa + bundle-plugin + ${project.version} + true + + true + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java new file mode 100644 index 00000000000..4e89e57f0ca --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java @@ -0,0 +1,108 @@ +package com.yahoo.vespa.hosted.testrunner; + +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Generates a pom.xml file that sets up build profile to test against the provided + * jar artifacts. + * + * @author valerijf + */ +public class PomXmlGenerator { + private static final String PROPERTY_TEMPLATE = + " <%ARTIFACT_ID%.path>%JAR_PATH%\n"; + private static final String TEST_ARTIFACT_GROUP_ID = "com.yahoo.vespa.testrunner.test"; + private static final String DEPENDENCY_TEMPLATE = + " \n" + + " " + TEST_ARTIFACT_GROUP_ID + "\n" + + " %ARTIFACT_ID%\n" + + " system\n" + + " test-jar\n" + + " test\n" + + " ${%ARTIFACT_ID%.path}\n" + + " \n"; + private static final String DEPENDENCY_TO_SCAN_TEMPLATE = + " " + TEST_ARTIFACT_GROUP_ID + ":%ARTIFACT_ID%\n"; + private static final String POM_XML_TEMPLATE = + "\n" + + "\n" + + " 4.0.0\n" + + " com.yahoo.vespa\n" + + " tester-application\n" + + " 1.0.0\n" + + "\n" + + " \n" + + " 4.12\n" + + " 2.22.0\n" + + "%PROPERTIES%" + + " \n" + + "\n" + + " \n" + + " \n" + + " junit\n" + + " junit\n" + + " ${maven_version}\n" + + " test\n" + + " \n" + + "%DEPENDENCIES%" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " org.apache.maven.plugins\n" + + " maven-surefire-plugin\n" + + " ${surefire_version}\n" + + " \n" + + " \n" + + "%DEPENDENCIES_TO_SCAN%" + + " \n" + + " %GROUPS%\n" + + " com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class\n" + + " \n" + + " %GROUPS%\n" + + " \n" + + " ${env.TEST_DIR}\n" + + " false\n" + + " \n" + + " /home/y/lib64\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " org.apache.maven.plugins\n" + + " maven-surefire-report-plugin\n" + + " ${surefire_version}\n" + + " \n" + + " ${env.TEST_DIR}\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n"; + + static String generatePomXml(TestProfile testProfile, List artifacts, Path testArtifact) { + String properties = artifacts.stream() + .map(path -> PROPERTY_TEMPLATE + .replace("%ARTIFACT_ID%", path.getFileName().toString()) + .replace("%JAR_PATH%", path.toString())) + .collect(Collectors.joining()); + String dependencies = artifacts.stream() + .map(path -> DEPENDENCY_TEMPLATE + .replace("%ARTIFACT_ID%", path.getFileName().toString())) + .collect(Collectors.joining()); + String dependenciesToScan = + DEPENDENCY_TO_SCAN_TEMPLATE + .replace("%ARTIFACT_ID%", testArtifact.getFileName().toString()); + + return POM_XML_TEMPLATE + .replace("%PROPERTIES%", properties) + .replace("%DEPENDENCIES_TO_SCAN%", dependenciesToScan) + .replace("%DEPENDENCIES%", dependencies) + .replace("%GROUPS%", testProfile.group()); + } + + private PomXmlGenerator() {} +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java new file mode 100644 index 00000000000..b7d3a06f30d --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.hosted.testrunner; + +/** + * @author valerijf + * @author jvenstad + */ +enum TestProfile { + + SYSTEM_TEST("com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true), + STAGING_TEST("com.yahoo.vespa.tenant.cd.StagingTest, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true), + PRODUCTION_TEST("com.yahoo.vespa.tenant.cd.ProductionTest, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false); + + private final String group; + private final boolean failIfNoTests; + + TestProfile(String group, boolean failIfNoTests) { + this.group = group; + this.failIfNoTests = failIfNoTests; + } + + String group() { + return group; + } + + boolean failIfNoTests() { + return failIfNoTests; + } + +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java new file mode 100644 index 00000000000..fb5dccc551d --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -0,0 +1,195 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.google.inject.Inject; +import com.yahoo.vespa.defaults.Defaults; +import org.fusesource.jansi.AnsiOutputStream; +import org.fusesource.jansi.HtmlAnsiOutputStream; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.List; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.log.LogLevel.ERROR; + +/** + * @author valerijf + * @author jvenstad + */ +public class TestRunner { + + private static final Logger logger = Logger.getLogger(TestRunner.class.getName()); + private static final Level HTML = new Level("html", 1) { }; + private static final Path vespaHome = Paths.get(Defaults.getDefaults().vespaHome()); + private static final String settingsXml = "\n" + + "\n" + + " \n" + + " \n" + + " maven central\n" + + " *\n" + // Use this for everything! + " https://repo.maven.apache.org/maven2/\n" + + " \n" + + " \n" + + ""; + + private final Path artifactsPath; + private final Path testPath; + private final Path logFile; + private final Path configFile; + private final Path settingsFile; + private final Function testBuilder; + private final SortedMap log = new ConcurrentSkipListMap<>(); + + private volatile Status status = Status.NOT_STARTED; + + @Inject + public TestRunner(TestRunnerConfig config) { + this(config.artifactsPath(), + vespaHome.resolve("tmp/test"), + vespaHome.resolve("logs/vespa/maven.log"), + vespaHome.resolve("tmp/config.json"), + vespaHome.resolve("tmp/settings.xml"), + profile -> { // Anything to make this testable! >_< + String[] command = new String[]{ + "mvn", + "test", + + "--batch-mode", // Run in non-interactive (batch) mode (disables output color) + "--show-version", // Display version information WITHOUT stopping build + "--settings", // Need to override repository settings in ymaven config >_< + vespaHome.resolve("tmp/settings.xml").toString(), + + // Disable maven download progress indication + "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", + "-Dstyle.color=always", // Enable ANSI color codes again + "-DfailIfNoTests=" + profile.failIfNoTests(), + "-Dvespa.test.config=" + vespaHome.resolve("tmp/config.json"), + "-Dvespa.test.credentials.root=" + Defaults.getDefaults().vespaHome() + "/var/vespa/sia", + String.format("-DargLine=-Xms%1$dm -Xmx%1$dm", config.surefireMemoryMb()) + }; + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().merge("MAVEN_OPTS", " -Djansi.force=true", String::concat); + builder.directory(vespaHome.resolve("tmp/test").toFile()); + builder.redirectErrorStream(true); + return builder; + }); + } + + TestRunner(Path artifactsPath, Path testPath, Path logFile, Path configFile, Path settingsFile, Function testBuilder) { + this.artifactsPath = artifactsPath; + this.testPath = testPath; + this.logFile = logFile; + this.configFile = configFile; + this.settingsFile = settingsFile; + this.testBuilder = testBuilder; + } + + public synchronized void test(TestProfile testProfile, byte[] testConfig) { + if (status == Status.RUNNING) + throw new IllegalArgumentException("Tests are already running; should not receive this request now."); + + log.clear(); + status = Status.RUNNING; + + new Thread(() -> runTests(testProfile, testConfig)).start(); + } + + public Collection getLog(long after) { + return log.tailMap(after + 1).values(); + } + + public synchronized Status getStatus() { + return status; + } + + private void runTests(TestProfile testProfile, byte[] testConfig) { + ProcessBuilder builder = testBuilder.apply(testProfile); + { + LogRecord record = new LogRecord(Level.INFO, + String.format("Starting %s. Artifacts directory: %s Config file: %s\nCommand to run: %s", + testProfile.name(), artifactsPath, configFile, String.join(" ", builder.command()))); + log.put(record.getSequenceNumber(), record); + logger.log(record); + } + + boolean success; + // The AnsiOutputStream filters out ANSI characters, leaving the file contents pure. + try (PrintStream fileStream = new PrintStream(new AnsiOutputStream(new BufferedOutputStream(new FileOutputStream(logFile.toFile())))); + ByteArrayOutputStream logBuffer = new ByteArrayOutputStream(); + PrintStream logFormatter = new PrintStream(new HtmlAnsiOutputStream(logBuffer))){ + writeTestApplicationPom(testProfile); + Files.write(configFile, testConfig); + Files.write(settingsFile, settingsXml.getBytes()); + + Process mavenProcess = builder.start(); + BufferedReader in = new BufferedReader(new InputStreamReader(mavenProcess.getInputStream())); + in.lines().forEach(line -> { + fileStream.println(line); + logFormatter.print(line); + LogRecord record = new LogRecord(HTML, logBuffer.toString()); + log.put(record.getSequenceNumber(), record); + logBuffer.reset(); + }); + success = mavenProcess.waitFor() == 0; + } + catch (Exception exception) { + LogRecord record = new LogRecord(ERROR, "Failed to execute maven command: " + String.join(" ", builder.command())); + record.setThrown(exception); + logger.log(record); + log.put(record.getSequenceNumber(), record); + try (PrintStream file = new PrintStream(new FileOutputStream(logFile.toFile(), true))) { + file.println(record.getMessage()); + exception.printStackTrace(file); + } + catch (IOException ignored) { } + status = Status.ERROR; + return; + } + status = success ? Status.SUCCESS : Status.FAILURE; + } + + private void writeTestApplicationPom(TestProfile testProfile) throws IOException { + List files = listFiles(artifactsPath); + Path testJar = files.stream().filter(file -> file.toString().endsWith("tests.jar")).findFirst() + .orElseThrow(() -> new IllegalStateException("No file ending with 'tests.jar' found under '" + artifactsPath + "'!")); + String pomXml = PomXmlGenerator.generatePomXml(testProfile, files, testJar); + testPath.toFile().mkdirs(); + Files.write(testPath.resolve("pom.xml"), pomXml.getBytes()); + } + + private static List listFiles(Path directory) { + try (Stream element = Files.walk(directory)) { + return element + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jar")) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to list files under " + directory, e); + } + } + + + public enum Status { + NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS + } + +} diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java new file mode 100644 index 00000000000..d3393ce8dbe --- /dev/null +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -0,0 +1,166 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.yolean.Exceptions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Collection; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static com.yahoo.jdisc.Response.Status; + +/** + * @author valerijf + * @author jvenstad + */ +public class TestRunnerHandler extends LoggingRequestHandler { + + private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json"; + + private final TestRunner testRunner; + + @Inject + public TestRunnerHandler(Executor executor, AccessLog accessLog, TestRunner testRunner) { + super(executor, accessLog); + this.testRunner = testRunner; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case POST: return handlePOST(request); + + default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported"); + } + } catch (IllegalArgumentException e) { + return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e)); + } catch (Exception e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.equals("/tester/v1/log")) { + return new SlimeJsonResponse(toSlime(testRunner.getLog(request.hasProperty("after") + ? Long.parseLong(request.getProperty("after")) + : -1))); + } else if (path.equals("/tester/v1/status")) { + log.info("Responding with status " + testRunner.getStatus()); + return new Response(testRunner.getStatus().name()); + } + return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); + } + + private HttpResponse handlePOST(HttpRequest request) throws IOException, InterruptedException { + final String path = request.getUri().getPath(); + if (path.startsWith("/tester/v1/run/")) { + String type = lastElement(path); + TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST"); + byte[] config = IOUtils.readBytes(request.getData(), 1 << 16); + testRunner.test(testProfile, config); + log.info("Started tests of type " + type + " and status is " + testRunner.getStatus()); + return new Response("Successfully started " + type + " tests"); + } + return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); + } + + private static String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash + 1, path.length()); + } + + static Slime toSlime(Collection log) { + Slime root = new Slime(); + Cursor recordArray = root.setArray(); + log.forEach(record -> { + Cursor recordObject = recordArray.addObject(); + recordObject.setLong("id", record.getSequenceNumber()); + recordObject.setLong("at", record.getMillis()); + recordObject.setString("type", typeOf(record.getLevel())); + String message = record.getMessage(); + if (record.getThrown() != null) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + record.getThrown().printStackTrace(new PrintStream(buffer)); + message += "\n" + buffer; + } + recordObject.setString("message", message); + }); + return root; + } + + public static String typeOf(Level level) { + return level.getName().equals("html") ? "html" + : level.intValue() < LogLevel.INFO.intValue() ? "debug" + : level.intValue() < LogLevel.WARNING.intValue() ? "info" + : level.intValue() < LogLevel.ERROR.intValue() ? "warning" + : "error"; + } + + private class SlimeJsonResponse extends HttpResponse { + private final Slime slime; + + private SlimeJsonResponse(Slime slime) { + super(200); + this.slime = slime; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return CONTENT_TYPE_APPLICATION_JSON; + } + } + + private static class Response extends HttpResponse { + private static final ObjectMapper objectMapper = new ObjectMapper(); + private final String message; + + private Response(String response) { + this(200, response); + } + + private Response(int statusCode, String message) { + super(statusCode); + this.message = message; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.put("message", message); + objectMapper.writeValue(outputStream, objectNode); + } + + @Override + public String getContentType() { + return CONTENT_TYPE_APPLICATION_JSON; + } + } +} diff --git a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def new file mode 100644 index 00000000000..a2d0eacd9be --- /dev/null +++ b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def @@ -0,0 +1,4 @@ +package=com.yahoo.vespa.hosted.testrunner + +artifactsPath path +surefireMemoryMb int diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java new file mode 100644 index 00000000000..dce02922c63 --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.testrunner; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author valerijf + */ +public class PomXmlGeneratorTest { + + @Test + public void write_system_tests_pom_xml() throws IOException { + List artifacts = Arrays.asList( + Paths.get("components/my-comp.jar"), + Paths.get("main.jar")); + + String actual = PomXmlGenerator.generatePomXml(TestProfile.SYSTEM_TEST, artifacts, artifacts.get(1)); + assertFile("/pom.xml_system_tests", actual); + } + + private void assertFile(String resourceFile, String actual) throws IOException { + String expected = IOUtils.toString(this.getClass().getResourceAsStream(resourceFile)); + assertEquals(resourceFile, expected, actual); + } +} \ No newline at end of file diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java new file mode 100644 index 00000000000..a91b1308080 --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.hosted.testrunner; + +import com.yahoo.vespa.config.SlimeUtils; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.time.Instant; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static org.junit.Assert.assertEquals; + +/** + * @author jvenstad + */ +public class TestRunnerHandlerTest { + + @Test + public void logSerialization() throws IOException { + LogRecord record = new LogRecord(Level.INFO, "Hello."); + record.setSequenceNumber(1); + record.setInstant(Instant.ofEpochMilli(2)); + Exception exception = new RuntimeException(); + record.setThrown(exception); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + exception.printStackTrace(new PrintStream(buffer)); + String trace = buffer.toString() + .replaceAll("\n", "\\\\n") + .replaceAll("\t", "\\\\t"); + assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]", + new String(SlimeUtils.toJsonBytes(TestRunnerHandler.toSlime(Collections.singletonList(record))))); + } + +} diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java new file mode 100644 index 00000000000..49c95fa4b6f --- /dev/null +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java @@ -0,0 +1,127 @@ +package com.yahoo.vespa.hosted.testrunner; + +import org.fusesource.jansi.Ansi; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.logging.LogRecord; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Unit tests relying on a UNIX shell >_< + * + * @author jvenstad + */ +public class TestRunnerTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private Path artifactsPath; + private Path testPath; + private Path logFile; + private Path configFile; + private Path settingsFile; + + @Before + public void setup() throws IOException { + artifactsPath = tmp.newFolder("artifacts").toPath(); + Files.createFile(artifactsPath.resolve("my-tests.jar")); + Files.createFile(artifactsPath.resolve("my-fat-test.jar")); + testPath = tmp.newFolder("testData").toPath(); + logFile = tmp.newFile("maven.log").toPath(); + configFile = tmp.newFile("testConfig.json").toPath(); + settingsFile = tmp.newFile("settings.xml").toPath(); + } + + @Test + public void ansiCodesAreConvertedToHtml() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("echo", Ansi.ansi().fg(Ansi.Color.RED).a("Hello!").reset().toString())); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + Iterator log = runner.getLog(-1).iterator(); + log.next(); + LogRecord record = log.next(); + assertEquals("Hello!", record.getMessage()); + assertEquals(0, runner.getLog(record.getSequenceNumber()).size()); + assertEquals(TestRunner.Status.SUCCESS, runner.getStatus()); + } + + @Test + public void errorLeadsToError() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("This is a command that doesn't exist, for sure!")); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + Iterator log = runner.getLog(-1).iterator(); + log.next(); + LogRecord record = log.next(); + assertEquals("Failed to execute maven command: This is a command that doesn't exist, for sure!", record.getMessage()); + assertNotNull(record.getThrown()); + assertEquals(TestRunner.Status.ERROR, runner.getStatus()); + } + + @Test + public void failureLeadsToFailure() throws InterruptedException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("false")); + runner.test(TestProfile.SYSTEM_TEST, new byte[0]); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals(1, runner.getLog(-1).size()); + assertEquals(TestRunner.Status.FAILURE, runner.getStatus()); + } + + @Test + public void filesAreGenerated() throws InterruptedException, IOException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("echo", "Hello!")); + runner.test(TestProfile.SYSTEM_TEST, "config".getBytes()); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals("config", new String(Files.readAllBytes(configFile))); + assertTrue(Files.exists(testPath.resolve("pom.xml"))); + assertTrue(Files.exists(settingsFile)); + assertEquals("Hello!\n", new String(Files.readAllBytes(logFile))); + } + + @Test + public void runnerCanBeReused() throws InterruptedException, IOException { + TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile, + __ -> new ProcessBuilder("sleep", "0.1")); + runner.test(TestProfile.SYSTEM_TEST, "config".getBytes()); + assertEquals(TestRunner.Status.RUNNING, runner.getStatus()); + + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + assertEquals(1, runner.getLog(-1).size()); + assertEquals(TestRunner.Status.SUCCESS, runner.getStatus()); + + runner.test(TestProfile.STAGING_TEST, "newConfig".getBytes()); + while (runner.getStatus() == TestRunner.Status.RUNNING) { + Thread.sleep(10); + } + + assertEquals("newConfig", new String(Files.readAllBytes(configFile))); + assertEquals(1, runner.getLog(-1).size()); + } + +} diff --git a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests new file mode 100644 index 00000000000..4f7565e7449 --- /dev/null +++ b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests @@ -0,0 +1,72 @@ + + + 4.0.0 + com.yahoo.vespa + tester-application + 1.0.0 + + + 4.12 + 2.22.0 + components/my-comp.jar + main.jar + + + + + junit + junit + ${maven_version} + test + + + com.yahoo.vespa.testrunner.test + my-comp.jar + system + test-jar + test + ${my-comp.jar.path} + + + com.yahoo.vespa.testrunner.test + main.jar + system + test-jar + test + ${main.jar.path} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire_version} + + + com.yahoo.vespa.testrunner.test:main.jar + + com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest + com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class + + com.yahoo.vespa.tenant.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest + + ${env.TEST_DIR} + false + + /home/y/lib64 + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + ${surefire_version} + + ${env.TEST_DIR} + + + + + -- cgit v1.2.3