diff options
author | Morten Tokle <mortent@verizonmedia.com> | 2020-06-22 08:55:36 +0200 |
---|---|---|
committer | Morten Tokle <mortent@verizonmedia.com> | 2020-06-22 08:55:36 +0200 |
commit | e2dace4bc4f72e86a671664cb1b32eea48361cfb (patch) | |
tree | 2aadf30b9343cdba51646248ad5588f4f22d8bbc /vespa-osgi-testrunner | |
parent | 3a66636ae7790b334467898ab31badf522f70a1e (diff) |
Execute Junit tests in container
Diffstat (limited to 'vespa-osgi-testrunner')
6 files changed, 452 insertions, 0 deletions
diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml new file mode 100644 index 00000000000..31bf6eba18b --- /dev/null +++ b/vespa-osgi-testrunner/pom.xml @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>parent</artifactId> + <groupId>com.yahoo.vespa</groupId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>vespa-osgi-testrunner</artifactId> + <packaging>container-plugin</packaging> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>7-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + + <!-- Verify that we need all junit deps --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>5.6.2</version> + <exclusions> + <exclusion> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.junit.platform</groupId> + <artifactId>junit-platform-launcher</artifactId> + <version>1.6.2</version> + <exclusions> + <exclusion> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <version>5.6.2</version> + <exclusions> + <exclusion> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>7-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${project.version}</version> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project>
\ No newline at end of file diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java new file mode 100644 index 00000000000..5534ea29d13 --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java @@ -0,0 +1,58 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import com.yahoo.container.handler.metrics.JsonResponse; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; +import org.osgi.framework.Bundle; + +import java.util.List; +import java.util.concurrent.Executor; +import java.util.function.Function; + +/** + * @author mortent + */ +public class JunitHandler extends LoggingRequestHandler { + + private final JunitRunner junitRunner; + + public JunitHandler(Executor executor, AccessLog accessLog, JunitRunner junitRunner) { + super(executor, accessLog); + this.junitRunner = junitRunner; + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + String mode = property("mode", "help", httpRequest, String::valueOf); + TestDescriptor.TestCategory category = property("category", TestDescriptor.TestCategory.systemtest, httpRequest, TestDescriptor.TestCategory::valueOf); + + if ("help".equalsIgnoreCase(mode)) { + return new MessageResponse("Accepted modes: \n help \n list \n execute"); + } + + if (!"list".equalsIgnoreCase(mode) && !"execute".equalsIgnoreCase(mode)) { + return new ErrorResponse(400, "client error", "Unknown mode \"" + mode + "\""); + } + + Bundle testBundle = junitRunner.findTestBundle("-tests"); + TestDescriptor testDescriptor = junitRunner.loadTestDescriptor(testBundle); + List<Class<?>> testClasses = junitRunner.loadClasses(testBundle, testDescriptor, category); + + String jsonResponse = junitRunner.executeTests(testClasses); + + return new JsonResponse(200, jsonResponse); + } + + private static <VAL> VAL property(String name, VAL defaultValue, HttpRequest request, Function<String, VAL> converter) { + final String propertyString = request.getProperty(name); + if (propertyString != null) { + return converter.apply(propertyString); + } + return defaultValue; + } +} diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java new file mode 100644 index 00000000000..68d3ea4c124 --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -0,0 +1,167 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.exception.ExceptionUtils; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.yolean.Exceptions; +import org.junit.jupiter.engine.JupiterTestEngine; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.Launcher; +import org.junit.platform.launcher.LauncherConstants; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.core.LauncherConfig; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.LoggingListener; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.launcher.listeners.TestExecutionSummary; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author mortent + */ +public class JunitRunner extends AbstractComponent { + private static final Logger logger = Logger.getLogger(JunitRunner.class.getName()); + + private final BundleContext bundleContext; + + @Inject + public JunitRunner(OsgiFramework osgiFramework) { + // TODO mortent: Find a way to workaround this hack + var tmp = osgiFramework.bundleContext(); + try { + var field = tmp.getClass().getDeclaredField("wrapped"); + field.setAccessible(true); + bundleContext = (BundleContext) field.get(tmp); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public Bundle findTestBundle(String bundleNameSuffix) { + return Stream.of(bundleContext.getBundles()) + .filter(bundle -> bundle.getSymbolicName().endsWith(bundleNameSuffix)) + .findAny() + .orElseThrow(() -> new RuntimeException("No bundle on classpath with name ending on " + bundleNameSuffix)); + } + + public TestDescriptor loadTestDescriptor(Bundle bundle) { + URL resource = bundle.getEntry("META-INF/testClasses.json"); + TestDescriptor testDescriptor; + try { + var jsonDescriptor = IOUtils.readAll(resource.openStream(), Charset.defaultCharset()).trim(); + testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor); + logger.info( "Test classes in bundle :" + testDescriptor.toString()); + return testDescriptor; + } catch (IOException e) { + throw new RuntimeException("Could not load META-INF/testClasses.json [" + e.getMessage() + "]"); + } + } + + public List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) { + List<Class<?>> testClasses = testDescriptor.getConfiguredTests(testCategory).stream() + .map(className -> loadClass(bundle, className)) + .collect(Collectors.toList()); + + StringBuffer buffer = new StringBuffer(); + testClasses.forEach(cl -> buffer.append("\t").append(cl.toString()).append(" / ").append(cl.getClassLoader().toString()).append("\n")); + logger.info("Loaded testClasses: \n" + buffer.toString()); + return testClasses; + } + + private Class<?> loadClass(Bundle bundle, String className) { + try { + return bundle.loadClass(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Could not find class: " + className + " in bundle " + bundle.getSymbolicName(), e); + } + } + + public String executeTests(List<Class<?>> testClasses) { + LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request() + .selectors( + testClasses.stream().map(DiscoverySelectors::selectClass).collect(Collectors.toList()) + ) + .configurationParameter(LauncherConstants.CAPTURE_STDERR_PROPERTY_NAME,"true") + .configurationParameter(LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME,"true") + .build(); + + var launcherConfig = LauncherConfig.builder() + .addTestEngines(new JupiterTestEngine()) + + .build(); + Launcher launcher = LauncherFactory.create(launcherConfig); + + // Create log listener: + var logLines = new ArrayList<String>(); + var logListener = LoggingListener.forBiConsumer((t, m) -> log(logLines, m.get(), t)); + // Create a summary listener: + var summaryListener = new SummaryGeneratingListener(); + launcher.registerTestExecutionListeners(logListener, summaryListener); + + // Execute request + launcher.execute(discoveryRequest); + + var report = summaryListener.getSummary(); + + return createJsonTestReport(report, logLines); + } + + private String createJsonTestReport(TestExecutionSummary report, List<String> logLines) { + var slime = new Slime(); + var root = slime.setObject(); + var summary = root.setObject("summary"); + summary.setLong("Total tests", report.getTestsFoundCount()); + summary.setLong("Test success", report.getTestsSucceededCount()); + summary.setLong("Test failed", report.getTestsFailedCount()); + summary.setLong("Test ignored", report.getTestsSkippedCount()); + summary.setLong("Test success", report.getTestsAbortedCount()); + summary.setLong("Test started", report.getTestsStartedCount()); + var failures = summary.setArray("failures"); + report.getFailures().forEach(failure -> serializeFailure(failure, failures.addObject())); + + var output = root.setArray("output"); + logLines.forEach(output::addString); + + return Exceptions.uncheck(() -> new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8)); + } + + private void serializeFailure(TestExecutionSummary.Failure failure, Cursor slime) { + var testIdentifier = failure.getTestIdentifier(); + slime.setString("testName", testIdentifier.getUniqueId()); + slime.setString("testError",failure.getException().getMessage()); + slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.getException())); + } + + private void log(List<String> logs, String message, Throwable t) { + logs.add(message); + if(t != null) { + logs.add(t.getMessage()); + List.of(t.getStackTrace()).stream() + .map(StackTraceElement::toString) + .forEach(logs::add); + } + } + + @Override + public void deconstruct() { + super.deconstruct(); + } +} diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java new file mode 100644 index 00000000000..1c7092de93a --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java @@ -0,0 +1,60 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.testrunner; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.SlimeStream; +import com.yahoo.slime.SlimeUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author mortent + */ +public class TestDescriptor { + private final Map<TestCategory, List<String>> configuredTestClasses; + private final String version; + + private TestDescriptor(String version, Map<TestCategory, List<String>> configuredTestClasses) { + this.version = version; + this.configuredTestClasses = configuredTestClasses; + } + + public static TestDescriptor fromJsonString(String testDescriptor) { + var slime = SlimeUtils.jsonToSlime(testDescriptor); + var root = slime.get(); + var version = root.field("version").asString(); + var testRoot = root.field("configuredTests"); + var systemTests = getJsonArray(testRoot, "systemTests"); + var stagingTests = getJsonArray(testRoot, "stagingTests"); + var productionTests = getJsonArray(testRoot, "productionTests"); + return new TestDescriptor(version, Map.of( + TestCategory.systemtest, systemTests, + TestCategory.stagingtest, stagingTests, + TestCategory.productiontest, productionTests + )); + } + + private static List<String> getJsonArray(Cursor cursor, String field) { + return SlimeStream.fromArray(cursor.field(field), Inspector::asString).collect(Collectors.toList()); + } + + public String version() { + return version; + } + + public List<String> getConfiguredTests(TestCategory category) { + return List.copyOf(configuredTestClasses.get(category)); + } + + @Override + public String toString() { + return "TestClassDescriptor{" + + "configuredTestClasses=" + configuredTestClasses + + '}'; + } + + public enum TestCategory {systemtest, stagingtest, productiontest} +} diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java new file mode 100644 index 00000000000..2606b3f839d --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author mortent + */ +@ExportPackage +package com.yahoo.vespa.testrunner; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java b/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java new file mode 100644 index 00000000000..9462a52ed9f --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java @@ -0,0 +1,70 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.vespa.testrunner.TestDescriptor; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +/** + * @author mortent + */ +public class TestDescriptorTest { + + @Test + public void parses_system_test_only () { + String testDescriptor = "{\n" + + " \"version\": \"1.0\",\n" + + " \"configuredTests\": {\n" + + " \"systemTests\": [\n" + + " \"ai.vespa.test.SystemTest1\",\n" + + " \"ai.vespa.test.SystemTest2\"\n" + + " ]\n" + + " " + + "}\n" + + "}"; + var testClassDescriptor = TestDescriptor.fromJsonString(testDescriptor); + + var systemTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.systemtest); + Assertions.assertIterableEquals(List.of("ai.vespa.test.SystemTest1", "ai.vespa.test.SystemTest2"), systemTests); + + var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest); + Assertions.assertIterableEquals(Collections.emptyList(), stagingTests); + + var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest); + Assertions.assertIterableEquals(Collections.emptyList(), productionTests); + } + + @Test + public void parsesDescriptorFile() { + String testDescriptor = "{\n" + + " \"version\": \"1.0\",\n" + + " \"configuredTests\": {\n" + + " \"systemTests\": [\n" + + " \"ai.vespa.test.SystemTest1\",\n" + + " \"ai.vespa.test.SystemTest2\"\n" + + " ],\n" + + " \"stagingTests\": [\n" + + " \"ai.vespa.test.StagingTest1\",\n" + + " \"ai.vespa.test.StagingTest2\"\n" + + " ],\n" + + " \"productionTests\": [\n" + + " \"ai.vespa.test.ProductionTest1\",\n" + + " \"ai.vespa.test.ProductionTest2\"\n" + + " ]\n" + + " " + + "}\n" + + "}"; + var testClassDescriptor = TestDescriptor.fromJsonString(testDescriptor); + + var systemTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.systemtest); + Assertions.assertIterableEquals(List.of("ai.vespa.test.SystemTest1", "ai.vespa.test.SystemTest2"), systemTests); + + var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest); + Assertions.assertIterableEquals(List.of("ai.vespa.test.StagingTest1", "ai.vespa.test.StagingTest2"), stagingTests); + + var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest); + Assertions.assertIterableEquals(List.of("ai.vespa.test.ProductionTest1", "ai.vespa.test.ProductionTest2"), productionTests); + } +} |