Execute Junit tests in container

6 files changed, 452 insertions, 0 deletions

diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml
new file mode 100644
index 00000000000..31bf6eba18b
--- /dev/null
+++ b/vespa-osgi-testrunner/pom.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>parent</artifactId>
+        <groupId>com.yahoo.vespa</groupId>
+        <version>7-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>vespa-osgi-testrunner</artifactId>
+    <packaging>container-plugin</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-dev</artifactId>
+            <version>7-SNAPSHOT</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- Verify that we need all junit deps -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>5.6.2</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.jupiter</groupId>
+                    <artifactId>junit-jupiter-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-launcher</artifactId>
+            <version>1.6.2</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.jupiter</groupId>
+                    <artifactId>junit-jupiter-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>5.6.2</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.jupiter</groupId>
+                    <artifactId>junit-jupiter-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>tenant-cd-api</artifactId>
+            <version>7-SNAPSHOT</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
+                <version>${project.version}</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <attachBundleArtifact>true</attachBundleArtifact>
+                    <bundleClassifierName>deploy</bundleClassifierName>
+                    <useCommonAssemblyIds>false</useCommonAssemblyIds>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java
new file mode 100644
index 00000000000..5534ea29d13
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitHandler.java
@@ -0,0 +1,58 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import com.yahoo.container.handler.metrics.JsonResponse;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.restapi.ErrorResponse;
+import com.yahoo.restapi.MessageResponse;
+import org.osgi.framework.Bundle;
+
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.function.Function;
+
+/**
+ * @author mortent
+ */
+public class JunitHandler extends LoggingRequestHandler {
+
+    private final JunitRunner junitRunner;
+
+    public JunitHandler(Executor executor, AccessLog accessLog, JunitRunner junitRunner) {
+        super(executor, accessLog);
+        this.junitRunner = junitRunner;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest httpRequest) {
+        String mode = property("mode", "help", httpRequest, String::valueOf);
+        TestDescriptor.TestCategory category = property("category", TestDescriptor.TestCategory.systemtest, httpRequest, TestDescriptor.TestCategory::valueOf);
+
+        if ("help".equalsIgnoreCase(mode)) {
+            return new MessageResponse("Accepted modes: \n help \n list \n execute");
+        }
+
+        if (!"list".equalsIgnoreCase(mode) && !"execute".equalsIgnoreCase(mode)) {
+            return new ErrorResponse(400, "client error", "Unknown mode \"" + mode + "\"");
+        }
+
+        Bundle testBundle = junitRunner.findTestBundle("-tests");
+        TestDescriptor testDescriptor = junitRunner.loadTestDescriptor(testBundle);
+        List<Class<?>> testClasses = junitRunner.loadClasses(testBundle, testDescriptor, category);
+
+        String jsonResponse = junitRunner.executeTests(testClasses);
+
+        return new JsonResponse(200, jsonResponse);
+    }
+
+    private static <VAL> VAL property(String name, VAL defaultValue, HttpRequest request, Function<String, VAL> converter) {
+        final String propertyString = request.getProperty(name);
+        if (propertyString != null) {
+            return converter.apply(propertyString);
+        }
+        return defaultValue;
+    }
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
new file mode 100644
index 00000000000..68d3ea4c124
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java
@@ -0,0 +1,167 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.exception.ExceptionUtils;
+import com.yahoo.io.IOUtils;
+import com.yahoo.jdisc.application.OsgiFramework;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.yolean.Exceptions;
+import org.junit.jupiter.engine.JupiterTestEngine;
+import org.junit.platform.engine.discovery.DiscoverySelectors;
+import org.junit.platform.launcher.Launcher;
+import org.junit.platform.launcher.LauncherConstants;
+import org.junit.platform.launcher.LauncherDiscoveryRequest;
+import org.junit.platform.launcher.core.LauncherConfig;
+import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
+import org.junit.platform.launcher.core.LauncherFactory;
+import org.junit.platform.launcher.listeners.LoggingListener;
+import org.junit.platform.launcher.listeners.SummaryGeneratingListener;
+import org.junit.platform.launcher.listeners.TestExecutionSummary;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author mortent
+ */
+public class JunitRunner extends AbstractComponent {
+    private static final Logger logger = Logger.getLogger(JunitRunner.class.getName());
+
+    private final BundleContext bundleContext;
+
+    @Inject
+    public JunitRunner(OsgiFramework osgiFramework) {
+        // TODO mortent: Find a way to workaround this hack
+        var tmp = osgiFramework.bundleContext();
+        try {
+            var field = tmp.getClass().getDeclaredField("wrapped");
+            field.setAccessible(true);
+            bundleContext = (BundleContext) field.get(tmp);
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public Bundle findTestBundle(String bundleNameSuffix) {
+        return Stream.of(bundleContext.getBundles())
+                .filter(bundle -> bundle.getSymbolicName().endsWith(bundleNameSuffix))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException("No bundle on classpath with name ending on " + bundleNameSuffix));
+    }
+
+    public TestDescriptor loadTestDescriptor(Bundle bundle) {
+        URL resource = bundle.getEntry("META-INF/testClasses.json");
+        TestDescriptor testDescriptor;
+        try {
+            var jsonDescriptor = IOUtils.readAll(resource.openStream(), Charset.defaultCharset()).trim();
+            testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor);
+            logger.info( "Test classes in bundle :" + testDescriptor.toString());
+            return testDescriptor;
+        } catch (IOException e) {
+            throw new RuntimeException("Could not load META-INF/testClasses.json [" + e.getMessage() + "]");
+        }
+    }
+
+    public List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) {
+        List<Class<?>> testClasses = testDescriptor.getConfiguredTests(testCategory).stream()
+                .map(className -> loadClass(bundle, className))
+                .collect(Collectors.toList());
+
+        StringBuffer buffer = new StringBuffer();
+        testClasses.forEach(cl -> buffer.append("\t").append(cl.toString()).append(" / ").append(cl.getClassLoader().toString()).append("\n"));
+        logger.info("Loaded testClasses: \n" + buffer.toString());
+        return testClasses;
+    }
+
+    private Class<?> loadClass(Bundle bundle, String className) {
+        try {
+            return bundle.loadClass(className);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException("Could not find class: " + className + " in bundle " + bundle.getSymbolicName(), e);
+        }
+    }
+
+    public String executeTests(List<Class<?>> testClasses) {
+        LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request()
+                .selectors(
+                        testClasses.stream().map(DiscoverySelectors::selectClass).collect(Collectors.toList())
+                )
+                .configurationParameter(LauncherConstants.CAPTURE_STDERR_PROPERTY_NAME,"true")
+                .configurationParameter(LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME,"true")
+                .build();
+
+        var launcherConfig = LauncherConfig.builder()
+                .addTestEngines(new JupiterTestEngine())
+
+                .build();
+        Launcher launcher = LauncherFactory.create(launcherConfig);
+
+        // Create log listener:
+        var logLines = new ArrayList<String>();
+        var logListener = LoggingListener.forBiConsumer((t, m) -> log(logLines, m.get(), t));
+        // Create a summary listener:
+        var summaryListener = new SummaryGeneratingListener();
+        launcher.registerTestExecutionListeners(logListener, summaryListener);
+
+        // Execute request
+        launcher.execute(discoveryRequest);
+
+        var report = summaryListener.getSummary();
+
+        return createJsonTestReport(report, logLines);
+    }
+
+    private String createJsonTestReport(TestExecutionSummary report, List<String> logLines) {
+        var slime = new Slime();
+        var root = slime.setObject();
+        var summary = root.setObject("summary");
+        summary.setLong("Total tests", report.getTestsFoundCount());
+        summary.setLong("Test success", report.getTestsSucceededCount());
+        summary.setLong("Test failed", report.getTestsFailedCount());
+        summary.setLong("Test ignored", report.getTestsSkippedCount());
+        summary.setLong("Test success", report.getTestsAbortedCount());
+        summary.setLong("Test started", report.getTestsStartedCount());
+        var failures = summary.setArray("failures");
+        report.getFailures().forEach(failure -> serializeFailure(failure, failures.addObject()));
+
+        var output = root.setArray("output");
+        logLines.forEach(output::addString);
+
+        return Exceptions.uncheck(() -> new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8));
+    }
+
+    private void serializeFailure(TestExecutionSummary.Failure failure, Cursor slime) {
+        var testIdentifier = failure.getTestIdentifier();
+        slime.setString("testName", testIdentifier.getUniqueId());
+        slime.setString("testError",failure.getException().getMessage());
+        slime.setString("exception", ExceptionUtils.getStackTraceAsString(failure.getException()));
+    }
+
+    private void log(List<String> logs, String message, Throwable t) {
+        logs.add(message);
+        if(t != null) {
+            logs.add(t.getMessage());
+            List.of(t.getStackTrace()).stream()
+                    .map(StackTraceElement::toString)
+                    .forEach(logs::add);
+        }
+    }
+
+    @Override
+    public void deconstruct() {
+        super.deconstruct();
+    }
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java
new file mode 100644
index 00000000000..1c7092de93a
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestDescriptor.java
@@ -0,0 +1,60 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.testrunner;
+
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.SlimeStream;
+import com.yahoo.slime.SlimeUtils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author mortent
+ */
+public class TestDescriptor {
+    private final Map<TestCategory, List<String>> configuredTestClasses;
+    private final String version;
+
+    private TestDescriptor(String version, Map<TestCategory, List<String>> configuredTestClasses) {
+        this.version = version;
+        this.configuredTestClasses = configuredTestClasses;
+    }
+
+    public static TestDescriptor fromJsonString(String testDescriptor) {
+        var slime = SlimeUtils.jsonToSlime(testDescriptor);
+        var root = slime.get();
+        var version = root.field("version").asString();
+        var testRoot = root.field("configuredTests");
+        var systemTests = getJsonArray(testRoot, "systemTests");
+        var stagingTests = getJsonArray(testRoot, "stagingTests");
+        var productionTests = getJsonArray(testRoot, "productionTests");
+        return new TestDescriptor(version, Map.of(
+                TestCategory.systemtest, systemTests,
+                TestCategory.stagingtest, stagingTests,
+                TestCategory.productiontest, productionTests
+        ));
+    }
+
+    private static List<String> getJsonArray(Cursor cursor, String field) {
+        return SlimeStream.fromArray(cursor.field(field), Inspector::asString).collect(Collectors.toList());
+    }
+
+    public String version() {
+        return version;
+    }
+
+    public List<String> getConfiguredTests(TestCategory category) {
+        return List.copyOf(configuredTestClasses.get(category));
+    }
+
+    @Override
+    public String toString() {
+        return "TestClassDescriptor{" +
+               "configuredTestClasses=" + configuredTestClasses +
+               '}';
+    }
+
+    public enum TestCategory {systemtest, stagingtest, productiontest}
+}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java
new file mode 100644
index 00000000000..2606b3f839d
--- /dev/null
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/package-info.java
@@ -0,0 +1,9 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @author mortent
+ */
+@ExportPackage
+package com.yahoo.vespa.testrunner;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java b/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java
new file mode 100644
index 00000000000..9462a52ed9f
--- /dev/null
+++ b/vespa-osgi-testrunner/src/test/java/TestDescriptorTest.java
@@ -0,0 +1,70 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import com.yahoo.vespa.testrunner.TestDescriptor;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+public class TestDescriptorTest {
+
+    @Test
+    public void parses_system_test_only () {
+        String testDescriptor = "{\n" +
+                                "  \"version\": \"1.0\",\n" +
+                                "  \"configuredTests\": {\n" +
+                                "    \"systemTests\": [\n" +
+                                "      \"ai.vespa.test.SystemTest1\",\n" +
+                                "      \"ai.vespa.test.SystemTest2\"\n" +
+                                "    ]\n" +
+                                "  " +
+                                "}\n" +
+                                "}";
+        var testClassDescriptor = TestDescriptor.fromJsonString(testDescriptor);
+
+        var systemTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.systemtest);
+        Assertions.assertIterableEquals(List.of("ai.vespa.test.SystemTest1", "ai.vespa.test.SystemTest2"), systemTests);
+
+        var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest);
+        Assertions.assertIterableEquals(Collections.emptyList(), stagingTests);
+
+        var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest);
+        Assertions.assertIterableEquals(Collections.emptyList(), productionTests);
+    }
+
+    @Test
+    public void parsesDescriptorFile() {
+        String testDescriptor = "{\n" +
+                                "  \"version\": \"1.0\",\n" +
+                                "  \"configuredTests\": {\n" +
+                                "    \"systemTests\": [\n" +
+                                "      \"ai.vespa.test.SystemTest1\",\n" +
+                                "      \"ai.vespa.test.SystemTest2\"\n" +
+                                "    ],\n" +
+                                "    \"stagingTests\": [\n" +
+                                "      \"ai.vespa.test.StagingTest1\",\n" +
+                                "      \"ai.vespa.test.StagingTest2\"\n" +
+                                "    ],\n" +
+                                "    \"productionTests\": [\n" +
+                                "      \"ai.vespa.test.ProductionTest1\",\n" +
+                                "      \"ai.vespa.test.ProductionTest2\"\n" +
+                                "    ]\n" +
+                                "  " +
+                                "}\n" +
+                                "}";
+        var testClassDescriptor = TestDescriptor.fromJsonString(testDescriptor);
+
+        var systemTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.systemtest);
+        Assertions.assertIterableEquals(List.of("ai.vespa.test.SystemTest1", "ai.vespa.test.SystemTest2"), systemTests);
+
+        var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest);
+        Assertions.assertIterableEquals(List.of("ai.vespa.test.StagingTest1", "ai.vespa.test.StagingTest2"), stagingTests);
+
+        var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest);
+        Assertions.assertIterableEquals(List.of("ai.vespa.test.ProductionTest1", "ai.vespa.test.ProductionTest2"), productionTests);
+    }
+}