Revert "Merge pull request #20066 from vespa-engine/revert-20065-jonmv/vespa-cli-test-runner"

This reverts commit e8e542a66a6bc9b2e33def248f208da63b7b934c, reversing
changes made to 69bd22c6dea4bae7d210a9ea36b31ecd8cd5c7fa.

7 files changed, 52 insertions, 249 deletions

diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
index b3c49fc6f2d..ff66b31dfa8 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.testrunner;
 
 import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.testrunner.legacy.TestProfile;
 
 import java.nio.file.Path;
 import java.util.List;
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java
new file mode 100644
index 00000000000..95a2b2723b8
--- /dev/null
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java
@@ -0,0 +1,30 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.testrunner;
+
+/**
+ * @author valerijf
+ * @author jvenstad
+ */
+public enum TestProfile {
+
+    SYSTEM_TEST("system, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true),
+    STAGING_SETUP_TEST("staging-setup", false),
+    STAGING_TEST("staging, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true),
+    PRODUCTION_TEST("production, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false);
+
+    private final String group;
+    private final boolean failIfNoTests;
+
+    TestProfile(String group, boolean failIfNoTests) {
+        this.group = group;
+        this.failIfNoTests = failIfNoTests;
+    }
+
+    public String group() {
+        return group;
+    }
+
+    public boolean failIfNoTests() {
+        return failIfNoTests;
+    }
+}
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
index 0f6e26d256f..8aa2cd7bc72 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java
@@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.testrunner;
 
 import com.google.inject.Inject;
 import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.testrunner.legacy.LegacyTestRunner;
-import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.fusesource.jansi.AnsiOutputStream;
 import org.fusesource.jansi.HtmlAnsiOutputStream;
 
@@ -38,7 +36,7 @@ import static java.util.logging.Level.SEVERE;
  * @author valerijf
  * @author jvenstad
  */
-public class TestRunner implements LegacyTestRunner {
+public class TestRunner implements com.yahoo.vespa.testrunner.TestRunner {
 
     private static final Logger logger = Logger.getLogger(TestRunner.class.getName());
     private static final Level HTML = new Level("html", 1) { };
@@ -114,14 +112,14 @@ public class TestRunner implements LegacyTestRunner {
         return builder;
     }
 
-    public synchronized void test(TestProfile testProfile, byte[] testConfig) {
+    public synchronized void test(Suite suite, byte[] testConfig) {
         if (status == Status.RUNNING)
             throw new IllegalArgumentException("Tests are already running; should not receive this request now.");
 
         log.clear();
         status = Status.RUNNING;
 
-        new Thread(() -> runTests(testProfile, testConfig)).start();
+        new Thread(() -> runTests(toProfile(suite), testConfig)).start();
     }
 
     public Collection<LogRecord> getLog(long after) {
@@ -210,4 +208,14 @@ public class TestRunner implements LegacyTestRunner {
         private NoTestsException(String message) { super(message); }
     }
 
+    static TestProfile toProfile(Suite suite) {
+        switch (suite) {
+            case SYSTEM_TEST: return TestProfile.SYSTEM_TEST;
+            case STAGING_SETUP_TEST: return TestProfile.STAGING_SETUP_TEST;
+            case STAGING_TEST: return TestProfile.STAGING_TEST;
+            case PRODUCTION_TEST: return TestProfile.PRODUCTION_TEST;
+            default: throw new IllegalArgumentException("Unknown test suite '" + suite + "'");
+        }
+    }
+
 }
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
deleted file mode 100644
index 621f0964a40..00000000000
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
+++ /dev/null
@@ -1,171 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.testrunner;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.inject.Inject;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.LoggingRequestHandler;
-import com.yahoo.io.IOUtils;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.JsonFormat;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.testrunner.legacy.TestProfile;
-import com.yahoo.yolean.Exceptions;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.util.Collection;
-import java.util.concurrent.Executor;
-import java.util.logging.Level;
-import java.util.logging.LogRecord;
-
-import static com.yahoo.jdisc.Response.Status;
-
-/**
- * @author valerijf
- * @author jvenstad
- */
-public class TestRunnerHandler extends LoggingRequestHandler {
-
-    private static final String CONTENT_TYPE_APPLICATION_JSON = "application/json";
-
-    private final TestRunner testRunner;
-
-    @Inject
-    public TestRunnerHandler(Executor executor, TestRunner testRunner) {
-        super(executor);
-        this.testRunner = testRunner;
-    }
-
-    @Override
-    public HttpResponse handle(HttpRequest request) {
-        try {
-            switch (request.getMethod()) {
-                case GET: return handleGET(request);
-                case POST: return handlePOST(request);
-
-                default: return new Response(Status.METHOD_NOT_ALLOWED, "Method '" + request.getMethod() + "' is not supported");
-            }
-        } catch (IllegalArgumentException e) {
-            return new Response(Status.BAD_REQUEST, Exceptions.toMessageString(e));
-        } catch (Exception e) {
-            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
-            return new Response(Status.INTERNAL_SERVER_ERROR, Exceptions.toMessageString(e));
-        }
-    }
-
-    private HttpResponse handleGET(HttpRequest request) {
-        String path = request.getUri().getPath();
-        if (path.equals("/tester/v1/log")) {
-            return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after")
-                                                                               ? Long.parseLong(request.getProperty("after"))
-                                                                               : -1)));
-        } else if (path.equals("/tester/v1/status")) {
-            log.info("Responding with status " + testRunner.getStatus());
-            return new Response(testRunner.getStatus().name());
-        }
-        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
-    }
-
-    private HttpResponse handlePOST(HttpRequest request) throws IOException {
-        final String path = request.getUri().getPath();
-        if (path.startsWith("/tester/v1/run/")) {
-            String type = lastElement(path);
-            TestProfile testProfile = TestProfile.valueOf(type.toUpperCase() + "_TEST");
-            byte[] config = IOUtils.readBytes(request.getData(), 1 << 16);
-            testRunner.test(testProfile, config);
-            log.info("Started tests of type " + type + " and status is " + testRunner.getStatus());
-            return new Response("Successfully started " + type + " tests");
-        }
-        return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
-    }
-
-    private static String lastElement(String path) {
-        if (path.endsWith("/"))
-            path = path.substring(0, path.length() - 1);
-        int lastSlash = path.lastIndexOf("/");
-        if (lastSlash < 0) return path;
-        return path.substring(lastSlash + 1);
-    }
-
-    static Slime logToSlime(Collection<LogRecord> log) {
-        Slime slime = new Slime();
-        Cursor root = slime.setObject();
-        Cursor recordArray = root.setArray("logRecords");
-        logArrayToSlime(recordArray, log);
-        return slime;
-    }
-
-    static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) {
-        log.forEach(record -> {
-            Cursor recordObject = recordArray.addObject();
-            recordObject.setLong("id", record.getSequenceNumber());
-            recordObject.setLong("at", record.getMillis());
-            recordObject.setString("type", typeOf(record.getLevel()));
-            String message = record.getMessage();
-            if (record.getThrown() != null) {
-                ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-                record.getThrown().printStackTrace(new PrintStream(buffer));
-                message += "\n" + buffer;
-            }
-            recordObject.setString("message", message);
-        });
-    }
-
-    public static String typeOf(Level level) {
-        return    level.getName().equals("html") ? "html"
-                : level.intValue() < Level.INFO.intValue() ? "debug"
-                : level.intValue() < Level.WARNING.intValue() ? "info"
-                : level.intValue() < Level.SEVERE.intValue() ? "warning"
-                : "error";
-    }
-
-    private static class SlimeJsonResponse extends HttpResponse {
-        private final Slime slime;
-
-        private SlimeJsonResponse(Slime slime) {
-            super(200);
-            this.slime = slime;
-        }
-
-        @Override
-        public void render(OutputStream outputStream) throws IOException {
-            new JsonFormat(true).encode(outputStream, slime);
-        }
-
-        @Override
-        public String getContentType() {
-            return CONTENT_TYPE_APPLICATION_JSON;
-        }
-    }
-
-    private static class Response extends HttpResponse {
-        private static final ObjectMapper objectMapper = new ObjectMapper();
-        private final String message;
-
-        private Response(String response) {
-            this(200, response);
-        }
-
-        private Response(int statusCode, String message) {
-            super(statusCode);
-            this.message = message;
-        }
-
-        @Override
-        public void render(OutputStream outputStream) throws IOException {
-            ObjectNode objectNode = objectMapper.createObjectNode();
-            objectNode.put("message", message);
-            objectMapper.writeValue(outputStream, objectNode);
-        }
-
-        @Override
-        public String getContentType() {
-            return CONTENT_TYPE_APPLICATION_JSON;
-        }
-    }
-}
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
index 391e2a0abbf..943583ae42b 100644
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.testrunner;
 
-import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.junit.Test;
 
 import java.io.IOException;
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
deleted file mode 100644
index fdc6b633630..00000000000
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.testrunner;
-
-import com.yahoo.slime.SlimeUtils;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.logging.Level;
-import java.util.logging.LogRecord;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author jvenstad
- */
-public class TestRunnerHandlerTest {
-
-    @Test
-    public void logSerialization() throws IOException {
-        Log log = new Log();
-        LogRecord record = log.getLogRecord();
-        String trace = log.getTrace();
-        assertEquals("{\"logRecords\":[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]}",
-                     new String(SlimeUtils.toJsonBytes(TestRunnerHandler.logToSlime(Collections.singletonList(record)))));
-    }
-
-    private static class Log {
-
-        private final LogRecord record;
-        private final String trace;
-
-        public Log() {
-            Exception exception = new RuntimeException();
-            record = createRecord(exception);
-            ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-            exception.printStackTrace(new PrintStream(buffer));
-            trace = buffer.toString()
-                    .replaceAll("\n", "\\\\n")
-                    .replaceAll("\t", "\\\\t");
-        }
-
-        LogRecord getLogRecord() {
-            return record;
-        }
-
-        String getTrace() {
-            return trace;
-        }
-
-        private static LogRecord createRecord(Exception exception) {
-            LogRecord record = new LogRecord(Level.INFO, "Hello.");
-            record.setSequenceNumber(1);
-            record.setInstant(Instant.ofEpochMilli(2));
-            record.setThrown(exception);
-            return record;
-        }
-    }
-
-}
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
index 4612f5b217a..b513dfba8b5 100644
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.testrunner;
 
-import com.yahoo.vespa.testrunner.legacy.TestProfile;
 import org.fusesource.jansi.Ansi;
 import org.junit.Before;
 import org.junit.Rule;
@@ -14,6 +13,8 @@ import java.nio.file.Path;
 import java.util.Iterator;
 import java.util.logging.LogRecord;
 
+import static com.yahoo.vespa.testrunner.TestRunner.Suite.STAGING_TEST;
+import static com.yahoo.vespa.testrunner.TestRunner.Suite.SYSTEM_TEST;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -48,7 +49,7 @@ public class TestRunnerTest {
     public void ansiCodesAreConvertedToHtml() throws InterruptedException {
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("echo", Ansi.ansi().fg(Ansi.Color.RED).a("Hello!").reset().toString()));
-        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        runner.test(SYSTEM_TEST, new byte[0]);
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }
@@ -65,7 +66,7 @@ public class TestRunnerTest {
         Files.delete(artifactsPath.resolve("my-tests.jar"));
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("This is a command that doesn't exist, for sure!"));
-        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        runner.test(SYSTEM_TEST, new byte[0]);
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }
@@ -81,7 +82,7 @@ public class TestRunnerTest {
     public void errorLeadsToError() throws InterruptedException {
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("false"));
-        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        runner.test(SYSTEM_TEST, new byte[0]);
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }
@@ -93,7 +94,7 @@ public class TestRunnerTest {
     public void failureLeadsToFailure() throws InterruptedException {
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("false"));
-        runner.test(TestProfile.SYSTEM_TEST, new byte[0]);
+        runner.test(SYSTEM_TEST, new byte[0]);
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }
@@ -105,7 +106,7 @@ public class TestRunnerTest {
     public void filesAreGenerated() throws InterruptedException, IOException {
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("echo", "Hello!"));
-        runner.test(TestProfile.SYSTEM_TEST, "config".getBytes());
+        runner.test(SYSTEM_TEST, "config".getBytes());
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }
@@ -119,7 +120,7 @@ public class TestRunnerTest {
     public void runnerCanBeReused() throws InterruptedException, IOException {
         TestRunner runner = new TestRunner(artifactsPath, testPath, logFile, configFile, settingsFile,
                                            __ -> new ProcessBuilder("sleep", "0.1"));
-        runner.test(TestProfile.SYSTEM_TEST, "config".getBytes());
+        runner.test(SYSTEM_TEST, "config".getBytes());
         assertEquals(TestRunner.Status.RUNNING, runner.getStatus());
 
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
@@ -128,7 +129,7 @@ public class TestRunnerTest {
         assertEquals(1, runner.getLog(-1).size());
         assertEquals(TestRunner.Status.SUCCESS, runner.getStatus());
 
-        runner.test(TestProfile.STAGING_TEST, "newConfig".getBytes());
+        runner.test(STAGING_TEST, "newConfig".getBytes());
         while (runner.getStatus() == TestRunner.Status.RUNNING) {
             Thread.sleep(10);
         }