Support aborting runs, and some JobRunner tests

3 files changed, 159 insertions, 90 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 2a2a6a5e1b9..67c1854e15a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -1,15 +1,46 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.deployment.JobController;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
+import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.AbstractExecutorService;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author jonmv
@@ -17,11 +48,116 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class JobRunnerTest {
 
     @Test
-    public void test() {
-        ControllerTester tester = new ControllerTester();
-        JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()),
-                                         inThreadExecutor(), new DummyStepRunner());
+    public void testMultiThreadedExecutionFinishes() throws InterruptedException {
+        DeploymentTester tester = new DeploymentTester();
+        JobController jobs = tester.controller().jobController();
+        JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
+                                         Executors.newFixedThreadPool(32), sleepy(new DummyStepRunner()));
+
+        ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
+        jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]);
+
+        jobs.run(id, systemTest);
+        try {
+            jobs.run(id, systemTest);
+            fail("Job is already running, so this should not be allowed!");
+        }
+        catch (IllegalStateException e) { }
+        jobs.run(id, stagingTest);
+
+        assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals));
         runner.maintain();
+        assertFalse(jobs.last(id, systemTest).get().hasEnded());
+        Thread.sleep(1000); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine".
+        assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals));
+    }
+
+    @Test
+    public void testStepLogic() {
+        DeploymentTester tester = new DeploymentTester();
+        JobController jobs = tester.controller().jobController();
+        Map<Step, Status> outcomes = new EnumMap<>(Step.class);
+        JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
+                                         inThreadExecutor(), mappedRunner(outcomes));
+
+        ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
+        jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]);
+        Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get();
+
+        jobs.run(id, systemTest);
+        RunId first = run.get().id();
+
+        // Unfinished steps change nothing.
+        Map<Step, Status> steps = run.get().steps();
+        runner.maintain();
+        assertEquals(steps, run.get().steps());
+        assertEquals(Arrays.asList(deployReal), run.get().readySteps());
+
+        // Deployment allows installation.
+        outcomes.put(deployReal, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(installReal), run.get().readySteps());
+
+        // Installation allows tester deployment.
+        outcomes.put(installReal, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(deployTester), run.get().readySteps());
+
+        // Tester deployment allows tester installation.
+        outcomes.put(deployTester, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(installTester), run.get().readySteps());
+
+        // Tester installation allows starting tests.
+        outcomes.put(installTester, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(startTests), run.get().readySteps());
+
+        // Starting tests allows storing data.
+        outcomes.put(startTests, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(runTests), run.get().readySteps());
+
+        // Storing data allows deactivating tester.
+        outcomes.put(runTests, succeeded);
+        runner.maintain();
+        assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps());
+
+        // Failure deactivating real fails the run, but run-always steps continue.
+        outcomes.put(deactivateReal, failed);
+        runner.maintain();
+        assertTrue(run.get().hasFailed());
+        assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps());
+        runner.maintain();
+        assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps());
+
+        // Aborting the run now does nothing, as only run-always steps are left.
+        jobs.abort(run.get().id());
+        runner.maintain();
+        assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps());
+
+        // Success of the remaining run-always steps ends the run.
+        outcomes.put(deactivateReal, succeeded);
+        outcomes.put(deactivateTester, succeeded);
+        outcomes.put(report, succeeded);
+        runner.maintain();
+        assertTrue(run.get().hasFailed());
+        assertTrue(run.get().hasEnded());
+        assertTrue(run.get().isAborted());
+
+        // A new run is attempted.
+        jobs.run(id, systemTest);
+        assertEquals(first.number() + 1, run.get().id().number());
+
+        // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes.
+        outcomes.put(deployTester, failed);
+        runner.maintain();
+        assertTrue(run.get().hasEnded());
+        assertTrue(run.get().hasFailed());
+        assertFalse(run.get().isAborted());
+        assertEquals(failed, run.get().steps().get(deployTester));
+        assertEquals(unfinished, run.get().steps().get(installTester));
+        assertEquals(succeeded, run.get().steps().get(report));
     }
 
     private static ExecutorService inThreadExecutor() {
@@ -36,4 +172,21 @@ public class JobRunnerTest {
         };
     }
 
+
+    private static StepRunner sleepy(StepRunner runner) {
+        return (step, id) -> {
+            try {
+                Thread.sleep(10);
+            }
+            catch (InterruptedException e) {
+                throw new AssertionError("Not supposed to happen.");
+            }
+            return runner.run(step, id);
+        };
+    }
+
+    private static StepRunner mappedRunner(Map<Step, Status> outcomes) {
+        return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java
deleted file mode 100644
index e6e726bec5e..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package com.yahoo.vespa.hosted.controller.persistence;
-
-import com.google.common.collect.ImmutableMap;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
-import com.yahoo.vespa.hosted.controller.deployment.Step;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.Instant;
-import java.util.Collections;
-
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class JobSerializerTest {
-
-    private static final JobSerializer serializer = new JobSerializer();
-    private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json");
-    private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"),
-                                               JobType.productionUsEast3,
-                                               (long) 112358);
-    private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z");
-
-    @Test
-    public void testSerialization() throws IOException {
-        for (Step step : Step.values())
-            assertEquals(step, JobSerializer.stepOf(JobSerializer.valueOf(step)));
-
-        for (Step.Status status : Step.Status.values())
-            assertEquals(status, JobSerializer.statusOf(JobSerializer.valueOf(status)));
-
-        // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date!
-        RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id);
-        for (Step step : Step.values())
-            assertTrue(run.steps().containsKey(step));
-
-        assertEquals(id, run.id());
-        assertEquals(start, run.start());
-        assertFalse(run.hasEnded());
-        assertEquals(ImmutableMap.<Step, Step.Status>builder()
-                    .put(deployInitialReal, unfinished)
-                    .put(installInitialReal, failed)
-                    .put(deployReal, succeeded)
-                    .put(installReal, unfinished)
-                    .put(deactivateReal, failed)
-                    .put(deployTester, succeeded)
-                    .put(installTester, unfinished)
-                    .put(deactivateTester, failed)
-                    .put(startTests, succeeded)
-                    .put(storeData, unfinished)
-                    .put(report, failed)
-                    .build(),
-                     run.steps());
-
-        RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id);
-        assertEquals(run.id(), phoenix.id());
-        assertEquals(run.start(), phoenix.start());
-        assertEquals(run.end(), phoenix.end());
-        assertEquals(run.steps(), phoenix.steps());
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index 4db94eb429a..e31d14ac181 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -14,7 +14,7 @@
       "IT":  "U",
       "DAT": "F",
       "ST":  "S",
-      "SD":  "U",
+      "RT":  "U",
       "R":   "F"
     }
   }