Adds steps to the run REST API

Adds a "steps" field to the
/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}
REST API with the following structure:

  "steps": {
    "deployTester": {
      "status": "succeeded",
      "startMillis": 1577972067092
    },
    "deployReal": {
      "status": "succeeded",
      "startMillis": 1577972067092
    },
    ...
  }

10 files changed, 125 insertions, 21 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index e43877cb7e5..e276bdac35a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -68,7 +68,7 @@ public class ControllerMaintenance extends AbstractComponent {
         deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl);
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, controller.serviceRegistry().ownershipIssues());
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
-        jobRunner = new JobRunner(controller, Duration.ofSeconds(90), jobControl);
+        jobRunner = new JobRunner(controller, Duration.ofSeconds(90), jobControl, controller.clock());
         osUpgraders = osUpgraders(controller, jobControl);
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index 4c61b341c20..eadb5e8203a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -12,8 +12,8 @@ import com.yahoo.vespa.hosted.controller.deployment.StepInfo;
 import com.yahoo.vespa.hosted.controller.deployment.StepRunner;
 import org.jetbrains.annotations.TestOnly;
 
+import java.time.Clock;
 import java.time.Duration;
-import java.time.Instant;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -34,18 +34,21 @@ public class JobRunner extends Maintainer {
     private final JobController jobs;
     private final ExecutorService executors;
     private final StepRunner runner;
+    private final Clock clock;
 
-    public JobRunner(Controller controller, Duration duration, JobControl jobControl) {
-        this(controller, duration, jobControl, Executors.newFixedThreadPool(32), new InternalStepRunner(controller));
+    public JobRunner(Controller controller, Duration duration, JobControl jobControl, Clock clock) {
+        this(controller, duration, jobControl, Executors.newFixedThreadPool(32), new InternalStepRunner(controller), clock);
     }
 
     @TestOnly
-    public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) {
+    public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors,
+                     StepRunner runner, Clock clock) {
         super(controller, duration, jobControl);
         this.jobs = controller.jobController();
         this.jobs.setRunner(this::advance);
         this.executors = executors;
         this.runner = runner;
+        this.clock = clock;
     }
 
     @Override
@@ -102,7 +105,7 @@ public class JobRunner extends Maintainer {
 
                     StepInfo stepInfo = run.stepInfo(lockedStep.get()).orElseThrow();
                     if (stepInfo.startTime().isEmpty()) {
-                        jobs.setStartTimestamp(run.id(), Instant.now(), lockedStep);
+                        jobs.setStartTimestamp(run.id(), clock.instant(), lockedStep);
                     }
 
                     runner.run(lockedStep, run.id()).ifPresent(status -> {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 415768bdea0..b8357818da9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -451,6 +451,13 @@ class JobControllerApiHandlerHelper {
         }
         runLog.lastId().ifPresent(id -> detailsObject.setLong("lastId", id));
 
+        Cursor stepsObject = detailsObject.setObject("steps");
+        run.steps().forEach((step, info) -> {
+            Cursor stepCursor = stepsObject.setObject(step.name());
+            stepCursor.setString("status", info.status().name());
+            info.startTime().ifPresent(startTime -> stepCursor.setLong("startMillis", startTime.toEpochMilli()));
+        });
+
         return new SlimeJsonResponse(slime);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 038a2afb306..6f1483e93b1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
@@ -29,9 +28,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
 import java.time.DayOfWeek;
 import java.time.Duration;
 import java.time.Instant;
-import java.time.LocalDateTime;
 import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAdjusters;
 import java.util.Collections;
 import java.util.logging.Logger;
@@ -90,7 +87,7 @@ public class DeploymentTester {
         cloud = (MockTesterCloud) tester.controller().jobController().cloud();
         var jobControl = new JobControl(tester.controller().curator());
         runner = new JobRunner(tester.controller(), Duration.ofDays(1), jobControl,
-                               JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller()));
+                               JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller()), tester.clock());
         upgrader = new Upgrader(tester.controller(), maintenanceInterval, jobControl, tester.curator());
         upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine.
         readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 74e387ef17e..182ff389d9e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -83,7 +83,7 @@ public class JobRunnerTest {
         StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? Optional.of(error) : Optional.of(running);
         Phaser phaser = new Phaser(1);
         JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                         phasedExecutor(phaser), stepRunner);
+                                         phasedExecutor(phaser), stepRunner, tester.clock());
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
@@ -115,7 +115,7 @@ public class JobRunnerTest {
         JobController jobs = tester.controller().jobController();
         Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class);
         JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                         inThreadExecutor(), mappedRunner(outcomes));
+                                         inThreadExecutor(), mappedRunner(outcomes), tester.clock());
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
@@ -223,7 +223,7 @@ public class JobRunnerTest {
         // Hang during tester deployment, until notified.
         CyclicBarrier barrier = new CyclicBarrier(2);
         JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                         Executors.newFixedThreadPool(32), waitingRunner(barrier));
+                                         Executors.newFixedThreadPool(32), waitingRunner(barrier), tester.clock());
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
@@ -260,7 +260,7 @@ public class JobRunnerTest {
         DeploymentTester tester = new DeploymentTester();
         JobController jobs = tester.controller().jobController();
         JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                         inThreadExecutor(), (id, step) -> Optional.of(running));
+                                         inThreadExecutor(), (id, step) -> Optional.of(running), tester.clock());
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId instanceId = appId.defaultInstance();
@@ -285,7 +285,7 @@ public class JobRunnerTest {
         assertTrue(jobs.details(new RunId(instanceId, systemTest, 257)).isPresent());
 
         JobRunner failureRunner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                                inThreadExecutor(), (id, step) -> Optional.of(error));
+                                                inThreadExecutor(), (id, step) -> Optional.of(error), tester.clock());
 
         // Make all but the oldest of the 256 jobs a failure.
         for (int i = 0; i < jobs.historyLength() - 1; i++) {
@@ -339,7 +339,7 @@ public class JobRunnerTest {
         JobController jobs = tester.controller().jobController();
         Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class);
         JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
-                                         inThreadExecutor(), mappedRunner(outcomes));
+                                         inThreadExecutor(), mappedRunner(outcomes), tester.clock());
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 9fe754c060f..10ea417fb7a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.restapi.application;
 
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
index 21ef5035481..e5b2fef2120 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
@@ -59,5 +59,18 @@
   },
   "active": true,
   "lastId": 9,
+  "steps": {
+    "deployReal": {
+      "startMillis": 0,
+      "status": "succeeded"
+    },
+    "installReal": {
+      "startMillis": 0,
+      "status": "unfinished"
+    },
+    "copyVespaLogs": {
+      "status": "unfinished"
+    }
+  },
   "status": "running"
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
index 6f0c5c8a384..3248bfa78a1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
@@ -50,5 +50,19 @@
   },
   "active": false,
   "lastId": 18,
+  "steps": {
+    "deployReal": {
+      "startMillis": 0,
+      "status": "succeeded"
+    },
+    "installReal": {
+      "startMillis": 0,
+      "status": "succeeded"
+    },
+    "copyVespaLogs": {
+      "startMillis": 0,
+      "status": "succeeded"
+    }
+  },
   "status": "success"
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
index 993c53b52d8..986109cd1be 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
@@ -399,6 +399,48 @@
       }
     ]
   },
-  "lastId": 75
+  "lastId": 75,
+  "steps": {
+    "deployTester": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "deployReal": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "installTester": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "installReal": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "startTests": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "endTests": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "copyVespaLogs": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "deactivateReal": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "deactivateTester": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    },
+    "report": {
+      "status": "succeeded",
+      "startMillis": "(ignore)"
+    }
+  }
 }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
index e344ef07762..4e501410324 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
@@ -1,6 +1,4 @@
 {
-  "active": false,
-  "status": "deploymentFailed",
   "log": {
     "deployTester": [
       {
@@ -17,5 +15,36 @@
       }
     ]
   },
-  "lastId": 2
+  "active": false,
+  "lastId": 2,
+  "steps": {
+    "startTests": {
+      "status": "unfinished"
+    },
+    "deployTester": {
+      "startMillis": 1000,
+      "status": "failed"
+    },
+    "report": {
+      "startMillis": 1000,
+      "status": "succeeded"
+    },
+    "installTester": {
+      "status": "unfinished"
+    },
+    "deployReal": {
+      "status": "unfinished"
+    },
+    "installReal": {
+      "status": "unfinished"
+    },
+    "deactivateTester": {
+      "startMillis": 1000,
+      "status": "succeeded"
+    },
+    "endTests": {
+      "status": "unfinished"
+    }
+  },
+  "status": "deploymentFailed"
 }