Include test report in job run details

7 files changed, 370 insertions, 2 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 89f57f6ea9b..3ebc8240889 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -640,12 +640,14 @@ public class InternalStepRunner implements StepRunner {
                 return Optional.empty();
             case FAILURE:
                 logger.log("Tests failed.");
+                controller.jobController().updateTestReport(id);
                 return Optional.of(testFailure);
             case ERROR:
                 logger.log(INFO, "Tester failed running its tests!");
                 return Optional.of(error);
             case SUCCESS:
                 logger.log("Tests completed successfully.");
+                controller.jobController().updateTestReport(id);
                 return Optional.of(running);
             default:
                 throw new IllegalStateException("Unknown status '" + testStatus + "'!");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index ac19b5dd5e0..5fcb3f30ae9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
@@ -190,6 +191,21 @@ public class JobController {
         });
     }
 
+    public void updateTestReport(RunId id) {
+        locked(id, run -> {
+            Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone(controller.system())));
+            if (report.isEmpty()) {
+                return run;
+            }
+            logs.writeTestReport(id, report.get());
+            return run;
+        });
+    }
+
+    public Optional<String> getTestReport(RunId id) {
+        return logs.readTestReport(id);
+    }
+
     /** Stores the given certificate as the tester certificate for this run, or throws if it's already set. */
     public void storeTesterCertificate(RunId id, X509Certificate testerCertificate) {
         locked(id, run -> run.with(testerCertificate));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
index d17232c3704..5f36499426b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.deployment.RunLog;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 
@@ -125,4 +126,11 @@ public class BufferedLogStore {
                                       after);
     }
 
+    public Optional<String> readTestReport(RunId id) {
+        return store.getTestReport(id).map(String::new);
+    }
+
+    public void writeTestReport(RunId id, TestReport report) {
+        store.putTestReport(id, report.toJson().getBytes());
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 2f0a462c69f..afe2f5684e5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -9,6 +9,7 @@ import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.NotExistsException;
@@ -189,6 +190,12 @@ class JobControllerApiHandlerHelper {
             });
         });
 
+        // If a test report is available, include it in the response.
+        Optional<String> testReport = jobController.getTestReport(runId);
+        testReport.map(SlimeUtils::jsonToSlime)
+                .map(Slime::get)
+                .ifPresent(reportCursor -> SlimeUtils.copyObject(reportCursor, detailsObject.setObject("testReport")));
+
         return new SlimeJsonResponse(slime);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index ec513e770fa..1dd3b4a7a47 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -31,10 +31,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
@@ -59,7 +59,6 @@ import java.util.UUID;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
-import org.json.JSONObject;
 
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
 import static com.yahoo.config.provision.NodeResources.StorageType.remote;
@@ -90,6 +89,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private RuntimeException prepareException = null;
     private ConfigChangeActions configChangeActions = null;
     private String log = "INFO - All good";
+    private Map<DeploymentId, TestReport> testReport = new HashMap<>();
 
     @Inject
     public ConfigServerMock(ZoneRegistryMock zoneRegistry) {
@@ -326,6 +326,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return false;
     }
 
+    @Override
+    public Optional<TestReport> getTestReport(DeploymentId deployment) {
+        return Optional.ofNullable(testReport.get(deployment));
+    }
+    public void setTestReport(DeploymentId deploymentId, TestReport report) {
+        testReport.put(deploymentId, report);
+    }
+
     /** Add any of given loadBalancers that do not already exist to the load balancers in zone */
     public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
         this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 17d234b0c0c..3d583f12a1c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -61,10 +62,15 @@ public class JobControllerApiHandlerHelperTest {
         var app = tester.newDeploymentContext();
         tester.clock().setInstant(Instant.EPOCH);
 
+        // All completed runs will have a test report.
+        tester.cloud().testReport(TestReport.fromJson("{\"summary\":{\"success\": 1, \"failed\": 0}}"));
+
         // Revision 1 gets deployed everywhere.
         app.submit(applicationPackage).deploy();
         ApplicationVersion revision1 = app.lastSubmission().get();
         assertEquals(1000, tester.application().projectId().getAsLong());
+        // System test includes test report
+        assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), systemTest).get().id(), "0"), "system-test-log.json");
 
         tester.clock().advance(Duration.ofMillis(1000));
         // Revision 2 gets deployed everywhere except in us-east-3.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json
new file mode 100644
index 00000000000..6bf4173a367
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json
@@ -0,0 +1,321 @@
+{
+  "active": false,
+  "status": "success",
+  "log": {
+    "deployTester": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "No services requiring restart."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Deployment successful."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "foo"
+      }
+    ],
+    "installTester": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Tester container successfully installed!"
+      }
+    ],
+    "deployReal": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "No services requiring restart."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Deployment successful."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "foo"
+      }
+    ],
+    "installReal": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "######## Details for all nodes ########"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "host-tenant:application:default-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- platform dockerImage:6.1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Found endpoints:"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "- test.us-east-1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Installation succeeded!"
+      }
+    ],
+    "startTests": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Attempting to find endpoints ..."
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Found endpoints:"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "- test.us-east-1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Starting tests ..."
+      }
+    ],
+    "endTests": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Tests completed successfully."
+      }
+    ],
+    "deactivateReal": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Deactivating deployment of tenant.application in test.us-east-1 ..."
+      }
+    ],
+    "deactivateTester": [
+      {
+        "at": 0,
+        "type": "info",
+        "message": "Deactivating tester of tenant.application in test.us-east-1 ..."
+      }
+    ]
+  },
+  "lastId": 50,
+  "steps": {
+    "deployTester": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "installTester": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "deployReal": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "installReal": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "startTests": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "endTests": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "copyVespaLogs": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "deactivateReal": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "deactivateTester": {
+      "status": "succeeded",
+      "startMillis": 0
+    },
+    "report": {
+      "status": "succeeded",
+      "startMillis": 0
+    }
+  },
+  "testReport": {
+    "summary": {
+      "success": 1,
+      "failed": 0
+    }
+  }
+}
\ No newline at end of file