Merge pull request #12407 from vespa-engine/jvenstad/integration-tests-with-new-deploy-path

Allow unknown packages in manual prod deployment jobs in CD

13 files changed, 322 insertions, 118 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 3fad76b2e07..bc31a0a02fa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -37,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -286,9 +287,65 @@ public class ApplicationController {
         return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options);
     }
 
-    /** Deploys an application. If the application does not exist it is created. */
-    // TODO: Get rid of the options arg
-    // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter.
+    /** Deploys an application package for an existing application instance. */
+    public ActivateResult deploy2(JobId job, boolean deploySourceVersions) { // TODO jonmv: make it number one!
+        if (job.application().instance().isTester())
+            throw new IllegalArgumentException("'" + job.application() + "' is a tester application!");
+
+        TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application());
+        ZoneId zone = job.type().zone(controller.system());
+
+        try (Lock deploymentLock = lockForDeployment(job.application(), zone)) {
+            Set<ContainerEndpoint> endpoints;
+            Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+
+            Run run = controller.jobController().last(job)
+                                .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'"));
+
+            if (run.hasEnded())
+                throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running");
+
+            Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform());
+            ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication());
+            ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision);
+
+            try (Lock lock = lock(applicationId)) {
+                LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
+                Instance instance = application.get().require(job.application().instance());
+
+                Deployment deployment = instance.deployments().get(zone);
+                if (   zone.environment().isProduction() && deployment != null
+                    && (   platform.compareTo(deployment.version()) < 0 && ! instance.change().isPinned()
+                        || revision.compareTo(deployment.applicationVersion()) < 0 && ! (revision.isUnknown() && controller.system().isCd())))
+                    throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" +
+                                                                     " are older than the currently deployed (platform: %s, application: %s).",
+                                                                     job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion()));
+
+                if (   ! applicationPackage.trustedCertificates().isEmpty()
+                    &&   run.testerCertificate().isPresent())
+                    applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get());
+
+                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone);
+
+                endpoints = controller.routing().registerEndpointsInDns(applicationPackage.deploymentSpec(), instance, zone);
+            } // Release application lock while doing the deployment, which is a lengthy task.
+
+            // Carry out deployment without holding the application lock.
+            ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata);
+
+            lockApplicationOrThrow(applicationId, application ->
+                    store(application.with(job.application().instance(),
+                                           instance -> instance.withNewDeployment(zone, revision, platform,
+                                                                                  clock.instant(), warningsFrom(result)))));
+            return result;
+        }
+    }
+
+    private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) {
+        return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone)
+                                                           : applicationStore.get(application.tenant(), application.application(), revision));
+    }
+
     public ActivateResult deploy(ApplicationId instanceId, ZoneId zone,
                                  Optional<ApplicationPackage> applicationPackageFromDeployer,
                                  Optional<ApplicationVersion> applicationVersionFromDeployer,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 861c8089ed5..dc7fdeb8a98 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -115,6 +115,7 @@ public class InternalStepRunner implements StepRunner {
     static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
             new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any);
 
+    static final Duration capacityTimeout = Duration.ofMinutes(5);
     static final Duration endpointTimeout = Duration.ofMinutes(15);
     static final Duration endpointCertificateTimeout = Duration.ofMinutes(15);
     static final Duration testerTimeout = Duration.ofMinutes(30);
@@ -173,34 +174,20 @@ public class InternalStepRunner implements StepRunner {
                    versions.sourcePlatform().orElse(versions.targetPlatform()) +
                    " and application version " +
                    versions.sourceApplication().orElse(versions.targetApplication()).id() + " ...");
-        return deployReal(id, true, versions, logger);
+        return deployReal(id, true, logger);
     }
 
     private Optional<RunStatus> deployReal(RunId id, DualLogger logger) {
         Versions versions = controller.jobController().run(id).get().versions();
         logger.log("Deploying platform version " + versions.targetPlatform() +
                          " and application version " + versions.targetApplication().id() + " ...");
-        return deployReal(id, false, versions, logger);
+        return deployReal(id, false, logger);
     }
 
-    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, Versions versions, DualLogger logger) {
-        Optional<ApplicationPackage> applicationPackage = id.type().environment().isManuallyDeployed()
-                ? Optional.of(new ApplicationPackage(controller.applications().applicationStore()
-                                                               .getDev(id.application(), id.type().zone(controller.system()))))
-                : Optional.empty();
-
-        Optional<Version> vespaVersion = id.type().environment().isManuallyDeployed()
-                ? Optional.of(versions.targetPlatform()) // TODO jonmv: This makes it impossible to deploy on older majors — fix.
-                : Optional.empty();
+    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) {
         return deploy(id.application(),
                       id.type(),
-                      () -> controller.applications().deploy(id.application(),
-                                                             id.type().zone(controller.system()),
-                                                             applicationPackage,
-                                                             new DeployOptions(false,
-                                                                               vespaVersion,
-                                                                               false,
-                                                                               setTheStage)),
+                      () -> controller.applications().deploy2(id.job(), setTheStage),
                       controller.jobController().run(id).get()
                                 .stepInfo(setTheStage ? deployInitialReal : deployReal).get()
                                 .startTime().get(),
@@ -289,7 +276,9 @@ public class InternalStepRunner implements StepRunner {
                     return result;
                 case OUT_OF_CAPACITY:
                     logger.log(e.getServerMessage());
-                    return Optional.of(outOfCapacity);
+                    return controller.system().isCd() && startTime.plus(capacityTimeout).isAfter(controller.clock().instant())
+                           ? Optional.empty()
+                           : Optional.of(outOfCapacity);
                 case INVALID_APPLICATION_PACKAGE:
                 case BAD_REQUEST:
                     logger.log(e.getMessage());
@@ -297,7 +286,8 @@ public class InternalStepRunner implements StepRunner {
             }
 
             throw e;
-        } catch (EndpointCertificateException e) {
+        }
+        catch (EndpointCertificateException e) {
             switch (e.type()) {
                 case CERT_NOT_AVAILABLE:
                     // Same as CERTIFICATE_NOT_READY above, only from the controller
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 007654b2a7c..5d8ad6594df 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -418,9 +418,6 @@ public class JobController {
 
     /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
     public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) {
-        if (profile != JobProfile.development && versions.targetApplication().isUnknown())
-            throw new IllegalArgumentException(" Target application must be a valid reference");
-
         controller.applications().lockApplicationIfPresent(TenantAndApplicationId.from(id), application -> {
             locked(id, type, __ -> {
                 Optional<Run> last = last(id, type);
@@ -436,9 +433,6 @@ public class JobController {
 
     /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */
     public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) {
-        if ( ! type.environment().isManuallyDeployed())
-            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
-
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             if ( ! application.get().instances().containsKey(id.instance()))
                 application = controller.applications().withNewInstance(application, id);
@@ -546,6 +540,7 @@ public class JobController {
             application.get().productionDeployments().values().stream()
                        .flatMap(List::stream)
                        .map(Deployment::applicationVersion)
+                       .filter(version -> ! version.isUnknown())
                        .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong()))
                        .ifPresent(oldestDeployed -> {
                            controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 1c8279f6d8d..7b2e1ad7a8e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1482,6 +1482,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) {
+        if ( ! type.environment().isManuallyDeployed() && ! isOperator(request))
+            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
+
         Map<String, byte[]> dataParts = parseDataParts(request);
         if ( ! dataParts.containsKey("applicationZip"))
             throw new IllegalArgumentException("Missing required form part 'applicationZip'");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index a1c8ccffaf0..e684d479158 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -3,13 +3,19 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
@@ -22,11 +28,14 @@ import java.util.Optional;
 import java.util.OptionalLong;
 import java.util.stream.Collectors;
 
+import static com.yahoo.config.provision.SystemName.cd;
 import static com.yahoo.config.provision.SystemName.main;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3;
@@ -57,7 +66,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class DeploymentTriggerTest {
 
-    private final DeploymentTester tester = new DeploymentTester();
+    private DeploymentTester tester = new DeploymentTester();
 
     @Test
     public void testTriggerFailing() {
@@ -1066,4 +1075,58 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(app.lastSubmission().get()), app.instance().change());
     }
 
+    @Test
+    public void mixedDirectAndPipelineJobsInProduction() {
+        ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1")
+                                                                      .region("cd-aws-us-east-1a")
+                                                                      .build();
+        ServiceRegistryMock services = new ServiceRegistryMock();
+        var zones = List.of(ZoneApiMock.fromId("test.cd-us-central-1"),
+                            ZoneApiMock.fromId("staging.cd-us-central-1"),
+                            ZoneApiMock.fromId("prod.cd-us-central-1"),
+                            ZoneApiMock.fromId("prod.cd-aws-us-east-1a"));
+        services.zoneRegistry()
+                .setSystemName(SystemName.cd)
+                .setZones(zones)
+                .setRoutingMethod(zones, RoutingMethod.exclusive);
+        tester = new DeploymentTester(new ControllerTester(services));
+        tester.configServer().bootstrap(services.zoneRegistry().zones().all().ids(), SystemApplication.values());
+        tester.controllerTester().upgradeSystem(Version.fromString("6.1"));
+        tester.controllerTester().computeVersionStatus();
+        var app = tester.newDeploymentContext();
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        app.submit(cdPackage);
+        app.runJob(systemTest);
+        // Staging test requires unknown initial version, and is broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .abortJob(stagingTest) // Complete failing run.
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        var version = new Version("7.1");
+        tester.controllerTester().upgradeSystem(version);
+        tester.upgrader().maintain();
+        // System and staging tests both require unknown versions, and are broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .abortJob(systemTest)
+           .abortJob(stagingTest)
+           .runJob(systemTest)
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        app.submit(cdPackage);
+        app.runJob(systemTest);
+        // Staging test requires unknown initial version, and is broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .jobAborted(stagingTest)
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index e611881e791..a57e302d939 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -371,12 +371,6 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(1, tester.jobs().active().size());
         assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version());
-
-        try {
-            tester.jobs().deploy(app.instanceId(), JobType.productionApNortheast1, Optional.empty(), applicationPackage);
-            fail("Deployments outside dev should not be allowed.");
-        }
-        catch (IllegalArgumentException expected) { }
     }
 
     @Test
@@ -420,8 +414,8 @@ public class InternalStepRunnerTest {
     public void certificateTimeoutAbortsJob() {
         tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public);
         var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
-                                      ZoneApiMock.fromId("staging.aws-us-east-1c"),
-                                      ZoneApiMock.fromId("prod.aws-us-east-1c"));
+                            ZoneApiMock.fromId("staging.aws-us-east-1c"),
+                            ZoneApiMock.fromId("prod.aws-us-east-1c"));
         tester.controllerTester().zoneRegistry()
               .setZones(zones)
               .setRoutingMethod(zones, RoutingMethod.exclusive);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 45d87c6756a..810b9c2550c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -257,8 +257,22 @@ public class ApplicationApiTest extends ControllerContainerTest {
         ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1");
         var app1 = deploymentTester.newDeploymentContext(id);
 
-        // POST (deploy) an application to start a manual deployment to dev
+        // POST (deploy) an application to start a manual deployment in prod is not allowed
         MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true);
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST)
+                                      .data(entity)
+                                      .userIdentity(USER_ID),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Direct deployments are only allowed to manually deployed environments.\"}", 400);
+
+        // POST (deploy) an application to start a manual deployment in prod is allowed for operators
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST)
+                                      .data(entity)
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
+                              "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}");
+        app1.runJob(JobType.productionUsEast3);
+        tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3"));
+
+        // POST (deploy) an application to start a manual deployment to dev
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST)
                                       .data(entity)
                                       .userIdentity(USER_ID),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
index 282c18046d3..fe9d0573ed2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
@@ -464,8 +464,8 @@
       ],
       "runs": [
         {
-          "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1",
+          "id": 2,
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2",
           "start": "(ignore)",
           "status": "aborted",
           "versions": {
@@ -491,6 +491,31 @@
               "status": "unfinished"
             }
           ]
+        },
+        {
+          "id": 1,
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1",
+          "start": "(ignore)",
+          "end": "(ignore)",
+          "status": "success",
+          "versions": {
+            "targetPlatform": "6.1.0",
+            "targetApplication": {}
+          },
+          "steps": [
+            {
+              "name": "deployReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "installReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "copyVespaLogs",
+              "status": "succeeded"
+            }
+          ]
         }
       ]
     },
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
index fd8bc256ac5..3d1c5ade300 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
@@ -13,8 +13,8 @@
   "projectId": 123,
   "deploying": {
     "revision": {
-      "buildNumber": "(ignore)",
-      "hash": "(ignore)",
+      "buildNumber": 1,
+      "hash": "1.0.1-commit1",
       "source": {
         "gitRepository": "repository1",
         "gitBranch": "master",
@@ -30,10 +30,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -47,10 +47,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -64,10 +64,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -85,10 +85,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -102,10 +102,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -119,10 +119,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -140,10 +140,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -157,10 +157,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -174,10 +174,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -192,13 +192,13 @@
     },
     {
       "type": "production-us-east-3",
-      "success": false,
+      "success": true,
       "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
+        "id": 2,
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -209,6 +209,18 @@
         },
         "reason": "unknown reason",
         "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "id": 1,
+        "version": "6.1.0",
+        "reason": "unknown reason",
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "id": 1,
+        "version": "6.1.0",
+        "reason": "unknown reason",
+        "at": "(ignore)"
       }
     },
     {
@@ -263,7 +275,7 @@
           "rotationId": "rotation-id-1",
           "clusterId": "foo",
           "status": "IN",
-          "lastUpdated":"(ignore)"
+          "lastUpdated": "(ignore)"
         }
       ],
       "environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
index ee75d129241..1b7e7893222 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
@@ -13,8 +13,8 @@
   "projectId": 123,
   "deploying": {
     "revision": {
-      "buildNumber": "(ignore)",
-      "hash": "(ignore)",
+      "buildNumber": 1,
+      "hash": "1.0.1-commit1",
       "source": {
         "gitRepository": "repository1",
         "gitBranch": "master",
@@ -30,10 +30,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -47,10 +47,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -64,10 +64,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -85,10 +85,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -102,10 +102,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -119,10 +119,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -140,10 +140,10 @@
       "success": true,
       "lastTriggered": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -157,10 +157,10 @@
       },
       "lastCompleted": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -174,10 +174,10 @@
       },
       "lastSuccess": {
         "id": 1,
-        "version": "(ignore)",
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -192,13 +192,13 @@
     },
     {
       "type": "production-us-east-3",
-      "success": false,
+      "success": true,
       "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
+        "id": 2,
+        "version": "6.1.0",
         "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
+          "buildNumber": 1,
+          "hash": "1.0.1-commit1",
           "source": {
             "gitRepository": "repository1",
             "gitBranch": "master",
@@ -209,6 +209,18 @@
         },
         "reason": "unknown reason",
         "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "id": 1,
+        "version": "6.1.0",
+        "reason": "unknown reason",
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "id": 1,
+        "version": "6.1.0",
+        "reason": "unknown reason",
+        "at": "(ignore)"
       }
     },
     {
@@ -241,7 +253,7 @@
       ]
     }
   ],
-  "compileVersion": "(ignore)",
+  "compileVersion": "6.0.0",
   "globalRotations": [
     "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"
   ],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
index 8cd102432d0..b16ca4cc67c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
@@ -275,7 +275,7 @@
     "us-east-3": {
       "runs": [
         {
-          "id": 1,
+          "id": 2,
           "status": "aborted",
           "start": "(ignore)",
           "wantedPlatform": "6.1",
@@ -296,6 +296,26 @@
             "report": "unfinished"
           },
           "tasks": {},
+          "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2"
+        },
+        {
+          "id": 1,
+          "status": "success",
+          "start": "(ignore)",
+          "end": "(ignore)",
+          "wantedPlatform": "6.1",
+          "wantedApplication": {
+            "hash": "unknown"
+          },
+          "steps": {
+            "deployReal": "succeeded",
+            "installReal": "succeeded",
+            "copyVespaLogs": "succeeded"
+          },
+          "tasks": {
+            "deploy": "succeeded",
+            "install": "succeeded"
+          },
           "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1"
         }
       ],
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
index 6fba083e607..79c11cf86de 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
@@ -17,6 +17,7 @@ import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 
+import javax.net.ssl.SSLContext;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -77,6 +78,11 @@ public abstract class ControllerHttpClient {
         return new SigningControllerHttpClient(endpoint, privateKeyFile, id);
     }
 
+    /** Creates an HTTP client against the given endpoint, which uses the given SSL context for authentication. */
+    public static ControllerHttpClient withSSLContext(URI endpoint, SSLContext sslContext) {
+        return new MutualTlsControllerHttpClient(endpoint, sslContext);
+    }
+
     /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */
     public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) {
         var privateKey = unchecked(() -> KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKeyFile, UTF_8)));
@@ -150,11 +156,12 @@ public abstract class ControllerHttpClient {
     public DeploymentLog followDeploymentUntilDone(ApplicationId id, ZoneId zone, long run,
                                                    Consumer<DeploymentLog.Entry> out) {
         long last = -1;
-        DeploymentLog log;
+        DeploymentLog log = null;
         while (true) {
-            log = deploymentLog(id, zone, run, last);
-            for (DeploymentLog.Entry entry : log.entries())
+            DeploymentLog update = deploymentLog(id, zone, run, last);
+            for (DeploymentLog.Entry entry : update.entries())
                 out.accept(entry);
+            log = (log == null ? update : log.updatedWith(update));
             last = log.last().orElse(last);
 
             if ( ! log.isActive())
@@ -410,14 +417,17 @@ public abstract class ControllerHttpClient {
     /** Client that uses a given key / certificate identity to authenticate to the remote controller. */
     private static class MutualTlsControllerHttpClient extends ControllerHttpClient {
 
+        private MutualTlsControllerHttpClient(URI endpoint, SSLContext sslContext) {
+            super(endpoint, HttpClient.newBuilder().sslContext(sslContext));
+        }
+
         private MutualTlsControllerHttpClient(URI endpoint, PrivateKey privateKey, List<X509Certificate> certs) {
-            super(endpoint,
-                  HttpClient.newBuilder()
-                            .sslContext(new SslContextBuilder().withKeyStore(privateKey, certs).build()));
+            this(endpoint, new SslContextBuilder().withKeyStore(privateKey, certs).build());
         }
 
     }
 
+
     private static DeploymentLog.Status valueOf(String status) {
         switch (status) {
             case "running":             return DeploymentLog.Status.running;
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
index 177c72107e0..9eae9a33cff 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
@@ -4,6 +4,7 @@ package ai.vespa.hosted.api;
 import java.time.Instant;
 import java.util.List;
 import java.util.OptionalLong;
+import java.util.stream.Stream;
 
 import static java.util.Comparator.comparing;
 import static java.util.stream.Collectors.toUnmodifiableList;
@@ -27,6 +28,14 @@ public class DeploymentLog {
         this.last = last;
     }
 
+    /** Returns this log updated with the content of the other. */
+    public DeploymentLog updatedWith(DeploymentLog other) {
+        return new DeploymentLog(Stream.concat(entries.stream(), other.entries.stream()).collect(toUnmodifiableList()),
+                                 other.active,
+                                 other.status,
+                                 other.last);
+    }
+
     public List<Entry> entries() {
         return entries;
     }