Merge pull request #10532 from vespa-engine/jvenstad/tester-app-with-cert

Jvenstad/tester app with cert

24 files changed, 408 insertions, 135 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index cbf90db84f9..06baa0a7720 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -59,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
 import com.yahoo.vespa.hosted.controller.concurrent.Once;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
 import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -318,10 +319,11 @@ public class ApplicationController {
                                                              : triggered.application();
 
                     applicationPackage = getApplicationPackage(application.get(), applicationVersion);
+                    applicationPackage = withTesterCertificate(applicationPackage, applicationId, jobType);
                     validateRun(application.get(), zone, platformVersion, applicationVersion);
                 }
 
-                // TODO(jvenstad): Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...).
+                // TODO jonmv: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...).
                 verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity);
 
                 // Assign and register endpoints
@@ -356,6 +358,19 @@ public class ApplicationController {
         }
     }
 
+    private ApplicationPackage withTesterCertificate(ApplicationPackage applicationPackage, ApplicationId id, JobType type) {
+        if (applicationPackage.trustedCertificates().isEmpty())
+            return applicationPackage;
+
+        // TODO jonmv: move this to the caller, when external build service is removed.
+        Run run = controller.jobController().last(id, type)
+                            .orElseThrow(() -> new IllegalStateException("Last run of " + type + " for " + id + " not found"));
+        if (run.testerCertificate().isEmpty())
+            return applicationPackage;
+
+        return applicationPackage.withTrustedCertificate(run.testerCertificate().get());
+    }
+
     /** Fetches the requested application package from the artifact store(s). */
     public ApplicationPackage getApplicationPackage(Application application, ApplicationVersion version) {
         try {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
index 78500d26865..f37c9d5a394 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
@@ -6,20 +6,27 @@ import com.google.common.hash.Hashing;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UncheckedIOException;
+import java.security.cert.X509Certificate;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * A representation of the content of an application package.
  * Only the deployment.xml content can be accessed as anything other than compressed data.
@@ -31,12 +38,15 @@ import java.util.Set;
  */
 public class ApplicationPackage {
 
+    private static final String trustedCertificatesFile = "security/clients.pem";
+
     private final String contentHash;
     private final byte[] zippedContent;
     private final DeploymentSpec deploymentSpec;
     private final ValidationOverrides validationOverrides;
     private final Optional<Version> compileVersion;
     private final Optional<Instant> buildTime;
+    private final List<X509Certificate> trustedCertificates;
     
     /** 
      * Creates an application package from its zipped content.
@@ -47,14 +57,26 @@ public class ApplicationPackage {
         this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null");
         this.contentHash = Hashing.sha1().hashBytes(zippedContent).toString();
 
-        Files files = Files.extract(Set.of("deployment.xml", "validation-overrides.xml", "build-meta.json"), zippedContent);
+        Files files = Files.extract(Set.of("deployment.xml", "validation-overrides.xml", "build-meta.json", trustedCertificatesFile), zippedContent);
         this.deploymentSpec = files.getAsReader("deployment.xml").map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty);
         this.validationOverrides = files.getAsReader("validation-overrides.xml").map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty);
         Optional<Inspector> buildMetaObject = files.get("build-meta.json").map(SlimeUtils::jsonToSlime).map(Slime::get);
         this.compileVersion = buildMetaObject.map(object -> Version.fromString(object.field("compileVersion").asString()));
         this.buildTime = buildMetaObject.map(object -> Instant.ofEpochMilli(object.field("buildTime").asLong()));
+        this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of());
     }
-    
+
+    /** Returns a copy of this with the given certificate appended. */
+    public ApplicationPackage withTrustedCertificate(X509Certificate certificate) {
+        List<X509Certificate> trustedCertificates = new ArrayList<>(this.trustedCertificates);
+        trustedCertificates.add(certificate);
+        byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8);
+
+        ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length);
+        ZipStreamReader.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes);
+        return new ApplicationPackage(modified.toByteArray());
+    }
+
     /** Returns a hash of the content of this package */
     public String hash() { return contentHash; }
     
@@ -79,6 +101,10 @@ public class ApplicationPackage {
     /** Returns the time this package was built, if known. */
     public Optional<Instant> buildTime() { return buildTime; }
 
+    /** Returns the list of certificates trusted by this application, or an empty list if no trust configured. */
+    public List<X509Certificate> trustedCertificates() {
+        return trustedCertificates;
+    }
 
     private static class Files {
 
@@ -109,6 +135,7 @@ public class ApplicationPackage {
             return new Files(builder.build());
         }
 
+
         /** Get content of given file name */
         public Optional<byte[]> get(String name) {
             return Optional.ofNullable(files.get(name));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java
index 578f5c98d8e..2322b251fe0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java
@@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableList;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -13,6 +14,7 @@ import java.util.List;
 import java.util.function.Predicate;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
 
 /**
  * @author bratseth
@@ -37,6 +39,27 @@ public class ZipStreamReader {
         }
     }
 
+    /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */
+    public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) {
+        try (ZipOutputStream zipOut = new ZipOutputStream(out);
+             ZipInputStream zipIn = new ZipInputStream(in)) {
+            for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) {
+                if (entry.getName().equals(name))
+                    continue;
+
+                zipOut.putNextEntry(entry);
+                zipIn.transferTo(zipOut);
+                zipOut.closeEntry();
+            }
+            zipOut.putNextEntry(new ZipEntry(name));
+            zipOut.write(content);
+            zipOut.closeEntry();
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     private byte[] readContent(ZipInputStream zipInput) {
         try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
             byte[] buffer = new byte[2048];
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 8aa640a6882..994fe10be0e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -15,6 +15,11 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
@@ -36,14 +41,20 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
 import com.yahoo.yolean.Exceptions;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.io.UncheckedIOException;
+import java.math.BigInteger;
 import java.net.URI;
-import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -74,6 +85,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.WARNING;
 
@@ -93,6 +105,7 @@ public class InternalStepRunner implements StepRunner {
 
     static final Duration endpointTimeout = Duration.ofMinutes(15);
     static final Duration installationTimeout = Duration.ofMinutes(150);
+    static final Duration certificateTimeout = Duration.ofMinutes(300);
 
     private final Controller controller;
     private final TestConfigSerializer testConfigSerializer;
@@ -109,12 +122,12 @@ public class InternalStepRunner implements StepRunner {
         DualLogger logger = new DualLogger(id, step.get());
         try {
             switch (step.get()) {
+                case deployTester: return deployTester(id, logger);
                 case deployInitialReal: return deployInitialReal(id, logger);
                 case installInitialReal: return installInitialReal(id, logger);
                 case deployReal: return deployReal(id, logger);
-                case deployTester: return deployTester(id, logger);
-                case installReal: return installReal(id, logger);
                 case installTester: return installTester(id, logger);
+                case installReal: return installReal(id, logger);
                 case startTests: return startTests(id, logger);
                 case endTests: return endTests(id, logger);
                 case copyVespaLogs: return copyVespaLogs(id, logger);
@@ -159,6 +172,7 @@ public class InternalStepRunner implements StepRunner {
                 ? Optional.of(new ApplicationPackage(controller.applications().applicationStore()
                                                                .getDev(id.application(), id.type().zone(controller.system()))))
                 : Optional.empty();
+
         Optional<Version> vespaVersion = id.type().environment().isManuallyDeployed()
                 ? Optional.of(versions.targetPlatform())
                 : Optional.empty();
@@ -428,6 +442,17 @@ public class InternalStepRunner implements StepRunner {
             return Optional.of(aborted);
         }
 
+        Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate();
+        if (testerCertificate.isPresent()) {
+            try {
+                testerCertificate.get().checkValidity(Date.from(controller.clock().instant()));
+            }
+            catch (CertificateExpiredException | CertificateNotYetValidException e) {
+                logger.log(INFO, "Tester certificate expired before tests could complete.");
+                return Optional.of(aborted);
+            }
+        };
+
         Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
         if ( ! testerEndpoint.isPresent()) {
             logger.log("Endpoints for tester not found -- trying again later.");
@@ -464,7 +489,7 @@ public class InternalStepRunner implements StepRunner {
                 List<LogEntry> entries = new ArrayList<>();
                 String logs = IOUtils.readAll(controller.serviceRegistry().configServer().getLogs(new DeploymentId(id.application(), zone),
                                                                                      Collections.emptyMap()), // Get all logs.
-                                              StandardCharsets.UTF_8);
+                                              UTF_8);
                 for (String line : logs.split("\n")) {
                     String[] parts = line.split("\t");
                     if (parts.length != 7) continue;
@@ -591,8 +616,10 @@ public class InternalStepRunner implements StepRunner {
         ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication();
         DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec();
 
+        boolean useTesterCertificate = controller.system().isPublic() && id.type().isTest();
         byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(),
                                          spec.athenzDomain().isPresent(),
+                                         useTesterCertificate,
                                          testerFlavorFor(id, spec));
         byte[] testPackage = controller.applications().applicationStore().get(id.tester(), version);
 
@@ -603,11 +630,29 @@ public class InternalStepRunner implements StepRunner {
             zipBuilder.add(testPackage);
             zipBuilder.add("services.xml", servicesXml);
             zipBuilder.add("deployment.xml", deploymentXml);
+            if (useTesterCertificate)
+                appendAndStoreCertificate(zipBuilder, id);
+
             zipBuilder.close();
             return new ApplicationPackage(zipBuilder.toByteArray());
         }
     }
 
+    private void appendAndStoreCertificate(ZipBuilder zipBuilder, RunId id) {
+        KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+        X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number());
+        X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair,
+                                                                         subject,
+                                                                         controller.clock().instant(),
+                                                                         controller.clock().instant().plus(certificateTimeout),
+                                                                         SignatureAlgorithm.SHA512_WITH_ECDSA,
+                                                                         BigInteger.valueOf(1))
+                                                            .build();
+        controller.jobController().storeTesterCertificate(id, certificate);
+        zipBuilder.add("key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8));
+        zipBuilder.add("cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8));
+    }
+
     private static Optional<String> testerFlavorFor(RunId id, DeploymentSpec spec) {
         for (DeploymentSpec.Step step : spec.steps())
             if (step.deploysTo(id.type().environment()))
@@ -625,7 +670,8 @@ public class InternalStepRunner implements StepRunner {
     }
 
     /** Returns the generated services.xml content for the tester application. */
-    static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, Optional<String> testerFlavor) {
+    static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, boolean useTesterCertificate,
+                              Optional<String> testerFlavor) {
         String flavor = testerFlavor.orElse("d-1-4-50");
         int memoryGb = Integer.parseInt(flavor.split("-")[2]); // Memory available in tester container.
         int jdiscMemoryPercentage = (int) Math.ceil(200.0 / memoryGb); // 2Gb memory for tester application (excessive?).
@@ -641,6 +687,7 @@ public class InternalStepRunner implements StepRunner {
                 "                <artifactsPath>artifacts</artifactsPath>\n" +
                 "                <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" +
                 "                <useAthenzCredentials>" + useAthenzCredentials + "</useAthenzCredentials>\n" +
+                "                <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" +
                 "            </config>\n" +
                 "        </component>\n" +
                 "\n" +
@@ -677,7 +724,7 @@ public class InternalStepRunner implements StepRunner {
                 "    </container>\n" +
                 "</services>\n";
 
-        return servicesXml.getBytes(StandardCharsets.UTF_8);
+        return servicesXml.getBytes(UTF_8);
     }
 
     /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */
@@ -688,7 +735,7 @@ public class InternalStepRunner implements StepRunner {
                 athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") +
                 athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("")
                 + "/>";
-        return deploymentSpec.getBytes(StandardCharsets.UTF_8);
+        return deploymentSpec.getBytes(UTF_8);
     }
 
     /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 3024c36c991..2f8c08c15be 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.net.URI;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -154,6 +155,11 @@ public class JobController {
             });
     }
 
+    /** Stores the given certificate as the tester certificate for this run, or throws if it's already set. */
+    public void storeTesterCertificate(RunId id, X509Certificate testerCertificate) {
+        locked(id, run -> run.with(testerCertificate));
+    }
+
     /** Returns a list of all application which have registered. */
     public List<ApplicationId> applications() {
         return copyOf(controller.applications().asList().stream()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index f052c7d91ab..94cf0343776 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 
+import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.EnumMap;
@@ -33,10 +34,11 @@ public class Run {
     private final Optional<Instant> end;
     private final RunStatus status;
     private final long lastTestRecord;
+    private final Optional<X509Certificate> testerCertificate;
 
     // For deserialisation only -- do not use!
     public Run(RunId id, Map<Step, Step.Status> steps, Versions versions, Instant start,
-               Optional<Instant> end, RunStatus status, long lastTestRecord) {
+               Optional<Instant> end, RunStatus status, long lastTestRecord, Optional<X509Certificate> testerCertificate) {
         this.id = id;
         this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
         this.versions = versions;
@@ -44,47 +46,57 @@ public class Run {
         this.end = end;
         this.status = status;
         this.lastTestRecord = lastTestRecord;
+        this.testerCertificate = testerCertificate;
     }
 
     public static Run initial(RunId id, Versions versions, Instant now) {
         EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class);
         JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished));
-        return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1);
+        return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1, Optional.empty());
     }
 
     /** Returns a new Run with the new status, and with the status of the given, completed step set accordingly. */
     public Run with(RunStatus status, LockedStep step) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be further modified!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
 
         if (steps.get(step.get()) != unfinished)
-            throw new AssertionError("Step '" + step.get() + "' can't be set to '" + status + "'" +
+            throw new IllegalStateException("Step '" + step.get() + "' can't be set to '" + status + "'" +
                                      " -- it already completed with status '" + steps.get(step.get()) + "'!");
 
         EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), Step.Status.of(status));
-        return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, lastTestRecord);
+        return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, lastTestRecord, testerCertificate);
     }
 
     public Run finished(Instant now) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be ended again!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be ended again!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status, lastTestRecord, testerCertificate);
     }
 
     public Run aborted() {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be aborted now!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be aborted now!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, end, aborted, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, end, aborted, lastTestRecord, testerCertificate);
     }
 
     public Run with(long lastTestRecord) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be further modified!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord, testerCertificate);
+    }
+
+    public Run with(X509Certificate testerCertificate) {
+        if (hasEnded())
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
+        if (this.testerCertificate.isPresent())
+            throw new IllegalStateException("Certificate for this run was already set");
+
+        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord, Optional.of(testerCertificate));
     }
 
     /** Returns the id of this run. */
@@ -131,6 +143,11 @@ public class Run {
         return lastTestRecord;
     }
 
+    /** Returns the tester certificate for this run, or empty. */
+    public Optional<X509Certificate> testerCertificate() {
+        return testerCertificate;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index a5f9ef86da4..b37e3a1105d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -25,24 +25,24 @@ import java.util.List;
  */
 public enum Step {
 
+    /** Download test-jar and assemble and deploy tester application. */
+    deployTester,
+
     /** Download and deploy the initial real application, for staging tests. */
-    deployInitialReal,
+    deployInitialReal(deployTester),
 
     /** See that the real application has had its nodes converge to the initial state. */
     installInitialReal(deployInitialReal),
 
     /** Download and deploy real application, restarting services if required. */
-    deployReal(installInitialReal),
-
-    /** See that real application has had its nodes converge to the wanted version and generation. */
-    installReal(deployReal),
-
-    /** Download test-jar and assemble and deploy tester application. */
-    deployTester,
+    deployReal(deployTester, installInitialReal),
 
     /** See that tester is done deploying, and is ready to serve. */
     installTester(deployReal, deployTester),
 
+    /** See that real application has had its nodes converge to the wanted version and generation. */
+    installReal(deployReal),
+
     /** Ask the tester to run its tests. */
     startTests(installReal, installTester),
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 1c95c9766f5..aa7d29585cb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -81,6 +82,7 @@ class RunSerializer {
     private static final String buildField = "build";
     private static final String sourceField = "source";
     private static final String lastTestRecordField = "lastTestRecord";
+    private static final String testerCertificateField = "testerCertificate";
 
     Run runFromSlime(Slime slime) {
         return runFromSlime(slime.get());
@@ -111,7 +113,10 @@ class RunSerializer {
                                .filter(Inspector::valid)
                                .map(end -> Instant.ofEpochMilli(end.asLong())),
                        runStatusOf(runObject.field(statusField).asString()),
-                       runObject.field(lastTestRecordField).asLong());
+                       runObject.field(lastTestRecordField).asLong(),
+                       Optional.of(runObject.field(testerCertificateField))
+                               .filter(Inspector::valid)
+                               .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())));
     }
 
     private Versions versionsFromSlime(Inspector versionsObject) {
@@ -169,6 +174,7 @@ class RunSerializer {
         run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli()));
         runObject.setString(statusField, valueOf(run.status()));
         runObject.setLong(lastTestRecordField, run.lastTestLogEntry());
+        run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate)));
 
         Cursor stepsObject = runObject.setObject(stepsField);
         run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status)));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 10f536498f2..6155191fb8f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -599,7 +599,7 @@ public class ControllerTest {
         tester.controller().applications().deploy(app.id(), zone, Optional.of(applicationPackage), options);
 
         assertTrue("Application deployed and activated",
-                   tester.controllerTester().configServer().application(app.id()).get().activated());
+                   tester.controllerTester().configServer().application(app.id(), zone).get().activated());
 
         assertTrue("No job status added",
                    tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty());
@@ -625,7 +625,7 @@ public class ControllerTest {
         // Deploy
         tester.controller().applications().deploy(app.id(), zone, Optional.of(applicationPackage), DeployOptions.none());
         assertTrue("Application deployed and activated",
-                   tester.controllerTester().configServer().application(app.id()).get().activated());
+                   tester.controllerTester().configServer().application(app.id(), zone).get().activated());
         assertTrue("No job status added",
                    tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty());
         assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, tester.applications().require(app.id()).deploymentSpec());
@@ -721,7 +721,7 @@ public class ControllerTest {
         // Deploy app2 in dev
         tester.controller().applications().deploy(app2.id(), zone, Optional.of(applicationPackage), DeployOptions.none());
         assertTrue("Application deployed and activated",
-                   tester.controllerTester().configServer().application(app2.id()).get().activated());
+                   tester.controllerTester().configServer().application(app2.id(), zone).get().activated());
         assertFalse("Does not provision certificate in " + Environment.dev, certificate.apply(app2).isPresent());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java
index b71dc4da08d..abd234f0fa4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java
@@ -1,14 +1,26 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
 import org.junit.Test;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -61,6 +73,30 @@ public class ZipStreamReaderTest {
         });
     }
 
+    @Test
+    public void test_replacement() {
+        ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]);
+        List<X509Certificate> certificates = IntStream.range(0, 3)
+                                                      .mapToObj(i -> {
+                                                          KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+                                                          X500Principal subject = new X500Principal("CN=subject" + i);
+                                                          return X509CertificateBuilder.fromKeypair(keyPair,
+                                                                                                    subject,
+                                                                                                    Instant.now(),
+                                                                                                    Instant.now().plusSeconds(1),
+                                                                                                    SignatureAlgorithm.SHA512_WITH_ECDSA,
+                                                                                                    BigInteger.valueOf(1))
+                                                                                       .build();
+                                                      })
+                                                      .collect(Collectors.toUnmodifiableList());
+        
+        assertEquals(List.of(), applicationPackage.trustedCertificates());
+        for (int i = 0; i < certificates.size(); i++) {
+            applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i));
+            assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates());
+        }
+    }
+
     private static byte[] zip(Map<String, String> entries) {
         ByteArrayOutputStream zip = new ByteArrayOutputStream();
         try (ZipOutputStream out = new ZipOutputStream(zip)) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 6635547e9be..25e562ed046 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -6,15 +6,18 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
+import java.security.cert.X509Certificate;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.LinkedHashMap;
@@ -25,6 +28,8 @@ import java.util.StringJoiner;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * A builder that builds application packages for testing purposes.
  *
@@ -39,6 +44,7 @@ public class ApplicationPackageBuilder {
                                                                 "<notifications>\n  <email ",
                                                                 "/>\n</notifications>\n").setEmptyValue("");
     private final StringBuilder endpointsBody = new StringBuilder();
+    private final List<X509Certificate> trustedCertificates = new ArrayList<>();
 
     private OptionalInt majorVersion = OptionalInt.empty();
     private String upgradePolicy = null;
@@ -146,7 +152,12 @@ public class ApplicationPackageBuilder {
         this.searchDefinition = testSearchDefinition;
         return this;
     }
-    
+
+    public ApplicationPackageBuilder trust(X509Certificate certificate) {
+        this.trustedCertificates.add(certificate);
+        return this;
+    }
+
     private byte[] deploymentSpec() {
         StringBuilder xml = new StringBuilder();
         xml.append("<deployment version='1.0' ");
@@ -178,22 +189,22 @@ public class ApplicationPackageBuilder {
         xml.append(endpointsBody);
         xml.append("  </endpoints>\n");
         xml.append("</deployment>");
-        return xml.toString().getBytes(StandardCharsets.UTF_8);
+        return xml.toString().getBytes(UTF_8);
     }
     
     private byte[] validationOverrides() {
         String xml = "<validation-overrides version='1.0'>\n" +
                 validationOverridesBody +
                 "</validation-overrides>\n";
-        return xml.getBytes(StandardCharsets.UTF_8);
+        return xml.getBytes(UTF_8);
     }
 
     private byte[] searchDefinition() { 
-        return searchDefinition.getBytes(StandardCharsets.UTF_8);
+        return searchDefinition.getBytes(UTF_8);
     }
 
     private byte[] buildMeta() {
-        return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(StandardCharsets.UTF_8);
+        return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(UTF_8);
     }
 
     public ApplicationPackage build() {
@@ -219,6 +230,9 @@ public class ApplicationPackageBuilder {
             out.putNextEntry(new ZipEntry(dir + "build-meta.json"));
             out.write(buildMeta());
             out.closeEntry();
+            out.putNextEntry(new ZipEntry(dir + "security/clients.pem"));
+            out.write(X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8));
+            out.closeEntry();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
index e5fe658cf3b..0e5bf774441 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
@@ -31,13 +31,13 @@ public class BadgesTest {
 
     private static final ApplicationId id = ApplicationId.from("tenant", "application", "default");
     private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.of(now()), RunStatus.success, 0);
+                                               null, null, Optional.of(now()), RunStatus.success, 0, Optional.empty());
 
     private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.empty(), RunStatus.running, 0);
+                                               null, null, Optional.empty(), RunStatus.running, 0, Optional.empty());
 
     private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.of(now()), RunStatus.testFailure, 0);
+                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, Optional.empty());
 
     @Test
     public void test() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index fdb797a6bcd..2691a152f77 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -5,7 +5,12 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.log.LogLevel;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
@@ -26,10 +31,18 @@ import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest;
 
+import javax.security.auth.x500.X500Principal;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
@@ -51,6 +64,14 @@ public class InternalDeploymentTester {
             .emailRole("author")
             .emailAddress("b@a")
             .build();
+    public static final ApplicationPackage publicCdApplicationPackage = new ApplicationPackageBuilder()
+            .athenzIdentity(AthenzDomain.from(ATHENZ_DOMAIN), AthenzService.from(ATHENZ_SERVICE))
+            .upgradePolicy("default")
+            .region("aws-us-east-1c")
+            .emailRole("author")
+            .emailAddress("b@a")
+            .trust(generateCertificate())
+            .build();
     public static final ApplicationId appId = ApplicationId.from("tenant", "application", "default");
     public static final TesterId testerId = TesterId.of(appId);
     public static final String athenzDomain = "domain";
@@ -98,7 +119,8 @@ public class InternalDeploymentTester {
      * Submits a new application, and returns the version of the new submission.
      */
     public ApplicationVersion newSubmission() {
-        return jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage, new byte[0]);
+        return jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2,
+                           tester.controller().system().isPublic() ? publicCdApplicationPackage : applicationPackage, new byte[0]);
     }
 
     /**
@@ -276,4 +298,16 @@ public class InternalDeploymentTester {
         return run.id();
     }
 
+    static X509Certificate generateCertificate() {
+        KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+        X500Principal subject = new X500Principal("CN=subject");
+        return X509CertificateBuilder.fromKeypair(keyPair,
+                                                  subject,
+                                                  Instant.now(),
+                                                  Instant.now().plusSeconds(1),
+                                                  SignatureAlgorithm.SHA512_WITH_ECDSA,
+                                                  BigInteger.valueOf(1))
+                                     .build();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 1ae00231364..7aa18dba5db 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
@@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -31,7 +33,9 @@ import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -45,6 +49,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.in
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.applicationPackage;
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.publicCdApplicationPackage;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
@@ -68,6 +73,10 @@ public class InternalStepRunnerTest {
         tester = new InternalDeploymentTester();
     }
 
+    private SystemName system() {
+        return tester.tester().controller().system();
+    }
+
     @Test
     public void canRegisterAndRunDirectly() {
         tester.deployNewSubmission();
@@ -79,9 +88,9 @@ public class InternalStepRunnerTest {
     public void canSwitchFromScrewdriverAndBackAgain() {
         // Deploys a default application package with default build number.
         tester.tester().deployCompletely(tester.app(), InternalDeploymentTester.applicationPackage);
-        tester.setEndpoints(appId, JobType.productionUsCentral1.zone(tester.tester().controller().system()));
-        tester.setEndpoints(appId, JobType.productionUsWest1.zone(tester.tester().controller().system()));
-        tester.setEndpoints(appId, JobType.productionUsEast3.zone(tester.tester().controller().system()));
+        tester.setEndpoints(appId, JobType.productionUsCentral1.zone(system()));
+        tester.setEndpoints(appId, JobType.productionUsWest1.zone(system()));
+        tester.setEndpoints(appId, JobType.productionUsEast3.zone(system()));
 
         // Let application have an ongoing upgrade when it switches (but kill the jobs, as the tester assumes they aren't running).
         tester.tester().upgradeSystem(new Version("7.1"));
@@ -105,15 +114,24 @@ public class InternalStepRunnerTest {
     public void testerHasAthenzIdentity() {
         tester.newRun(JobType.stagingTest);
         tester.runner().run();
-        DeploymentSpec spec = tester.configServer().application(InternalDeploymentTester.testerId.id()).get().applicationPackage().deploymentSpec();
+        DeploymentSpec spec = tester.configServer()
+                                    .application(InternalDeploymentTester.testerId.id(), JobType.stagingTest.zone(system())).get()
+                                    .applicationPackage().deploymentSpec();
         assertEquals("domain", spec.athenzDomain().get().value());
-        ZoneId zone = JobType.stagingTest.zone(tester.tester().controller().system());
+        ZoneId zone = JobType.stagingTest.zone(system());
         assertEquals("service", spec.athenzService(zone.environment(), zone.region()).get().value());
     }
 
     @Test
     public void refeedRequirementBlocksDeployment() {
-        RunId id = tester.newRun(JobType.productionUsCentral1);
+        RunId id = tester.newRun(JobType.stagingTest);
+
+        tester.setEndpoints(testerId.id(), JobType.stagingTest.zone(system()));
+        tester.runner().run();
+        assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installInitialReal));
+
+        tester.setEndpoints(appId, JobType.stagingTest.zone(system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(Collections.emptyList(),
                                                                                       singletonList(new RefeedAction("Refeed",
                                                                                                                         false,
@@ -129,8 +147,12 @@ public class InternalStepRunnerTest {
     @Test
     public void restartsServicesAndWaitsForRestartAndReboot() {
         RunId id = tester.newRun(JobType.productionUsCentral1);
-        ZoneId zone = id.type().zone(tester.tester().controller().system());
+        ZoneId zone = id.type().zone(system());
         HostName host = tester.configServer().hostFor(appId, zone);
+
+        tester.setEndpoints(testerId.id(), JobType.productionUsCentral1.zone(system()));
+        tester.runner().run();
+
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster",
                                                                                                              "container",
                                                                                                              "search",
@@ -153,7 +175,7 @@ public class InternalStepRunnerTest {
 
         tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1)));
         tester.runner().run();
-        assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.installReal));
+        assertEquals(RunStatus.error, tester.jobs().run(id).get().status());
     }
 
     @Test
@@ -161,16 +183,16 @@ public class InternalStepRunnerTest {
         tester.newRun(JobType.systemTest);
 
         // Tester fails to show up for staging tests, and the real deployment for system tests.
-        tester.setEndpoints(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.setEndpoints(appId, JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.setEndpoints(testerId.id(), JobType.systemTest.zone(system()));
+        tester.setEndpoints(appId, JobType.stagingTest.zone(system()));
 
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(system()));
         tester.runner().run();
 
         tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1)));
@@ -183,12 +205,12 @@ public class InternalStepRunnerTest {
     public void installationFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.setEndpoints(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.setEndpoints(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal));
 
-        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester));
         assertTrue(tester.jobs().last(appId, JobType.systemTest).get().hasEnded());
@@ -199,11 +221,11 @@ public class InternalStepRunnerTest {
     public void startTestsFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
         tester.runner().run();
 
-        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests));
     }
@@ -212,20 +234,20 @@ public class InternalStepRunnerTest {
     public void alternativeEndpointsAreDetected() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();;
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal));
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester));
 
         tester.tester().controller().curator().writeRoutingPolicies(appId, Set.of(new RoutingPolicy(appId,
                                                                                                     ClusterSpec.Id.from("default"),
-                                                                                                    JobType.systemTest.zone(tester.tester().controller().system()),
+                                                                                                    JobType.systemTest.zone(system()),
                                                                                                     HostName.from("host"),
                                                                                                     Optional.empty(),
                                                                                                     emptySet())));
         tester.tester().controller().curator().writeRoutingPolicies(testerId.id(), Set.of(new RoutingPolicy(testerId.id(),
                                                                                                             ClusterSpec.Id.from("default"),
-                                                                                                            JobType.systemTest.zone(tester.tester().controller().system()),
+                                                                                                            JobType.systemTest.zone(system()),
                                                                                                             HostName.from("host"),
                                                                                                             Optional.empty(),
                                                                                                             emptySet())));
@@ -275,16 +297,15 @@ public class InternalStepRunnerTest {
         RunId id = tester.startSystemTestTests();
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.endTests));
-        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()))).get(0).endpoint()),
+        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(system()))).get(0).endpoint()),
                      tester.cloud().testerUrl());
         Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get();
         assertEquals(appId.serializedForm(), configObject.field("application").asString());
-        assertEquals(JobType.systemTest.zone(tester.tester().controller().system()).value(), configObject.field("zone").asString());
-        assertEquals(tester.tester().controller().system().value(), configObject.field("system").asString());
+        assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString());
+        assertEquals(system().value(), configObject.field("system").asString());
         assertEquals(1, configObject.field("endpoints").children());
-        assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).entries());
-        configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).traverse((ArrayTraverser) (__, endpoint) ->
-                assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(tester.tester().controller().system()))).get(0).endpoint(), endpoint.asString()));
+        assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).entries());
+        configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(system()))).get(0).endpoint(), endpoint.asString()));
 
         long lastId = tester.jobs().details(id).get().lastId().getAsLong();
         tester.cloud().add(new LogEntry(0, 123, info, "Ready!"));
@@ -311,7 +332,7 @@ public class InternalStepRunnerTest {
 
     @Test
     public void deployToDev() {
-        ZoneId zone = JobType.devUsEast1.zone(tester.tester().controller().system());
+        ZoneId zone = JobType.devUsEast1.zone(system());
         tester.jobs().deploy(appId, JobType.devUsEast1, Optional.empty(), applicationPackage);
         tester.runner().run();
         RunId id = tester.jobs().last(appId, JobType.devUsEast1).get().id();
@@ -332,8 +353,9 @@ public class InternalStepRunnerTest {
         tester.configServer().convergeServices(appId, zone);
         tester.setEndpoints(appId, zone);
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal));
-        assertEquals(otherPackage.hash(), tester.configServer().application(appId).get().applicationPackage().hash());
-        
+        assertEquals(applicationPackage.hash(), tester.configServer().application(appId, zone).get().applicationPackage().hash());
+        assertEquals(otherPackage.hash(), tester.configServer().application(appId, JobType.perfUsEast3.zone(system())).get().applicationPackage().hash());
+
         tester.configServer().setVersion(appId, zone, version);
         tester.runner().run();
         assertEquals(1, tester.jobs().active().size());
@@ -387,6 +409,21 @@ public class InternalStepRunnerTest {
                                           "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"));
     }
 
+    @Test
+    public void certificateTimeoutAbortsJob() {
+        tester.tester().controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
+        tester.tester().controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.aws-us-east-1c"));
+        RunId id = tester.startSystemTestTests();
+
+        List<X509Certificate> trusted = new ArrayList<>(publicCdApplicationPackage.trustedCertificates());
+        trusted.add(tester.jobs().run(id).get().testerCertificate().get());
+        assertEquals(trusted, tester.configServer().application(appId, id.type().zone(system())).get().applicationPackage().trustedCertificates());
+
+        tester.clock().advance(InternalStepRunner.certificateTimeout.plus(Duration.ofSeconds(1)));
+        tester.runner().run();
+        assertEquals(RunStatus.aborted, tester.jobs().run(id).get().status());
+    }
+
     private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) {
         assertEquals(List.of(entries), tester.jobs().details(id).get().get(step));
     }
@@ -401,6 +438,7 @@ public class InternalStepRunnerTest {
     public void generates_correct_services_xml_test() {
         assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"),
                                                                                             true,
+                                                                                            false,
                                                                                             Optional.of("d-2-12-75"))));
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index c160ed99a20..4d16b546d6b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -54,7 +54,7 @@ import java.util.stream.Stream;
  */
 public class ConfigServerMock extends AbstractComponent implements ConfigServer {
 
-    private final Map<ApplicationId, Application> applications = new LinkedHashMap<>();
+    private final Map<DeploymentId, Application> applications = new LinkedHashMap<>();
     private final Map<String, EndpointStatus> endpoints = new HashMap<>();
     private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock();
     private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>();
@@ -161,8 +161,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     /** Get deployed application by ID */
-    public Optional<Application> application(ApplicationId id) {
-        return Optional.ofNullable(applications.get(id));
+    public Optional<Application> application(ApplicationId id, ZoneId zone) {
+        return Optional.ofNullable(applications.get(new DeploymentId(id, zone)));
     }
 
     public void setSuspended(DeploymentId deployment, boolean suspend) {
@@ -234,7 +234,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
             this.prepareException = null;
             throw prepareException;
         }
-        applications.put(deployment.applicationId(), new Application(deployment.applicationId(), lastPrepareVersion, new ApplicationPackage(content)));
+        applications.put(deployment, new Application(deployment.applicationId(), lastPrepareVersion, new ApplicationPackage(content)));
 
         if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty())
             provision(deployment.zoneId(), deployment.applicationId());
@@ -252,7 +252,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         );
 
         return () -> {
-            Application application = applications.get(deployment.applicationId());
+            Application application = applications.get(deployment);
             application.activate();
             List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId());
             for (Node node : nodes) {
@@ -302,9 +302,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         ApplicationId applicationId = deployment.applicationId();
         nodeRepository().removeByHostname(deployment.zoneId(),
                                           nodeRepository().list(deployment.zoneId(), applicationId));
-        if ( ! applications.containsKey(applicationId))
+        if ( ! applications.containsKey(deployment))
             throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate");
-        applications.remove(applicationId);
+        applications.remove(deployment);
         serviceStatus.remove(deployment);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 280a25acc08..9404b1d0c54 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -124,17 +124,17 @@ public class JobRunnerTest {
         Map<Step, Status> steps = run.get().steps();
         runner.maintain();
         assertEquals(steps, run.get().steps());
-        assertEquals(List.of(deployReal, deployTester), run.get().readySteps());
+        assertEquals(List.of(deployTester), run.get().readySteps());
 
-        outcomes.put(deployReal, running);
+        outcomes.put(deployTester, running);
         runner.maintain();
-        assertEquals(List.of(installReal, deployTester), run.get().readySteps());
+        assertEquals(List.of(deployReal), run.get().readySteps());
 
-        outcomes.put(installReal, running);
+        outcomes.put(deployReal, running);
         runner.maintain();
-        assertEquals(List.of(deployTester), run.get().readySteps());
+        assertEquals(List.of(installTester, installReal), run.get().readySteps());
 
-        outcomes.put(deployTester, running);
+        outcomes.put(installReal, running);
         runner.maintain();
         assertEquals(List.of(installTester), run.get().readySteps());
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 7b817c175b8..9677df6fd18 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -81,7 +81,7 @@ public class SystemUpgraderTest {
         assertWantedVersion(SystemApplication.proxy, version2, zone1);
         completeUpgrade(SystemApplication.proxy, version2, zone1);
         assertTrue("Deployed proxy application",
-                   tester.configServer().application(SystemApplication.proxy.id()).isPresent());
+                   tester.configServer().application(SystemApplication.proxy.id(), zone1.getId()).isPresent());
 
         // zone 2, 3 and 4: still targets old version
         assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index 9cdaf925545..bf6cf716b7d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -91,6 +92,15 @@ public class RunSerializerTest {
                                                                 "badb17"),
                                              122),
                      run.versions().sourceApplication().get());
+        assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" +
+                                                  "MIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\n" +
+                                                  "ZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\n" +
+                                                  "c2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\n" +
+                                                  "Y50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n" +
+                                                  "0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n" +
+                                                  "5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n" +
+                                                  "-----END CERTIFICATE-----"),
+                     run.testerCertificate().get());
         assertEquals(ImmutableMap.<Step, Step.Status>builder()
                              .put(deployInitialReal, unfinished)
                              .put(installInitialReal, failed)
@@ -117,8 +127,12 @@ public class RunSerializerTest {
         assertEquals(run.end(), phoenix.end());
         assertEquals(run.status(), phoenix.status());
         assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry());
+        assertEquals(run.testerCertificate(), phoenix.testerCertificate());
         assertEquals(run.versions(), phoenix.versions());
         assertEquals(run.steps(), phoenix.steps());
+
+        Run initial = Run.initial(id, run.versions(), run.start());
+        assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial)));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index 880d2ebf527..e112493cb94 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -6,6 +6,7 @@
     "start": 1196676930000,
     "status": "running",
     "lastTestRecord": 3,
+    "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----",
     "steps": {
       "deployInitialReal": "unfinished",
       "installInitialReal": "failed",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
index 709a4dc2de0..222cf88ed87 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
@@ -51,10 +51,10 @@
             }
           },
           "steps": {
-            "deployReal": "unfinished",
-            "installReal": "unfinished",
             "deployTester": "unfinished",
+            "deployReal": "unfinished",
             "installTester": "unfinished",
+            "installReal": "unfinished",
             "startTests": "unfinished",
             "endTests": "unfinished",
             "copyVespaLogs": "unfinished",
@@ -62,9 +62,7 @@
             "deactivateTester": "unfinished",
             "report": "unfinished"
           },
-          "tasks": {
-            "deploy": "running"
-          },
+          "tasks": {},
           "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1"
         }
       ],
@@ -87,12 +85,12 @@
             }
           },
           "steps": {
+            "deployTester": "unfinished",
             "deployInitialReal": "unfinished",
             "installInitialReal": "unfinished",
             "deployReal": "unfinished",
-            "installReal": "unfinished",
-            "deployTester": "unfinished",
             "installTester": "unfinished",
+            "installReal": "unfinished",
             "startTests": "unfinished",
             "endTests": "unfinished",
             "copyVespaLogs": "unfinished",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
index a8ec9295868..2fbf203982f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
@@ -110,10 +110,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -154,10 +154,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -188,10 +188,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -264,12 +264,12 @@
             }
           },
           "steps": {
+            "deployTester": "succeeded",
             "deployInitialReal": "succeeded",
             "installInitialReal": "failed",
             "deployReal": "unfinished",
-            "installReal": "unfinished",
-            "deployTester": "succeeded",
             "installTester": "unfinished",
+            "installReal": "unfinished",
             "startTests": "unfinished",
             "endTests": "unfinished",
             "copyVespaLogs": "succeeded",
@@ -306,12 +306,12 @@
             }
           },
           "steps": {
+            "deployTester": "succeeded",
             "deployInitialReal": "succeeded",
             "installInitialReal": "succeeded",
             "deployReal": "succeeded",
-            "installReal": "succeeded",
-            "deployTester": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -352,12 +352,12 @@
             }
           },
           "steps": {
+            "deployTester": "succeeded",
             "deployInitialReal": "succeeded",
             "installInitialReal": "succeeded",
             "deployReal": "succeeded",
-            "installReal": "succeeded",
-            "deployTester": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -388,12 +388,12 @@
             }
           },
           "steps": {
+            "deployTester": "succeeded",
             "deployInitialReal": "succeeded",
             "installInitialReal": "succeeded",
             "deployReal": "succeeded",
-            "installReal": "succeeded",
-            "deployTester": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "copyVespaLogs": "succeeded",
@@ -438,10 +438,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "unfinished",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "unfinished",
+            "installReal": "unfinished",
             "startTests": "unfinished",
             "endTests": "unfinished",
             "deactivateTester": "unfinished",
@@ -479,10 +479,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "deactivateTester": "succeeded",
@@ -511,10 +511,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "deactivateTester": "succeeded",
@@ -584,10 +584,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "failed",
             "deactivateTester": "succeeded",
@@ -616,10 +616,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "deactivateTester": "succeeded",
@@ -690,18 +690,16 @@
             }
           },
           "steps": {
-            "deployReal": "failed",
-            "installReal": "unfinished",
-            "deployTester": "unfinished",
+            "deployTester": "failed",
+            "deployReal": "unfinished",
             "installTester": "unfinished",
+            "installReal": "unfinished",
             "startTests": "unfinished",
             "endTests": "unfinished",
             "deactivateTester": "succeeded",
             "report": "succeeded"
           },
-          "tasks": {
-            "deploy": "failed"
-          },
+          "tasks": {},
           "log": "https://some.url:43/root/production-us-east-3/run/2"
         },
         {
@@ -720,10 +718,10 @@
             }
           },
           "steps": {
-            "deployReal": "succeeded",
-            "installReal": "succeeded",
             "deployTester": "succeeded",
+            "deployReal": "succeeded",
             "installTester": "succeeded",
+            "installReal": "succeeded",
             "startTests": "succeeded",
             "endTests": "succeeded",
             "deactivateTester": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
index 800c3976188..228bdb17ecb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
@@ -14,10 +14,10 @@
       }
     },
     "steps": {
-      "deployReal": "unfinished",
-      "installReal": "unfinished",
       "deployTester": "unfinished",
+      "deployReal": "unfinished",
       "installTester": "unfinished",
+      "installReal": "unfinished",
       "startTests": "unfinished",
       "endTests": "unfinished",
       "copyVespaLogs": "unfinished",
@@ -25,9 +25,7 @@
       "deactivateTester": "unfinished",
       "report": "unfinished"
     },
-    "tasks": {
-      "deploy": "running"
-    },
+    "tasks": { },
     "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1"
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
index 390024fe33d..2891ea943f2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
@@ -1,7 +1,7 @@
 {
   "active": false,
   "log": {
-    "deployReal": [
+    "deployTester": [
       {
         "at": 1000,
         "type": "debug",
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd
index 776c91eb7ef..b4fc6de5e95 100644
--- a/controller-server/src/test/resources/test_runner_services.xml-cd
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd
@@ -7,6 +7,7 @@
                 <artifactsPath>artifacts</artifactsPath>
                 <surefireMemoryMb>5120</surefireMemoryMb>
                 <useAthenzCredentials>true</useAthenzCredentials>
+                <useTesterCertificate>false</useTesterCertificate>
             </config>
         </component>