Create and store tester certificate when deploying tester

9 files changed, 155 insertions, 54 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 8aa640a6882..bb6f9314112 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -15,6 +15,10 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
@@ -36,14 +40,22 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
 import com.yahoo.yolean.Exceptions;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.io.UncheckedIOException;
+import java.math.BigInteger;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -93,6 +105,7 @@ public class InternalStepRunner implements StepRunner {
 
     static final Duration endpointTimeout = Duration.ofMinutes(15);
     static final Duration installationTimeout = Duration.ofMinutes(150);
+    static final Duration certificateTimeout = Duration.ofMinutes(300);
 
     private final Controller controller;
     private final TestConfigSerializer testConfigSerializer;
@@ -109,12 +122,12 @@ public class InternalStepRunner implements StepRunner {
         DualLogger logger = new DualLogger(id, step.get());
         try {
             switch (step.get()) {
+                case deployTester: return deployTester(id, logger);
                 case deployInitialReal: return deployInitialReal(id, logger);
                 case installInitialReal: return installInitialReal(id, logger);
                 case deployReal: return deployReal(id, logger);
-                case deployTester: return deployTester(id, logger);
-                case installReal: return installReal(id, logger);
                 case installTester: return installTester(id, logger);
+                case installReal: return installReal(id, logger);
                 case startTests: return startTests(id, logger);
                 case endTests: return endTests(id, logger);
                 case copyVespaLogs: return copyVespaLogs(id, logger);
@@ -428,6 +441,17 @@ public class InternalStepRunner implements StepRunner {
             return Optional.of(aborted);
         }
 
+        Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate();
+        if (testerCertificate.isPresent()) {
+            try {
+                testerCertificate.get().checkValidity(Date.from(controller.clock().instant()));
+            }
+            catch (CertificateExpiredException | CertificateNotYetValidException e) {
+                logger.log(INFO, "Tester certificate expired before tests could complete.");
+                return Optional.of(aborted);
+            }
+        };
+
         Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
         if ( ! testerEndpoint.isPresent()) {
             logger.log("Endpoints for tester not found -- trying again later.");
@@ -599,6 +623,16 @@ public class InternalStepRunner implements StepRunner {
         ZoneId zone = id.type().zone(controller.system());
         byte[] deploymentXml = deploymentXml(spec.athenzDomain(), spec.athenzService(zone.environment(), zone.region()));
 
+        if (controller.system().isPublic()) {
+            KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+            X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number());
+            X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, subject,
+                                                                             Instant.now(), Instant.now().plus(certificateTimeout),
+                                                                             SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(1))
+                                                                .build();
+            controller.jobController().storeTesterCertificate(id, certificate);
+        }
+
         try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + 1000)) {
             zipBuilder.add(testPackage);
             zipBuilder.add("services.xml", servicesXml);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 282e83461ea..d103bd28f19 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.net.URI;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -155,6 +156,11 @@ public class JobController {
             });
     }
 
+    /** Stores the given certificate as the tester certificate for this run, or throws if it's already set. */
+    public void storeTesterCertificate(RunId id, X509Certificate testerCertificate) {
+        locked(id, run -> run.with(testerCertificate));
+    }
+
     /** Returns a list of all application which have registered. */
     public List<ApplicationId> applications() {
         return copyOf(controller.applications().asList().stream()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index f052c7d91ab..94cf0343776 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 
+import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.EnumMap;
@@ -33,10 +34,11 @@ public class Run {
     private final Optional<Instant> end;
     private final RunStatus status;
     private final long lastTestRecord;
+    private final Optional<X509Certificate> testerCertificate;
 
     // For deserialisation only -- do not use!
     public Run(RunId id, Map<Step, Step.Status> steps, Versions versions, Instant start,
-               Optional<Instant> end, RunStatus status, long lastTestRecord) {
+               Optional<Instant> end, RunStatus status, long lastTestRecord, Optional<X509Certificate> testerCertificate) {
         this.id = id;
         this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
         this.versions = versions;
@@ -44,47 +46,57 @@ public class Run {
         this.end = end;
         this.status = status;
         this.lastTestRecord = lastTestRecord;
+        this.testerCertificate = testerCertificate;
     }
 
     public static Run initial(RunId id, Versions versions, Instant now) {
         EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class);
         JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished));
-        return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1);
+        return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1, Optional.empty());
     }
 
     /** Returns a new Run with the new status, and with the status of the given, completed step set accordingly. */
     public Run with(RunStatus status, LockedStep step) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be further modified!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
 
         if (steps.get(step.get()) != unfinished)
-            throw new AssertionError("Step '" + step.get() + "' can't be set to '" + status + "'" +
+            throw new IllegalStateException("Step '" + step.get() + "' can't be set to '" + status + "'" +
                                      " -- it already completed with status '" + steps.get(step.get()) + "'!");
 
         EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), Step.Status.of(status));
-        return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, lastTestRecord);
+        return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, lastTestRecord, testerCertificate);
     }
 
     public Run finished(Instant now) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be ended again!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be ended again!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status, lastTestRecord, testerCertificate);
     }
 
     public Run aborted() {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be aborted now!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be aborted now!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, end, aborted, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, end, aborted, lastTestRecord, testerCertificate);
     }
 
     public Run with(long lastTestRecord) {
         if (hasEnded())
-            throw new AssertionError("This run ended at " + end.get() + " -- it can't be further modified!");
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
 
-        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord);
+        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord, testerCertificate);
+    }
+
+    public Run with(X509Certificate testerCertificate) {
+        if (hasEnded())
+            throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!");
+        if (this.testerCertificate.isPresent())
+            throw new IllegalStateException("Certificate for this run was already set");
+
+        return new Run(id, new EnumMap<>(steps), versions, start, end, status, lastTestRecord, Optional.of(testerCertificate));
     }
 
     /** Returns the id of this run. */
@@ -131,6 +143,11 @@ public class Run {
         return lastTestRecord;
     }
 
+    /** Returns the tester certificate for this run, or empty. */
+    public Optional<X509Certificate> testerCertificate() {
+        return testerCertificate;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 1c95c9766f5..14d0087cf93 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import org.eclipse.jetty.util.security.CertificateUtils;
 
 import java.time.Instant;
 import java.util.EnumMap;
@@ -81,6 +83,7 @@ class RunSerializer {
     private static final String buildField = "build";
     private static final String sourceField = "source";
     private static final String lastTestRecordField = "lastTestRecord";
+    private static final String testerCertificateField = "testerCertificate";
 
     Run runFromSlime(Slime slime) {
         return runFromSlime(slime.get());
@@ -111,7 +114,10 @@ class RunSerializer {
                                .filter(Inspector::valid)
                                .map(end -> Instant.ofEpochMilli(end.asLong())),
                        runStatusOf(runObject.field(statusField).asString()),
-                       runObject.field(lastTestRecordField).asLong());
+                       runObject.field(lastTestRecordField).asLong(),
+                       Optional.of(runObject.field(testerCertificateField))
+                               .filter(Inspector::valid)
+                               .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())));
     }
 
     private Versions versionsFromSlime(Inspector versionsObject) {
@@ -169,6 +175,7 @@ class RunSerializer {
         run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli()));
         runObject.setString(statusField, valueOf(run.status()));
         runObject.setLong(lastTestRecordField, run.lastTestLogEntry());
+        run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate)));
 
         Cursor stepsObject = runObject.setObject(stepsField);
         run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status)));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
index e5fe658cf3b..0e5bf774441 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
@@ -31,13 +31,13 @@ public class BadgesTest {
 
     private static final ApplicationId id = ApplicationId.from("tenant", "application", "default");
     private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.of(now()), RunStatus.success, 0);
+                                               null, null, Optional.of(now()), RunStatus.success, 0, Optional.empty());
 
     private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.empty(), RunStatus.running, 0);
+                                               null, null, Optional.empty(), RunStatus.running, 0, Optional.empty());
 
     private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, Step.Status.succeeded),
-                                               null, null, Optional.of(now()), RunStatus.testFailure, 0);
+                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, Optional.empty());
 
     @Test
     public void test() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index fdb797a6bcd..930af275559 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -51,6 +52,13 @@ public class InternalDeploymentTester {
             .emailRole("author")
             .emailAddress("b@a")
             .build();
+    public static final ApplicationPackage publicCdApplicationPackage = new ApplicationPackageBuilder()
+            .athenzIdentity(AthenzDomain.from(ATHENZ_DOMAIN), AthenzService.from(ATHENZ_SERVICE))
+            .upgradePolicy("default")
+            .region("aws-us-east-1c")
+            .emailRole("author")
+            .emailAddress("b@a")
+            .build();
     public static final ApplicationId appId = ApplicationId.from("tenant", "application", "default");
     public static final TesterId testerId = TesterId.of(appId);
     public static final String athenzDomain = "domain";
@@ -98,7 +106,8 @@ public class InternalDeploymentTester {
      * Submits a new application, and returns the version of the new submission.
      */
     public ApplicationVersion newSubmission() {
-        return jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage, new byte[0]);
+        return jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2,
+                           tester.controller().system().isPublic() ? publicCdApplicationPackage : applicationPackage, new byte[0]);
     }
 
     /**
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 04de6d51988..af840cf27c5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -7,8 +7,8 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.log.LogLevel;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.vespa.config.SlimeUtils;
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -40,7 +41,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.logging.Logger;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
@@ -65,12 +65,14 @@ import static org.junit.Assert.fail;
 public class InternalStepRunnerTest {
 
     private InternalDeploymentTester tester;
-    private SystemName system;
 
     @Before
     public void setup() {
         tester = new InternalDeploymentTester();
-        system = tester.tester().controller().system();
+    }
+
+    private SystemName system() {
+        return tester.tester().controller().system();
     }
 
     @Test
@@ -84,9 +86,9 @@ public class InternalStepRunnerTest {
     public void canSwitchFromScrewdriverAndBackAgain() {
         // Deploys a default application package with default build number.
         tester.tester().deployCompletely(tester.app(), InternalDeploymentTester.applicationPackage);
-        tester.setEndpoints(appId, JobType.productionUsCentral1.zone(system));
-        tester.setEndpoints(appId, JobType.productionUsWest1.zone(system));
-        tester.setEndpoints(appId, JobType.productionUsEast3.zone(system));
+        tester.setEndpoints(appId, JobType.productionUsCentral1.zone(system()));
+        tester.setEndpoints(appId, JobType.productionUsWest1.zone(system()));
+        tester.setEndpoints(appId, JobType.productionUsEast3.zone(system()));
 
         // Let application have an ongoing upgrade when it switches (but kill the jobs, as the tester assumes they aren't running).
         tester.tester().upgradeSystem(new Version("7.1"));
@@ -112,7 +114,7 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         DeploymentSpec spec = tester.configServer().application(InternalDeploymentTester.testerId.id()).get().applicationPackage().deploymentSpec();
         assertEquals("domain", spec.athenzDomain().get().value());
-        ZoneId zone = JobType.stagingTest.zone(system);
+        ZoneId zone = JobType.stagingTest.zone(system());
         assertEquals("service", spec.athenzService(zone.environment(), zone.region()).get().value());
     }
 
@@ -120,12 +122,12 @@ public class InternalStepRunnerTest {
     public void refeedRequirementBlocksDeployment() {
         RunId id = tester.newRun(JobType.stagingTest);
 
-        tester.setEndpoints(testerId.id(), JobType.stagingTest.zone(system));
+        tester.setEndpoints(testerId.id(), JobType.stagingTest.zone(system()));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installInitialReal));
 
-        tester.setEndpoints(appId, JobType.stagingTest.zone(system));
-        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system));
+        tester.setEndpoints(appId, JobType.stagingTest.zone(system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(Collections.emptyList(),
                                                                                       singletonList(new RefeedAction("Refeed",
                                                                                                                         false,
@@ -141,10 +143,10 @@ public class InternalStepRunnerTest {
     @Test
     public void restartsServicesAndWaitsForRestartAndReboot() {
         RunId id = tester.newRun(JobType.productionUsCentral1);
-        ZoneId zone = id.type().zone(system);
+        ZoneId zone = id.type().zone(system());
         HostName host = tester.configServer().hostFor(appId, zone);
 
-        tester.setEndpoints(testerId.id(), JobType.productionUsCentral1.zone(system));
+        tester.setEndpoints(testerId.id(), JobType.productionUsCentral1.zone(system()));
         tester.runner().run();
 
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster",
@@ -177,16 +179,16 @@ public class InternalStepRunnerTest {
         tester.newRun(JobType.systemTest);
 
         // Tester fails to show up for staging tests, and the real deployment for system tests.
-        tester.setEndpoints(testerId.id(), JobType.systemTest.zone(system));
-        tester.setEndpoints(appId, JobType.stagingTest.zone(system));
+        tester.setEndpoints(testerId.id(), JobType.systemTest.zone(system()));
+        tester.setEndpoints(appId, JobType.stagingTest.zone(system()));
 
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system));
-        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system));
-        tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(system));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(system()));
         tester.runner().run();
 
         tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1)));
@@ -199,12 +201,12 @@ public class InternalStepRunnerTest {
     public void installationFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system));
-        tester.setEndpoints(appId, JobType.systemTest.zone(system));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.setEndpoints(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal));
 
-        tester.applications().deactivate(appId, JobType.systemTest.zone(system));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester));
         assertTrue(tester.jobs().last(appId, JobType.systemTest).get().hasEnded());
@@ -215,11 +217,11 @@ public class InternalStepRunnerTest {
     public void startTestsFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
         tester.runner().run();
 
-        tester.applications().deactivate(appId, JobType.systemTest.zone(system));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests));
     }
@@ -228,20 +230,20 @@ public class InternalStepRunnerTest {
     public void alternativeEndpointsAreDetected() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();;
-        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system));
-        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(system()));
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal));
         assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester));
 
         tester.tester().controller().curator().writeRoutingPolicies(appId, Set.of(new RoutingPolicy(appId,
                                                                                                     ClusterSpec.Id.from("default"),
-                                                                                                    JobType.systemTest.zone(system),
+                                                                                                    JobType.systemTest.zone(system()),
                                                                                                     HostName.from("host"),
                                                                                                     Optional.empty(),
                                                                                                     emptySet())));
         tester.tester().controller().curator().writeRoutingPolicies(testerId.id(), Set.of(new RoutingPolicy(testerId.id(),
                                                                                                             ClusterSpec.Id.from("default"),
-                                                                                                            JobType.systemTest.zone(system),
+                                                                                                            JobType.systemTest.zone(system()),
                                                                                                             HostName.from("host"),
                                                                                                             Optional.empty(),
                                                                                                             emptySet())));
@@ -291,15 +293,15 @@ public class InternalStepRunnerTest {
         RunId id = tester.startSystemTestTests();
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.endTests));
-        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(system))).get(0).endpoint()),
+        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(system()))).get(0).endpoint()),
                      tester.cloud().testerUrl());
         Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get();
         assertEquals(appId.serializedForm(), configObject.field("application").asString());
-        assertEquals(JobType.systemTest.zone(system).value(), configObject.field("zone").asString());
-        assertEquals(system.value(), configObject.field("system").asString());
+        assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString());
+        assertEquals(system().value(), configObject.field("system").asString());
         assertEquals(1, configObject.field("endpoints").children());
-        assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system).value()).entries());
-        configObject.field("endpoints").field(JobType.systemTest.zone(system).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(system))).get(0).endpoint(), endpoint.asString()));
+        assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).entries());
+        configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(system()))).get(0).endpoint(), endpoint.asString()));
 
         long lastId = tester.jobs().details(id).get().lastId().getAsLong();
         tester.cloud().add(new LogEntry(0, 123, info, "Ready!"));
@@ -326,7 +328,7 @@ public class InternalStepRunnerTest {
 
     @Test
     public void deployToDev() {
-        ZoneId zone = JobType.devUsEast1.zone(system);
+        ZoneId zone = JobType.devUsEast1.zone(system());
         tester.jobs().deploy(appId, JobType.devUsEast1, Optional.empty(), applicationPackage);
         tester.runner().run();
         RunId id = tester.jobs().last(appId, JobType.devUsEast1).get().id();
@@ -402,6 +404,17 @@ public class InternalStepRunnerTest {
                                           "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"));
     }
 
+    @Test
+    public void certificateTimeoutAbortsJob() {
+        tester.tester().controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
+        tester.tester().controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.aws-us-east-1c"));
+        RunId id = tester.startSystemTestTests();
+
+        tester.clock().advance(InternalStepRunner.certificateTimeout.plus(Duration.ofSeconds(1)));
+        tester.runner().run();
+        assertEquals(RunStatus.aborted, tester.jobs().run(id).get().status());
+    }
+
     private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) {
         assertEquals(List.of(entries), tester.jobs().details(id).get().get(step));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index 9cdaf925545..bf6cf716b7d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -91,6 +92,15 @@ public class RunSerializerTest {
                                                                 "badb17"),
                                              122),
                      run.versions().sourceApplication().get());
+        assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" +
+                                                  "MIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\n" +
+                                                  "ZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\n" +
+                                                  "c2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\n" +
+                                                  "Y50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n" +
+                                                  "0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n" +
+                                                  "5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n" +
+                                                  "-----END CERTIFICATE-----"),
+                     run.testerCertificate().get());
         assertEquals(ImmutableMap.<Step, Step.Status>builder()
                              .put(deployInitialReal, unfinished)
                              .put(installInitialReal, failed)
@@ -117,8 +127,12 @@ public class RunSerializerTest {
         assertEquals(run.end(), phoenix.end());
         assertEquals(run.status(), phoenix.status());
         assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry());
+        assertEquals(run.testerCertificate(), phoenix.testerCertificate());
         assertEquals(run.versions(), phoenix.versions());
         assertEquals(run.steps(), phoenix.steps());
+
+        Run initial = Run.initial(id, run.versions(), run.start());
+        assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial)));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index 880d2ebf527..e112493cb94 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -6,6 +6,7 @@
     "start": 1196676930000,
     "status": "running",
     "lastTestRecord": 3,
+    "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----",
     "steps": {
       "deployInitialReal": "unfinished",
       "installInitialReal": "failed",