Take automated deployment expiry into account

3 files changed, 49 insertions, 1 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 39b81baa023..ead9388fc3b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -51,6 +51,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Con
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active;
+import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.failed;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error;
@@ -227,6 +228,11 @@ public class InternalStepRunner implements StepRunner {
     }
 
     private Optional<RunStatus> installReal(RunId id, boolean setTheStage, ByteArrayLogger logger) {
+        if (expired(id.application(), id.type())) {
+            logger.log(INFO, "Deployment expired before installation was successful.");
+            return Optional.of(installationFailed);
+        }
+
         Versions versions = controller.jobController().run(id).get().versions();
         Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform();
         ApplicationVersion application = setTheStage ? versions.sourceApplication().orElse(versions.targetApplication()) : versions.targetApplication();
@@ -248,6 +254,10 @@ public class InternalStepRunner implements StepRunner {
 
     private Optional<RunStatus> installTester(RunId id, ByteArrayLogger logger) {
         logger.log("Checking installation of tester container ...");
+        if (expired(id.application(), id.type())) {
+            logger.log(INFO, "Deployment expired before tester was installed.");
+            return Optional.of(installationFailed);
+        }
 
         if (servicesConverged(testerOf(id.application()), id.type())) {
             logger.log("Tester container successfully installed!");
@@ -289,6 +299,11 @@ public class InternalStepRunner implements StepRunner {
 
     private Optional<RunStatus> startTests(RunId id, ByteArrayLogger logger) {
         logger.log("Attempting to find endpoints ...");
+        if (expired(id.application(), id.type())) {
+            logger.log(INFO, "Deployment expired before tests could start.");
+            return Optional.of(installationFailed);
+        }
+
         Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application());
         logger.log("Found endpoints:\n" +
                          endpoints.entrySet().stream()
@@ -378,6 +393,11 @@ public class InternalStepRunner implements StepRunner {
         return application(id).deployments().get(type.zone(controller.system())).at().isBefore(controller.clock().instant().minus(timeout));
     }
 
+    /** Returns whether the real deployment for the given job type has expired, i.e., no longer exists. */
+    private boolean expired(ApplicationId id, JobType type) {
+        return ! application(id).deployments().containsKey(type.zone(controller.system()));
+    }
+
     /** Returns a generated job report for the given run. */
     private DeploymentJobs.JobReport report(Run run) {
         return new DeploymentJobs.JobReport(run.id().application(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index 6ad57851846..cb5b70ef80a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -40,7 +40,7 @@ public enum Step {
     deployTester,
 
     /** See that tester is done deploying, and is ready to serve. */
-    installTester(deployTester),
+    installTester(deployReal, deployTester),
 
     /** Ask the tester to run its tests. */
     startTests(installReal, installTester),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 66756dc4415..ba0607892e6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -291,6 +291,34 @@ public class InternalStepRunnerTest {
     }
 
     @Test
+    public void installationFailsIfDeploymentExpires() {
+        newRun(JobType.systemTest);
+        runner.run();
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.controller().system()));
+        runner.run();
+        assertEquals(succeeded, jobs.last(appId, JobType.systemTest).get().steps().get(Step.installReal));
+
+        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.controller().system()));
+        runner.run();
+        assertEquals(failed, jobs.last(appId, JobType.systemTest).get().steps().get(Step.installTester));
+        assertTrue(jobs.last(appId, JobType.systemTest).get().hasEnded());
+        assertTrue(jobs.last(appId, JobType.systemTest).get().hasFailed());
+    }
+
+    @Test
+    public void startTestsFailsIfDeploymentExpires() {
+        newRun(JobType.systemTest);
+        runner.run();
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.controller().system()));
+        tester.configServer().convergeServices(testerOf(appId), JobType.systemTest.zone(tester.controller().system()));
+        runner.run();
+
+        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.controller().system()));
+        runner.run();
+        assertEquals(failed, jobs.last(appId, JobType.systemTest).get().steps().get(Step.startTests));
+    }
+
+    @Test
     public void testsFailIfEndpointsAreGone() {
         RunId id = startSystemTestTests();
         cloud.set(new byte[0], TesterCloud.Status.NOT_STARTED);