Add abortion after 1 day, on job levle

2 files changed, 20 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index 99f222f58d2..dd89644b580 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -25,6 +25,7 @@ import java.util.logging.Logger;
  */
 public class JobRunner extends Maintainer {
 
+    static final Duration jobTimeout = Duration.ofDays(1);
     private static final Logger log = Logger.getLogger(JobRunner.class.getName());
 
     private final JobController jobs;
@@ -67,6 +68,8 @@ public class JobRunner extends Maintainer {
         steps.forEach(step -> executors.execute(() -> advance(run.id(), step)));
         if (steps.isEmpty())
             jobs.finish(run.id());
+        else if (run.start().isBefore(controller().clock().instant().minus(jobTimeout)))
+            jobs.abort(run.id());
     }
 
     /** Attempts to advance the status of the given step, for the given run. */
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index ee1567dae11..1ec07025812 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -226,6 +226,23 @@ public class JobRunnerTest {
         assertEquals(Optional.empty(), jobs.last(id, systemTest));
     }
 
+    @Test
+    public void timeout() {
+        DeploymentTester tester = new DeploymentTester();
+        JobController jobs = tester.controller().jobController();
+        Map<Step, Status> outcomes = new EnumMap<>(Step.class);
+        JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
+                                         inThreadExecutor(), mappedRunner(outcomes));
+
+        ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
+        jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]);
+
+        jobs.start(id, systemTest, versions);
+        tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1)));
+        runner.run();
+        assertTrue(jobs.last(id, systemTest).get().isAborted());
+    }
+
     public static ExecutorService inThreadExecutor() {
         return new AbstractExecutorService() {
             AtomicBoolean shutDown = new AtomicBoolean(false);