Add configurable deadline for cluster controller tasks

Prevents an unstable cluster from potentially holding up all
container request processing threads indefinitely.
Deadline errors are translated into HTTP 504 errors to REST API clients.

1 files changed, 36 insertions, 1 deletions

diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
index d9815241920..b10a8101c37 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
@@ -16,6 +16,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.time.Duration;
 import java.util.*;
 import java.util.logging.Logger;
 
@@ -1248,16 +1249,25 @@ public class StateChangeTest extends FleetControllerTest {
     private static abstract class MockTask extends RemoteClusterControllerTask {
         boolean invoked = false;
         boolean leadershipLost = false;
+        boolean deadlineExceeded = false;
 
         boolean isInvoked() { return invoked; }
 
         boolean isLeadershipLost() { return leadershipLost; }
 
+        boolean isDeadlineExceeded() { return deadlineExceeded; }
+
         @Override
         public boolean hasVersionAckDependency() { return true; }
 
         @Override
-        public void handleLeadershipLost() { this.leadershipLost = true; }
+        public void handleFailure(FailureCondition condition) {
+            if (condition == FailureCondition.LEADERSHIP_LOST) {
+                this.leadershipLost = true;
+            } else if (condition == FailureCondition.DEADLINE_EXCEEDED) {
+                this.deadlineExceeded = true;
+            }
+        }
     }
 
     // We create an explicit mock task class instead of using mock() simply because of
@@ -1541,4 +1551,29 @@ public class StateChangeTest extends FleetControllerTest {
         assertTrue(task.isCompleted());
     }
 
+    @Test
+    public void task_not_completed_within_deadline_is_failed_with_deadline_exceeded_error() throws Exception {
+        FleetControllerOptions options = defaultOptions();
+        options.setMaxDeferredTaskVersionWaitTime(Duration.ofSeconds(60));
+        RemoteTaskFixture fixture = createFixtureWith(options);
+
+        MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects();
+        communicator.setShouldDeferDistributorClusterStateAcks(true);
+        fixture.processScheduledTask();
+
+        assertTrue(task.isInvoked());
+        assertFalse(task.isCompleted());
+        assertFalse(task.isDeadlineExceeded());
+
+        timer.advanceTime(59_000);
+        ctrl.tick();
+        assertFalse(task.isCompleted());
+        assertFalse(task.isDeadlineExceeded());
+
+        timer.advanceTime(1_001);
+        ctrl.tick();
+        assertTrue(task.isCompleted());
+        assertTrue(task.isDeadlineExceeded());
+    }
+
 }