Add configurable deadline for cluster controller tasks

Prevents an unstable cluster from potentially holding up all
container request processing threads indefinitely.
Deadline errors are translated into HTTP 504 errors to REST API clients.

1 files changed, 6 insertions, 0 deletions

diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
index 395958063f5..71e6bc36de1 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
@@ -89,6 +89,12 @@ public class RestApiHandler implements HttpRequestHandler {
             result.setHttpCode(503, "Service Unavailable");
             result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
             return result;
+        } catch (DeadlineExceededException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(504, "Gateway Timeout");
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
         } catch (StateRestApiException exception) {
             logRequestException(request, exception, Level.WARNING);
             JsonHttpResult result = new JsonHttpResult();