Add configurable deadline for cluster controller tasks

Prevents an unstable cluster from potentially holding up all
container request processing threads indefinitely.
Deadline errors are translated into HTTP 504 errors to REST API clients.

3 files changed, 27 insertions, 0 deletions

diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/DeadlineExceededException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/DeadlineExceededException.java
new file mode 100644
index 00000000000..55032d515ce
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/DeadlineExceededException.java
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class DeadlineExceededException extends StateRestApiException {
+    public DeadlineExceededException(String description) {
+        super(description);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
index 395958063f5..71e6bc36de1 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
@@ -89,6 +89,12 @@ public class RestApiHandler implements HttpRequestHandler {
             result.setHttpCode(503, "Service Unavailable");
             result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
             return result;
+        } catch (DeadlineExceededException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(504, "Gateway Timeout");
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
         } catch (StateRestApiException exception) {
             logRequestException(request, exception, Level.WARNING);
             JsonHttpResult result = new JsonHttpResult();
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
index 8328ecc491f..3492c632e60 100644
--- a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
@@ -526,4 +526,17 @@ public class StateRestAPITest {
         } catch (IllegalArgumentException e) {
         }
     }
+
+    @Test
+    public void deadline_exceeded_exception_returns_http_504_error() throws Exception {
+        setupDummyStateApi();
+        stateApi.induceException(new DeadlineExceededException("argh!"));
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+
+        assertEquals(result.toString(true), 504, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "Gateway Timeout", result.getHttpReturnCodeDescription());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"message\":\"argh!\"}";
+        assertEquals(expected, result.getContent().toString());
+    }
 }