Avoid stacktrace in log on timeouts

3 files changed, 66 insertions, 8 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
index 52994db2b88..cc8a401ed78 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.resources;
 
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.container.jaxrs.annotation.Component;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -9,6 +10,7 @@ import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
 import com.yahoo.vespa.orchestrator.restapi.HostApi;
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.HostService;
@@ -73,6 +75,9 @@ public class HostResource implements HostApi {
                     host.getHostStatus().name(),
                     applicationUri.toString(),
                     hostServices);
+        } catch (UncheckedTimeoutException e) {
+            log.log(LogLevel.INFO, "Failed to get host " + hostName + ": " + e.getMessage());
+            throw webExceptionFromTimeout("getHost", hostName, e);
         } catch (HostNameNotFoundException e) {
             log.log(LogLevel.INFO, "Host not found: " + hostName);
             throw new NotFoundException(e);
@@ -96,6 +101,9 @@ public class HostResource implements HostApi {
             } catch (HostNameNotFoundException e) {
                 log.log(LogLevel.INFO, "Host not found: " + hostName);
                 throw new NotFoundException(e);
+            } catch (UncheckedTimeoutException e) {
+                log.log(LogLevel.INFO, "Failed to patch " + hostName + ": " + e.getMessage());
+                throw webExceptionFromTimeout("patch", hostName, e);
             } catch (OrchestrationException e) {
                 String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage();
                 log.log(LogLevel.INFO, message, e);
@@ -116,6 +124,9 @@ public class HostResource implements HostApi {
         } catch (HostNameNotFoundException e) {
             log.log(LogLevel.INFO, "Host not found: " + hostName);
             throw new NotFoundException(e);
+        } catch (UncheckedTimeoutException e) {
+            log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage());
+            throw webExceptionFromTimeout("suspend", hostName, e);
         } catch (HostStateChangeDeniedException e) {
             log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage());
             throw webExceptionWithDenialReason("suspend", hostName, e);
@@ -131,6 +142,9 @@ public class HostResource implements HostApi {
         } catch (HostNameNotFoundException e) {
             log.log(LogLevel.INFO, "Host not found: " + hostName);
             throw new NotFoundException(e);
+        } catch (UncheckedTimeoutException e) {
+            log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage());
+            throw webExceptionFromTimeout("resume", hostName, e);
         } catch (HostStateChangeDeniedException e) {
             log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage());
             throw webExceptionWithDenialReason("resume", hostName, e);
@@ -138,15 +152,25 @@ public class HostResource implements HostApi {
         return new UpdateHostResponse(hostName.s(), null);
     }
 
+    private static WebApplicationException webExceptionFromTimeout(String operationDescription, HostName hostName, UncheckedTimeoutException e) {
+        return createWebException(operationDescription, hostName, e, HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage());
+    }
+
     private static WebApplicationException webExceptionWithDenialReason(
             String operationDescription,
             HostName hostName,
             HostStateChangeDeniedException e) {
-        HostStateChangeDenialReason hostStateChangeDenialReason =
-                new HostStateChangeDenialReason(
-                        e.getConstraintName(),
-                        operationDescription + " failed: " + e.getMessage());
-        UpdateHostResponse response = new UpdateHostResponse(hostName.s(), hostStateChangeDenialReason);
+        return createWebException(operationDescription, hostName, e, e.getConstraintName(), e.getMessage());
+    }
+
+    private static WebApplicationException createWebException(String operationDescription,
+                                                              HostName hostname,
+                                                              Exception e,
+                                                              String constraint,
+                                                              String message) {
+        HostStateChangeDenialReason hostStateChangeDenialReason = new HostStateChangeDenialReason(
+                constraint, operationDescription + " failed: " + message);
+        UpdateHostResponse response = new UpdateHostResponse(hostname.s(), hostStateChangeDenialReason);
         return new WebApplicationException(
                 hostStateChangeDenialReason.toString(),
                 e,
@@ -154,8 +178,6 @@ public class HostResource implements HostApi {
                         .entity(response)
                         .type(MediaType.APPLICATION_JSON_TYPE)
                         .build());
-
     }
-
 }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
index 6fa1e06520e..24d42264a51 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.resources;
 
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.container.jaxrs.annotation.Component;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -41,7 +42,7 @@ public class HostSuspensionResource implements HostSuspensionApi {
         List<HostName> hostnames = hostnamesAsStrings.stream().map(HostName::new).collect(Collectors.toList());
         try {
             orchestrator.suspendAll(parentHostname, hostnames);
-        } catch (BatchHostStateChangeDeniedException e) {
+        } catch (BatchHostStateChangeDeniedException | UncheckedTimeoutException e) {
             log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e);
             throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT);
         } catch (BatchHostNameNotFoundException e) {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 035d9cd686f..e943a4f105b 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.resources;
 
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
@@ -12,7 +13,10 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.BatchInternalErrorException;
 import com.yahoo.vespa.orchestrator.Host;
+import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
 import com.yahoo.vespa.orchestrator.InstanceLookupService;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
@@ -20,10 +24,12 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
+import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.Policy;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
+import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
@@ -360,4 +366,33 @@ public class HostResourceTest {
         assertEquals("UP", response.services().get(0).serviceStatus);
         assertEquals("serviceType", response.services().get(0).serviceType);
     }
+
+    @Test
+    public void throws_409_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).resume(any(HostName.class));
+
+        try {
+            HostResource hostResource = new HostResource(orchestrator, uriInfo);
+            hostResource.resume("hostname");
+            fail();
+        } catch (WebApplicationException w) {
+            assertThat(w.getResponse().getStatus()).isEqualTo(409);
+            assertEquals("resume failed: Timeout Message [deadline]", w.getMessage());
+        }
+    }
+
+    @Test
+    public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any());
+
+        try {
+            HostSuspensionResource resource = new HostSuspensionResource(orchestrator);
+            resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3"));
+            fail();
+        } catch (WebApplicationException w) {
+            assertThat(w.getResponse().getStatus()).isEqualTo(409);
+        }
+    }
 }