Orchestrator support for setting host status

Directly setting the host status can be useful for an operator, e.g. to break a
deadlock.

Unfortunately, some code use the term "host status" while others use "host
state".  I haven't settled yet on which is preferred, e.g. 'state' is used in
REST APIs, but 'status' is the original term and slightly more used.  This PR
maintains the local use of the terms, meaning it adds 'state' where that's
normally used and 'status' other places.

Setting the deadlock is done with a PATCH request, which is defined in
jaxrs_utils. jaxrs_utils only defines PATCH, and that's not used anywhere else
except in jaxrs_client_utils tests.  So I remove this module as a dependency a
couple of places.

5 files changed, 104 insertions, 1 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index 1a1ffe13b37..6925ee26ccd 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -41,6 +41,8 @@ public interface Orchestrator {
      */
     HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
 
+    void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
+
     /**
      * Resume normal operation for this host.
      *
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 5cd5f0ab638..23d723b84e2 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -83,7 +83,15 @@ public class OrchestratorImpl implements Orchestrator {
 
     @Override
     public HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException {
-        return getNodeStatus(getApplicationInstance(hostName).reference(),hostName);
+        return getNodeStatus(getApplicationInstance(hostName).reference(), hostName);
+    }
+
+    @Override
+    public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException {
+        ApplicationInstanceReference reference = getApplicationInstance(hostName).reference();
+        try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly(reference)) {
+            statusRegistry.setHostState(hostName, status);
+        }
     }
 
     @Override
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
index c6e9e5415ce..54f1723c914 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
@@ -5,17 +5,23 @@ import com.yahoo.container.jaxrs.annotation.Component;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.restapi.HostApi;
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import javax.inject.Inject;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.InternalServerErrorException;
 import javax.ws.rs.NotFoundException;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
@@ -48,6 +54,35 @@ public class HostResource implements HostApi {
     }
 
     @Override
+    public PatchHostResponse patch(@PathParam("hostname") String hostNameString, PatchHostRequest request) {
+        HostName hostName = new HostName(hostNameString);
+
+        if (request.state != null) {
+            HostStatus state;
+            try {
+                state = HostStatus.valueOf(request.state);
+            } catch (IllegalArgumentException dummy) {
+                throw new BadRequestException("Bad state in request: '" + request.state + "'");
+            }
+
+            try {
+                orchestrator.setNodeStatus(hostName, state);
+            } catch (HostNameNotFoundException e) {
+                log.log(LogLevel.INFO, "Host not found: " + hostName);
+                throw new NotFoundException(e);
+            } catch (OrchestrationException e) {
+                String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage();
+                log.log(LogLevel.INFO, message, e);
+                throw new InternalServerErrorException(message);
+            }
+        }
+
+        PatchHostResponse response = new PatchHostResponse();
+        response.description = "ok";
+        return response;
+    }
+
+    @Override
     public UpdateHostResponse suspend(String hostNameString) {
         HostName hostName = new HostName(hostNameString);
         try {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index ccee70630af..448ae78a21a 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -218,6 +218,15 @@ public class OrchestratorImplTest {
     }
 
     @Test
+    public void testSetNodeState() throws OrchestrationException {
+        assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+        orchestrator.setNodeStatus(app1_host1, HostStatus.ALLOWED_TO_BE_DOWN);
+        assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+        orchestrator.setNodeStatus(app1_host1, HostStatus.NO_REMARKS);
+        assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+    }
+
+    @Test
     public void rollbackWorks() throws Exception {
         // A spy is preferential because suspendAll() relies on delegating the hard work to suspend() and resume().
         OrchestratorImpl orchestrator = spy(this.orchestrator);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 3d3117f9e07..aecfd79e505 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -7,6 +7,8 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.orchestrator.InstanceLookupService;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
@@ -14,6 +16,8 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.Policy;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
@@ -22,6 +26,8 @@ import com.yahoo.vespa.orchestrator.status.StatusService;
 import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
 import org.junit.Test;
 
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.InternalServerErrorException;
 import javax.ws.rs.WebApplicationException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -31,9 +37,13 @@ import java.util.Set;
 import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
 import static org.fest.assertions.Assertions.assertThat;
 import static org.fest.assertions.Fail.fail;
+import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class HostResourceTest {
@@ -262,4 +272,43 @@ public class HostResourceTest {
             assertThat(w.getResponse().getStatus()).isEqualTo(409);
         }
     }
+
+    @Test(expected = BadRequestException.class)
+    public void patch_state_may_throw_bad_request() {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "bad state";
+
+        hostResource.patch(hostNameString, request);
+    }
+
+    @Test
+    public void patch_works() throws OrchestrationException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "NO_REMARKS";
+
+        PatchHostResponse response = hostResource.patch(hostNameString, request);
+        assertEquals(response.description, "ok");
+        verify(orchestrator, times(1)).setNodeStatus(new HostName(hostNameString), HostStatus.NO_REMARKS);
+    }
+
+    @Test(expected = InternalServerErrorException.class)
+    public void patch_handles_exception_in_orchestrator() throws OrchestrationException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "NO_REMARKS";
+
+        doThrow(new OrchestrationException("error")).when(orchestrator).setNodeStatus(new HostName(hostNameString), HostStatus.NO_REMARKS);
+        hostResource.patch(hostNameString, request);
+    }
 }