Orchestrator support for setting host status

Directly setting the host status can be useful for an operator, e.g. to break a
deadlock.

Unfortunately, some code use the term "host status" while others use "host
state".  I haven't settled yet on which is preferred, e.g. 'state' is used in
REST APIs, but 'status' is the original term and slightly more used.  This PR
maintains the local use of the terms, meaning it adds 'state' where that's
normally used and 'status' other places.

Setting the deadlock is done with a PATCH request, which is defined in
jaxrs_utils. jaxrs_utils only defines PATCH, and that's not used anywhere else
except in jaxrs_client_utils tests.  So I remove this module as a dependency a
couple of places.

12 files changed, 151 insertions, 13 deletions

diff --git a/docker-api/pom.xml b/docker-api/pom.xml
index d35cd158edf..655030a02b5 100644
--- a/docker-api/pom.xml
+++ b/docker-api/pom.xml
@@ -97,12 +97,6 @@
         </dependency>
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
-            <artifactId>jaxrs_utils</artifactId>
-            <version>${project.version}</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.yahoo.vespa</groupId>
             <artifactId>jaxrs_client_utils</artifactId>
             <version>${project.version}</version>
             <scope>compile</scope>
diff --git a/node-admin/pom.xml b/node-admin/pom.xml
index 10fe37cfc4b..628b774d729 100644
--- a/node-admin/pom.xml
+++ b/node-admin/pom.xml
@@ -69,12 +69,6 @@
         </dependency>
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
-            <artifactId>jaxrs_utils</artifactId>
-            <version>${project.version}</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.yahoo.vespa</groupId>
             <artifactId>jaxrs_client_utils</artifactId>
             <version>${project.version}</version>
             <scope>compile</scope>
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 538feeb042d..5b559aef1a5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -33,6 +33,9 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
+    public void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException {}
+
+    @Override
     public void resume(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException {}
 
     @Override
diff --git a/orchestrator-restapi/pom.xml b/orchestrator-restapi/pom.xml
index b2b581a7fc6..79e8dc3e618 100644
--- a/orchestrator-restapi/pom.xml
+++ b/orchestrator-restapi/pom.xml
@@ -22,6 +22,12 @@
       <scope>provided</scope>
     </dependency>
     <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>jaxrs_utils</artifactId>
+      <version>${project.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson2.version}</version>
diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java
index a71db5b2749..2ddf05777ae 100644
--- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java
+++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.orchestrator.restapi;
 
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 
+import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.PUT;
@@ -37,6 +40,15 @@ public interface HostApi {
     GetHostResponse getHost(@PathParam("hostname") String hostNameString);
 
     /**
+     * Tweak internal Orchestrator state for host.
+     */
+    @PUT
+    @Path("/{hostname}")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    PatchHostResponse patch(@PathParam("hostname") String hostNameString, PatchHostRequest request);
+
+    /**
      * Ask for permission to temporarily suspend all services on a host.
      *
      * On success, none, some, or all services on the host may already have been effectively suspended,
diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostRequest.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostRequest.java
new file mode 100644
index 00000000000..6ce029e7f9c
--- /dev/null
+++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostRequest.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.restapi.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PatchHostRequest {
+    /** String representation of HostStatus enum value. */
+    @JsonProperty public String state;
+}
diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostResponse.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostResponse.java
new file mode 100644
index 00000000000..f7468fe5907
--- /dev/null
+++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/PatchHostResponse.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.restapi.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PatchHostResponse {
+    /** Description of the HTTP response status code. */
+    @JsonProperty public String description;
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index 1a1ffe13b37..6925ee26ccd 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -41,6 +41,8 @@ public interface Orchestrator {
      */
     HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
 
+    void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
+
     /**
      * Resume normal operation for this host.
      *
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 5cd5f0ab638..23d723b84e2 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -83,7 +83,15 @@ public class OrchestratorImpl implements Orchestrator {
 
     @Override
     public HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException {
-        return getNodeStatus(getApplicationInstance(hostName).reference(),hostName);
+        return getNodeStatus(getApplicationInstance(hostName).reference(), hostName);
+    }
+
+    @Override
+    public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException {
+        ApplicationInstanceReference reference = getApplicationInstance(hostName).reference();
+        try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly(reference)) {
+            statusRegistry.setHostState(hostName, status);
+        }
     }
 
     @Override
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
index c6e9e5415ce..54f1723c914 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
@@ -5,17 +5,23 @@ import com.yahoo.container.jaxrs.annotation.Component;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.restapi.HostApi;
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import javax.inject.Inject;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.InternalServerErrorException;
 import javax.ws.rs.NotFoundException;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
@@ -48,6 +54,35 @@ public class HostResource implements HostApi {
     }
 
     @Override
+    public PatchHostResponse patch(@PathParam("hostname") String hostNameString, PatchHostRequest request) {
+        HostName hostName = new HostName(hostNameString);
+
+        if (request.state != null) {
+            HostStatus state;
+            try {
+                state = HostStatus.valueOf(request.state);
+            } catch (IllegalArgumentException dummy) {
+                throw new BadRequestException("Bad state in request: '" + request.state + "'");
+            }
+
+            try {
+                orchestrator.setNodeStatus(hostName, state);
+            } catch (HostNameNotFoundException e) {
+                log.log(LogLevel.INFO, "Host not found: " + hostName);
+                throw new NotFoundException(e);
+            } catch (OrchestrationException e) {
+                String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage();
+                log.log(LogLevel.INFO, message, e);
+                throw new InternalServerErrorException(message);
+            }
+        }
+
+        PatchHostResponse response = new PatchHostResponse();
+        response.description = "ok";
+        return response;
+    }
+
+    @Override
     public UpdateHostResponse suspend(String hostNameString) {
         HostName hostName = new HostName(hostNameString);
         try {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index ccee70630af..448ae78a21a 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -218,6 +218,15 @@ public class OrchestratorImplTest {
     }
 
     @Test
+    public void testSetNodeState() throws OrchestrationException {
+        assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+        orchestrator.setNodeStatus(app1_host1, HostStatus.ALLOWED_TO_BE_DOWN);
+        assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+        orchestrator.setNodeStatus(app1_host1, HostStatus.NO_REMARKS);
+        assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+    }
+
+    @Test
     public void rollbackWorks() throws Exception {
         // A spy is preferential because suspendAll() relies on delegating the hard work to suspend() and resume().
         OrchestratorImpl orchestrator = spy(this.orchestrator);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 3d3117f9e07..aecfd79e505 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -7,6 +7,8 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.orchestrator.InstanceLookupService;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
@@ -14,6 +16,8 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.Policy;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
@@ -22,6 +26,8 @@ import com.yahoo.vespa.orchestrator.status.StatusService;
 import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
 import org.junit.Test;
 
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.InternalServerErrorException;
 import javax.ws.rs.WebApplicationException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -31,9 +37,13 @@ import java.util.Set;
 import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
 import static org.fest.assertions.Assertions.assertThat;
 import static org.fest.assertions.Fail.fail;
+import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class HostResourceTest {
@@ -262,4 +272,43 @@ public class HostResourceTest {
             assertThat(w.getResponse().getStatus()).isEqualTo(409);
         }
     }
+
+    @Test(expected = BadRequestException.class)
+    public void patch_state_may_throw_bad_request() {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "bad state";
+
+        hostResource.patch(hostNameString, request);
+    }
+
+    @Test
+    public void patch_works() throws OrchestrationException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "NO_REMARKS";
+
+        PatchHostResponse response = hostResource.patch(hostNameString, request);
+        assertEquals(response.description, "ok");
+        verify(orchestrator, times(1)).setNodeStatus(new HostName(hostNameString), HostStatus.NO_REMARKS);
+    }
+
+    @Test(expected = InternalServerErrorException.class)
+    public void patch_handles_exception_in_orchestrator() throws OrchestrationException {
+        Orchestrator orchestrator = mock(Orchestrator.class);
+        HostResource hostResource = new HostResource(orchestrator);
+
+        String hostNameString = "hostname";
+        PatchHostRequest request = new PatchHostRequest();
+        request.state = "NO_REMARKS";
+
+        doThrow(new OrchestrationException("error")).when(orchestrator).setNodeStatus(new HostName(hostNameString), HostStatus.NO_REMARKS);
+        hostResource.patch(hostNameString, request);
+    }
 }