Differentiate between trying and requiring node state change

3 files changed, 29 insertions, 12 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index db18a71c805..5a9d75a9d41 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -425,7 +425,7 @@ public class OrchestratorImpl implements Orchestrator {
                 ClusterControllerClient client = clusterControllerClientFactory.createClient(clusterControllers, cluster.clusterId().s());
                 for (ServiceInstance service : cluster.serviceInstances()) {
                     try {
-                        if ( ! client.setNodeState(context, service.hostName(), VespaModelUtil.getStorageNodeIndex(service.configId()), MAINTENANCE))
+                        if ( ! client.trySetNodeState(context, service.hostName(), VespaModelUtil.getStorageNodeIndex(service.configId()), MAINTENANCE))
                             return false;
                     }
                     catch (Exception e) {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java
index 2c31b475b21..98ca9f805b4 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java
@@ -18,8 +18,16 @@ public interface ClusterControllerClient {
      * @return false is this was a probe operation, and permission would be denied.
      * @throws HostStateChangeDeniedException if operation fails, or is otherwise disallowed.
      */
-    boolean setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex,
-                         ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException;
+    boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex,
+                            ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException;
+
+    /**
+     * Requests that a cluster controller sets the requested node to the requested state.
+     *
+     * @throws HostStateChangeDeniedException if operation fails, or is disallowed.
+     */
+    void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex,
+                      ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException;
 
     /**
      * Requests that a cluster controller sets all nodes in the cluster to the requested state.
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
index b4966e71bfa..7c7dca3b03e 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
@@ -52,8 +52,8 @@ public class ClusterControllerClientImpl implements ClusterControllerClient {
         this.client = client;
     }
 
-    @Override
-    public boolean setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) {
+    private boolean setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex,
+                              ClusterControllerNodeState wantedState, boolean throwOnFailure) {
         try {
             ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts();
             Inspector response = client.send(strategy(hosts), Method.POST)
@@ -64,13 +64,12 @@ public class ClusterControllerClientImpl implements ClusterControllerClient {
                                        .throwing(retryOnRedirect)
                                        .read(SlimeUtils::jsonToSlime).get();
             if ( ! response.field("wasModified").asBool()) {
-                if (context.isProbe())
-                    return false;
-
-                throw new HostStateChangeDeniedException(host,
-                                                         HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT,
-                                                         "Failed to set state to " + wantedState +
-                                                         " in cluster controller: " + response.field("reason").asString());
+                if (throwOnFailure)
+                    throw new HostStateChangeDeniedException(host,
+                                                             HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT,
+                                                             "Failed to set state to " + wantedState +
+                                                             " in cluster controller: " + response.field("reason").asString());
+                return false;
             }
             return true;
         }
@@ -100,6 +99,16 @@ public class ClusterControllerClientImpl implements ClusterControllerClient {
     }
 
     @Override
+    public boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException {
+        return setNodeState(context, host, storageNodeIndex, wantedState, false);
+    }
+
+    @Override
+    public void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException {
+        setNodeState(context, host, storageNodeIndex, wantedState, true);
+    }
+
+    @Override
     public void setApplicationState(OrchestratorContext context, ApplicationInstanceId applicationId,
                                     ClusterControllerNodeState wantedState) throws ApplicationStateChangeDeniedException {
         try {