Retry on suspend with unknown service status

2 files changed, 26 insertions, 1 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java
index f6333b7387b..87e2f6db761 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java
@@ -37,6 +37,12 @@ public interface ConfigServerApi extends AutoCloseable {
 
         public Optional<Duration> getConnectionTimeout() { return connectionTimeout; }
 
+        /** Set the retry policy to use against the config servers. */
+        public Params<T> setRetryPolicy(RetryPolicy<T> retryPolicy) {
+            this.retryPolicy = retryPolicy;
+            return this;
+        }
+
         public RetryPolicy<T> getRetryPolicy() { return retryPolicy; }
     }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java
index 63e157e8589..b8ea119c0be 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException;
 import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 
 import java.time.Duration;
@@ -44,7 +45,10 @@ public class OrchestratorImpl implements Orchestrator {
     public void suspend(final String hostName) {
         UpdateHostResponse response;
         try {
-            var params = new ConfigServerApi.Params<UpdateHostResponse>().setConnectionTimeout(CONNECTION_TIMEOUT);
+            var params = new ConfigServerApi
+                    .Params<UpdateHostResponse>()
+                    .setConnectionTimeout(CONNECTION_TIMEOUT)
+                    .setRetryPolicy(createRetryPolicyForSuspend());
             response = configServerApi.put(getSuspendPath(hostName), Optional.empty(), UpdateHostResponse.class, params);
         } catch (HttpException.NotFoundException n) {
             throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found");
@@ -61,6 +65,21 @@ public class OrchestratorImpl implements Orchestrator {
         });
     }
 
+    private static ConfigServerApi.RetryPolicy<UpdateHostResponse> createRetryPolicyForSuspend() {
+        return new ConfigServerApi.RetryPolicy<UpdateHostResponse>() {
+            @Override
+            public boolean tryNextConfigServer(UpdateHostResponse response) {
+                HostStateChangeDenialReason reason = response.reason();
+                if (reason == null) {
+                    return false;
+                }
+
+                // The config server has likely just bootstrapped, so try the next.
+                return "unknown-service-status".equals(reason.constraintName());
+            }
+        };
+    }
+
     @Override
     public void suspend(String parentHostName, List<String> hostNames) {
         final BatchOperationResult batchOperationResult;