Time out when no nodes are suspended, or the same node is suspended, for too long

3 files changed, 30 insertions, 2 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index 3877119e8b0..684dc3cbc25 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TenantName;
 
+import java.time.Instant;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -30,6 +31,7 @@ public class Node {
     private final Version currentOsVersion;
     private final Version wantedOsVersion;
     private final ServiceState serviceState;
+    private final Optional<Instant> suspendedSince;
     private final long restartGeneration;
     private final long wantedRestartGeneration;
     private final long rebootGeneration;
@@ -44,7 +46,7 @@ public class Node {
 
     public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner,
                 Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState,
-                long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
+                Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
                 int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision,
                 Optional<TenantName> reservedTo) {
         this.hostname = hostname;
@@ -58,6 +60,7 @@ public class Node {
         this.currentOsVersion = currentOsVersion;
         this.wantedOsVersion = wantedOsVersion;
         this.serviceState = serviceState;
+        this.suspendedSince = suspendedSince;
         this.restartGeneration = restartGeneration;
         this.wantedRestartGeneration = wantedRestartGeneration;
         this.rebootGeneration = rebootGeneration;
@@ -113,6 +116,10 @@ public class Node {
         return serviceState;
     }
 
+    public Optional<Instant> suspendedSince() {
+        return suspendedSince;
+    }
+
     public long restartGeneration() {
         return restartGeneration;
     }
@@ -209,6 +216,7 @@ public class Node {
         private Version currentOsVersion;
         private Version wantedOsVersion;
         private ServiceState serviceState;
+        private Optional<Instant> suspendedSince = Optional.empty();
         private long restartGeneration;
         private long wantedRestartGeneration;
         private long rebootGeneration;
@@ -235,6 +243,7 @@ public class Node {
             this.currentOsVersion = node.currentOsVersion;
             this.wantedOsVersion = node.wantedOsVersion;
             this.serviceState = node.serviceState;
+            this.suspendedSince = node.suspendedSince;
             this.restartGeneration = node.restartGeneration;
             this.wantedRestartGeneration = node.wantedRestartGeneration;
             this.rebootGeneration = node.rebootGeneration;
@@ -303,6 +312,11 @@ public class Node {
             return this;
         }
 
+        public Builder suspendedSince(Instant suspendedSince) {
+            this.suspendedSince = Optional.ofNullable(suspendedSince);
+            return this;
+        }
+
         public Builder restartGeneration(long restartGeneration) {
             this.restartGeneration = restartGeneration;
             return this;
@@ -360,7 +374,7 @@ public class Node {
 
         public Node build() {
             return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion,
-                    wantedOsVersion, serviceState, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration,
+                    wantedOsVersion, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration,
                     cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo);
         }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index 58122b04712..43d8d1c5a6e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMemb
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState;
 
+import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
@@ -108,6 +109,7 @@ public interface NodeRepository {
                         versionFrom(node.getCurrentOsVersion()),
                         versionFrom(node.getWantedOsVersion()),
                         fromBoolean(node.getAllowedToBeDown()),
+                        Optional.ofNullable(node.suspendedSince()).map(Instant::ofEpochMilli),
                         toInt(node.getCurrentRestartGeneration()),
                         toInt(node.getRestartGeneration()),
                         toInt(node.getCurrentRebootGeneration()),
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
index d9957d77ff3..985b7e3a339 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
@@ -6,8 +6,10 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.JsonNode;
 
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -76,6 +78,8 @@ public class NodeRepositoryNode {
     private NodeHistory[] history;
     @JsonProperty("allowedToBeDown")
     private Boolean allowedToBeDown;
+    @JsonProperty("suspendedSince")
+    private Long suspendedSince;
     @JsonProperty("reports")
     private Map<String, JsonNode> reports;
     @JsonProperty("modelName")
@@ -309,6 +313,14 @@ public class NodeRepositoryNode {
         return allowedToBeDown;
     }
 
+    public Long suspendedSince() {
+        return suspendedSince;
+    }
+
+    public void setSuspendedSince(long suspendedSinceMillis) {
+        this.suspendedSince = suspendedSinceMillis;
+    }
+
     public String getCurrentOsVersion() {
         return currentOsVersion;
     }