Remove ParkedExpirer, ProvisionedExpirer deprovisions if excessive expiry

1 files changed, 28 insertions, 2 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
index 856d534bbd2..c71eef0abea 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
@@ -4,15 +4,18 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 
 /**
  * This moves nodes of type {@link NodeType#host} from provisioned to parked if they have been in provisioned too long.
+ * Parked hosts are deprovisioned as well, if too many hosts are being expired.
  *
  * Only {@link NodeType#host} is moved because any number of nodes of that type can exist. Other node types such as
  * {@link NodeType#confighost} have a fixed number and thus cannot be replaced while the fixed number of nodes exist in
@@ -22,17 +25,40 @@ import java.util.List;
  */
 public class ProvisionedExpirer extends Expirer {
 
+    private final NodeRepository nodeRepository;
+    private static final int MAXIMUM_ALLOWED_EXPIRED_HOSTS = 20;
+
     ProvisionedExpirer(NodeRepository nodeRepository, Duration dirtyTimeout, Metric metric) {
         super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, dirtyTimeout, metric);
+        this.nodeRepository = nodeRepository;
     }
 
     @Override
     protected void expire(List<Node> expired) {
+        int previouslyExpired = numberOfPreviouslyExpired();
         for (Node expiredNode : expired) {
-            if (expiredNode.type() == NodeType.host) {
-                nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned");
+            if (expiredNode.type() != NodeType.host)
+                continue;
+            nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned");
+            if (MAXIMUM_ALLOWED_EXPIRED_HOSTS < ++previouslyExpired) {
+                nodeRepository.nodes().deprovision(expiredNode.hostname(), Agent.ProvisionedExpirer, Instant.now());
             }
         }
     }
 
+    private int numberOfPreviouslyExpired() {
+        return nodeRepository.nodes()
+                .list(Node.State.parked)
+                .nodeType(NodeType.host)
+                .matching(this::parkedByProvisionedExpirer)
+                .not().deprovisioning()
+                .size();
+    }
+
+    private boolean parkedByProvisionedExpirer(Node node) {
+        return node.history().event(History.Event.Type.parked)
+                .map(History.Event::agent)
+                .map(Agent.ProvisionedExpirer::equals)
+                .orElse(false);
+    }
 }