diff options
author | jonmv <venstad@gmail.com> | 2023-07-12 18:17:26 +0200 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2023-07-12 18:17:26 +0200 |
commit | 14f7beef22397deba6f80fb3f0de5a51053a7960 (patch) | |
tree | 46f60f514f5f85de4124c781ff4dbb0b9d2a65da | |
parent | ccb2515582ae7abec1bb2165991c10a36eb99978 (diff) |
Limit the maximum nomber of deprovisioned nodes in a zone to 1000
2 files changed, 33 insertions, 6 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java index c2b383a5a54..92062f13f1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java @@ -3,12 +3,19 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.History.Event.Type; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; +import static java.util.Comparator.comparing; + /** * This removes hosts from {@link com.yahoo.vespa.hosted.provision.Node.State#deprovisioned}, in dynamically provisioned * zones, after a grace period. @@ -17,21 +24,37 @@ import java.util.List; */ public class DeprovisionedExpirer extends Expirer { + private static final int maxDeprovisionedNodes = 1000; + DeprovisionedExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) { super(Node.State.deprovisioned, History.Event.Type.deprovisioned, nodeRepository, expiryTime, metric); } @Override protected boolean isExpired(Node node) { - return nodeRepository().zone().cloud().dynamicProvisioning() && - super.isExpired(node); + return nodeRepository().zone().cloud().dynamicProvisioning() && super.isExpired(node); } @Override - protected void expire(List<Node> expired) { - for (var node : expired) { - nodeRepository().nodes().forget(node); + protected NodeList getExpiredNodes() { + List<Node> deprovisioned = nodeRepository().nodes().list(Node.State.deprovisioned) + .sortedBy(comparing(node -> node.history().event(Type.deprovisioned) + .map(History.Event::at) + .orElse(Instant.EPOCH))) + .asList(); + Deque<Node> expired = new ArrayDeque<>(deprovisioned); + int kept = 0; + while ( ! expired.isEmpty()) { + if (isExpired(expired.getLast()) || kept++ >= maxDeprovisionedNodes) break; // If we encounter an expired node, the rest are also expired. + expired.removeLast(); } + return NodeList.copyOf(List.copyOf(expired)); + } + + @Override + protected void expire(List<Node> expired) { + nodeRepository().nodes().performOn(NodeList.copyOf(expired), + (node, lock) -> { nodeRepository().nodes().forget(node); return node; }); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index a8929cf9d22..1684ebbb38f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -40,7 +40,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { @Override protected double maintain() { - NodeList expired = nodeRepository().nodes().list(fromState).matching(this::isExpired); + NodeList expired = getExpiredNodes(); if ( ! expired.isEmpty()) { log.info(fromState + " expirer found " + expired.size() + " expired nodes: " + expired); @@ -51,6 +51,10 @@ public abstract class Expirer extends NodeRepositoryMaintainer { return 1.0; } + protected NodeList getExpiredNodes() { + return nodeRepository().nodes().list(fromState).matching(this::isExpired); + } + protected boolean isExpired(Node node) { return isExpired(node, expiryTime); } |