Add ParkedExpirer

6 files changed, 141 insertions, 0 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 44ee8b5a8b3..40f9d17519c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -67,6 +67,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric));
         maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer));
         maintainers.add(new HostEncrypter(nodeRepository, defaults.hostEncrypterInterval, metric));
+        maintainers.add(new ParkedExpirer(nodeRepository, defaults.parkedExpirerInterval, metric));
 
         provisionServiceProvider.getLoadBalancerService(nodeRepository)
                                 .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric))
@@ -119,6 +120,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration scalingSuggestionsInterval;
         private final Duration switchRebalancerInterval;
         private final Duration hostEncrypterInterval;
+        private final Duration parkedExpirerInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -149,6 +151,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
             inactiveConfigServerExpiry = Duration.ofMinutes(5);
             inactiveControllerExpiry = Duration.ofMinutes(5);
+            parkedExpirerInterval = Duration.ofMinutes(30);
 
             if (zone.environment().isProduction() && ! zone.system().isCd()) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirer.java
new file mode 100644
index 00000000000..9c7e12cc5e1
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirer.java
@@ -0,0 +1,62 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.logging.Logger;
+
+/**
+ *
+ * Expires parked nodes in dynamically provisioned zones.
+ * If number of parked hosts exceed MAX_ALLOWED_PARKED_HOSTS, recycle in a queue order
+ *
+ * @author olaa
+ */
+public class ParkedExpirer extends NodeRepositoryMaintainer {
+
+    private static final int MAX_ALLOWED_PARKED_HOSTS = 20;
+    private static final Logger log = Logger.getLogger(ParkedExpirer.class.getName());
+
+
+    private final NodeRepository nodeRepository;
+
+    ParkedExpirer(NodeRepository nodeRepository, Duration interval, Metric metric) {
+        super(nodeRepository, interval, metric);
+        this.nodeRepository = nodeRepository;
+    }
+
+    @Override
+    protected double maintain() {
+        if (!nodeRepository.zone().getCloud().dynamicProvisioning())
+            return 1.0;
+
+        var parkedHosts = new ArrayList<>(nodeRepository.nodes().list(Node.State.parked)
+                                                        .nodeType(NodeType.host)
+                                                        .asList());
+
+        parkedHosts.sort(Comparator.comparing(this::getParkedTime));
+        var hostsToExpire = parkedHosts.size() - MAX_ALLOWED_PARKED_HOSTS;
+        for(int i = 0; i < hostsToExpire; i++) {
+            var parkedHost = parkedHosts.get(i);
+            log.info("Allowed number of parked nodes exceeded. Recycling " + parkedHost.hostname());
+            nodeRepository.nodes().deallocate(parkedHost, Agent.ParkedExpirer, "Expired by ParkedExpirer");
+        }
+
+        return 1.0;
+    }
+
+    private Instant getParkedTime(Node node) {
+        return node.history().event(History.Event.Type.parked)
+                .map(History.Event::at)
+                .orElse(Instant.EPOCH); // Should not happen
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
index d1c3f00ddca..ed82470fa42 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
@@ -21,6 +21,7 @@ public enum Agent {
     InactiveExpirer,
     ProvisionedExpirer,
     ReservationExpirer,
+    ParkedExpirer,
     DynamicProvisioningMaintainer,
     RetiringUpgrader,
     RebuildingOsUpgrader,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index d83f21e5fec..dff4a66bd42 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -482,6 +482,7 @@ public class NodeSerializer {
             case "SpareCapacityMaintainer": return Agent.SpareCapacityMaintainer;
             case "SwitchRebalancer": return Agent.SwitchRebalancer;
             case "HostEncrypter": return Agent.HostEncrypter;
+            case "ParkedExpirer": return Agent.ParkedExpirer;
         }
         throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'");
     }
@@ -504,6 +505,7 @@ public class NodeSerializer {
             case SpareCapacityMaintainer: return "SpareCapacityMaintainer";
             case SwitchRebalancer: return "SwitchRebalancer";
             case HostEncrypter: return "HostEncrypter";
+            case ParkedExpirer: return "ParkedExpirer";
         }
         throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirerTest.java
new file mode 100644
index 00000000000..7fb22f33453
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ParkedExpirerTest.java
@@ -0,0 +1,70 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.Cloud;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class ParkedExpirerTest {
+
+    private ProvisioningTester tester;
+
+    @Test
+    public void noop_if_not_dynamic_provisioning() {
+        tester = getTester(false);
+        populateNodeRepo();
+
+        var expirer = new ParkedExpirer(tester.nodeRepository(), Duration.ofMinutes(4), new TestMetric());
+        expirer.maintain();
+
+        assertEquals(0, tester.nodeRepository().nodes().list(Node.State.dirty).size());
+        assertEquals(25, tester.nodeRepository().nodes().list(Node.State.parked).size());
+    }
+
+    @Test
+    public void recycles_correct_subset_of_parked_hosts() {
+        tester = getTester(true);
+        populateNodeRepo();
+
+        var expirer = new ParkedExpirer(tester.nodeRepository(), Duration.ofMinutes(4), new TestMetric());
+        expirer.maintain();
+
+        assertEquals(5, tester.nodeRepository().nodes().list(Node.State.dirty).size());
+        assertEquals(20, tester.nodeRepository().nodes().list(Node.State.parked).size());
+
+    }
+
+    private ProvisioningTester getTester(boolean dynamicProvisioning) {
+        var zone = new Zone(Cloud.builder().dynamicProvisioning(dynamicProvisioning).build(), SystemName.main, Environment.prod, RegionName.from("us-east"));
+        return new ProvisioningTester.Builder().zone(zone)
+                .hostProvisioner(dynamicProvisioning ? new MockHostProvisioner(List.of()) : null)
+                .build();
+    }
+
+    private void populateNodeRepo() {
+        var nodes = IntStream.range(0, 25)
+                                        .mapToObj(i -> Node.create("id-" + i, "host-" + i, new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.host).build())
+                                        .collect(Collectors.toList());
+        tester.nodeRepository().database().addNodesInState(nodes, Node.State.parked, Agent.system);
+    }
+
+}
\ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
index 2dcf2d0b838..73ac692e37b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
@@ -43,6 +43,9 @@
       "name": "OsUpgradeActivator"
     },
     {
+      "name": "ParkedExpirer"
+    },
+    {
       "name": "PeriodicApplicationMaintainer"
     },
     {