1 files changed, 28 insertions, 20 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
index 00543058520..7aead77d080 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
@@ -1,18 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.collections.ListMap;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 
+import java.time.Clock;
 import java.time.Duration;
-import java.util.ArrayList;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -24,33 +25,35 @@ import java.util.stream.Collectors;
  *
  * @author hakon
  */
-// TODO: This should be consolidated with RetiredExpirer. The only difference between this and RetiredExpirer is that
-// this runs more often by default and asks orchestrator for permission to retire nodes.
 public class RetiredEarlyExpirer extends Maintainer {
 
     private final Deployer deployer;
     private final Orchestrator orchestrator;
+    private final Duration retiredDuration;
+    private final Clock clock;
 
     public RetiredEarlyExpirer(NodeRepository nodeRepository,
-                               Duration interval,
+                               Duration maintenanceInterval,
+                               Duration retiredDuration,
+                               Clock clock,
                                JobControl jobControl,
                                Deployer deployer,
                                Orchestrator orchestrator) {
-        super(nodeRepository, interval, jobControl);
+        super(nodeRepository, maintenanceInterval, jobControl);
         this.deployer = deployer;
         this.orchestrator = orchestrator;
+        this.retiredDuration = retiredDuration;
+        this.clock = clock;
     }
 
     @Override
     protected void maintain() {
         List<Node> activeNodes = nodeRepository().getNodes(Node.State.active);
 
-        ListMap<ApplicationId, Node> retiredNodesByApplication = new ListMap<>();
-        for (Node node : activeNodes) {
-            if (node.allocation().isPresent() && node.allocation().get().membership().retired()) {
-                retiredNodesByApplication.put(node.allocation().get().owner(), node);
-            }
-        }
+        Map<ApplicationId, List<Node>> retiredNodesByApplication = activeNodes.stream()
+                .filter(node -> node.allocation().isPresent())
+                .filter(node -> node.allocation().get().membership().retired())
+                .collect(Collectors.groupingBy(node -> node.allocation().get().owner()));
 
         for (Map.Entry<ApplicationId, List<Node>> entry : retiredNodesByApplication.entrySet()) {
             ApplicationId application = entry.getKey();
@@ -60,13 +63,7 @@ public class RetiredEarlyExpirer extends Maintainer {
                 Optional<Deployment> deployment = deployer.deployFromLocalActive(application);
                 if ( ! deployment.isPresent()) continue; // this will be done at another config server
 
-                List<Node> nodesToRemove = new ArrayList<>();
-                for (Node node : retiredNodes) {
-                    if (canRemove(node)) {
-                        nodesToRemove.add(node);
-                    }
-                }
-                
+                List<Node> nodesToRemove = retiredNodes.stream().filter(this::canRemove).collect(Collectors.toList());
                 if (nodesToRemove.isEmpty()) {
                     continue;
                 }
@@ -85,8 +82,19 @@ public class RetiredEarlyExpirer extends Maintainer {
         }
     }
 
-    /** Returns whether orchestrator permits given node to be removed */
+    /**
+     * Checks if the node can be removed, this is allowed if either of these are true:
+     * - The node has been in state {@link History.Event.Type#retired} for longer than {@link #retiredDuration}
+     * - Orchestrator allows it
+     */
     private boolean canRemove(Node node) {
+        Optional<Instant> timeOfRetiredEvent = node.history().event(History.Event.Type.retired).map(History.Event::at);
+        Optional<Instant> retireAfter = timeOfRetiredEvent.map(retiredEvent -> retiredEvent.plus(retiredDuration));
+        boolean shouldRetireNowBecauseExpried = retireAfter.map(time -> time.isBefore(clock.instant())).orElse(false);
+        if (shouldRetireNowBecauseExpried) {
+            return true;
+        }
+
         try {
             orchestrator.acquirePermissionToRemove(new HostName(node.hostname()));
             return true;