Let NodeRetirer redeploy applications where it retires a node

3 files changed, 18 insertions, 7 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index da3431b3d3a..74a6dfaeecb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -78,7 +78,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
 
         FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker(
                 NodeRetirer.SPARE_NODES_POLICY, FlavorSpareCount.constructFlavorSpareCountGraph(zone.nodeFlavors().get().getFlavors()));
-        nodeRetirer = new NodeRetirer(nodeRepository, zone, flavorSpareChecker, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), jobControl,
+        nodeRetirer = new NodeRetirer(nodeRepository, zone, flavorSpareChecker, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), deployer, jobControl,
                 new RetireIPv4OnlyNodes(),
                 new Zone(SystemName.cd, Environment.dev, RegionName.from("cd-us-central-1")),
                 new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-central-1")),
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
index c320fbcaff8..541addb247d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
@@ -1,6 +1,8 @@
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
@@ -16,6 +18,7 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -30,11 +33,12 @@ public class NodeRetirer extends Maintainer {
     private static final long MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION = 1;
     private static final Logger log = Logger.getLogger(NodeRetirer.class.getName());
 
+    private final Deployer deployer;
     private final FlavorSpareChecker flavorSpareChecker;
     private final RetirementPolicy retirementPolicy;
 
     public NodeRetirer(NodeRepository nodeRepository, Zone zone, FlavorSpareChecker flavorSpareChecker, Duration interval,
-                       JobControl jobControl, RetirementPolicy retirementPolicy, Zone... applies) {
+                       Deployer deployer, JobControl jobControl, RetirementPolicy retirementPolicy, Zone... applies) {
         super(nodeRepository, interval, jobControl);
         if (! Arrays.asList(applies).contains(zone)) {
             String targetZones = Arrays.stream(applies).map(Zone::toString).collect(Collectors.joining(", "));
@@ -42,6 +46,7 @@ public class NodeRetirer extends Maintainer {
             deconstruct();
         }
 
+        this.deployer = deployer;
         this.retirementPolicy = retirementPolicy;
         this.flavorSpareChecker = flavorSpareChecker;
     }
@@ -60,8 +65,8 @@ public class NodeRetirer extends Maintainer {
     boolean retireUnallocated() {
         try (Mutex lock = nodeRepository().lockUnallocated()) {
             List<Node> allNodes = nodeRepository().getNodes(NodeType.tenant);
-            Map<Flavor, Map<Node.State, Long>> numNodesByFlavorByState = getNumberOfNodesByFlavorByNodeState(allNodes);
-            flavorSpareChecker.updateReadyAndActiveCountsByFlavor(numNodesByFlavorByState);
+            Map<Flavor, Map<Node.State, Long>> numSpareNodesByFlavorByState = getNumberOfNodesByFlavorByNodeState(allNodes);
+            flavorSpareChecker.updateReadyAndActiveCountsByFlavor(numSpareNodesByFlavorByState);
 
             long numFlavorsWithUnsuccessfullyRetiredNodes = allNodes.stream()
                     .filter(node -> node.state() == Node.State.ready)
@@ -102,15 +107,18 @@ public class NodeRetirer extends Maintainer {
         flavorSpareChecker.updateReadyAndActiveCountsByFlavor(numSpareNodesByFlavorByState);
 
         for (ApplicationId applicationId : activeApplications) {
+            Optional<Deployment> deployment = deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30));
+            if ( ! deployment.isPresent()) continue; // this will be done at another config server
+
             try (Mutex lock = nodeRepository().lock(applicationId)) {
                 // Get nodes for current application under lock
                 List<Node> applicationNodes = nodeRepository().getNodes(applicationId);
                 Set<Node> retireableNodes = getRetireableNodesForApplication(applicationNodes);
                 long numNodesAllowedToRetire = getNumberNodesAllowToRetireForApplication(applicationNodes, MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION);
+                long numNodesWantedToRetire = 0;
 
-                for (Iterator<Node> iterator = retireableNodes.iterator(); iterator.hasNext() && numNodesAllowedToRetire > 0; ) {
+                for (Iterator<Node> iterator = retireableNodes.iterator(); iterator.hasNext() && numNodesAllowedToRetire > numNodesWantedToRetire; ) {
                     Node retireableNode = iterator.next();
-                    System.out.println(retireableNode);
 
                     if (flavorSpareChecker.canRetireAllocatedNodeWithFlavor(retireableNode.flavor())) {
                         log.info("Setting wantToRetire for host " + retireableNode.hostname() +
@@ -121,8 +129,11 @@ public class NodeRetirer extends Maintainer {
                                 .withWantToRetire(true)
                                 .withWantToDeprovision(true));
                         nodeRepository().write(updatedNode);
+                        numNodesWantedToRetire++;
                     }
                 }
+
+                if (numNodesWantedToRetire > 0) deployment.get().activate();
             }
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
index 5038b0f950b..f71d9b75f78 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
@@ -64,7 +64,7 @@ public class NodeRetirerTest {
                 .forEach(node -> tester.nodeRepository.write(node.withIpAddresses(Collections.singleton("::2"))));
 
         tester.assertCountsForStateByFlavor(Node.State.active, 9, -1, 2, 11, -1);
-        System.out.println("start");
+
         tester.setNumberAllowedAllocatedRetirementsPerFlavor(3, 2, 3, 2);
         retirer.retireAllocated();
         tester.assertParkedCountsByApplication(-1, -1, -1, -1, -1); // Nodes should be in retired, but not yet parked