Unreserve hosts with allocations

7 files changed, 34 insertions, 14 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index f9ddde9be16..9279c4e2332 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -341,7 +341,7 @@ public class NodeRepository extends AbstractComponent {
 
     /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */
     public List<Node> addNodes(List<Node> nodes) {
-        try (Mutex lock = lockAllocation()) {
+        try (Mutex lock = lockUnallocated()) {
             for (int i = 0; i < nodes.size(); i++) {
                 var node = nodes.get(i);
                 var message = "Cannot add " + node.hostname() + ": A node with this name already exists";
@@ -361,7 +361,7 @@ public class NodeRepository extends AbstractComponent {
 
     /** Sets a list of nodes ready and returns the nodes in the ready state */
     public List<Node> setReady(List<Node> nodes, Agent agent, String reason) {
-        try (Mutex lock = lockAllocation()) {
+        try (Mutex lock = lockUnallocated()) {
             List<Node> nodesWithResetFields = nodes.stream()
                     .map(node -> {
                         if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty)
@@ -585,7 +585,7 @@ public class NodeRepository extends AbstractComponent {
     }
 
     public List<Node> removeRecursively(Node node, boolean force) {
-        try (Mutex lock = lockAllocation()) {
+        try (Mutex lock = lockUnallocated()) {
             List<Node> removed = new ArrayList<>();
 
              if (node.type().isDockerHost()) {
@@ -713,7 +713,7 @@ public class NodeRepository extends AbstractComponent {
 
         // perform operation while holding locks
         List<Node> resultingNodes = new ArrayList<>();
-        try (Mutex lock = lockAllocation()) {
+        try (Mutex lock = lockUnallocated()) {
             for (Node node : unallocatedNodes)
                 resultingNodes.add(action.apply(node, lock));
         }
@@ -738,12 +738,12 @@ public class NodeRepository extends AbstractComponent {
     /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */
     public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); }
 
-    /** Create a lock which provides exclusive rights to allocating nodes */
-    public Mutex lockAllocation() { return db.lockInactive(); }
+    /** Create a lock which provides exclusive rights to modifying unallocated nodes */
+    public Mutex lockUnallocated() { return db.lockInactive(); }
 
     /** Acquires the appropriate lock for this node */
     public Mutex lock(Node node) {
-        return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockAllocation();
+        return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated();
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 1c27f9d1713..bb6d53d3304 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -59,7 +59,7 @@ public class DynamicProvisioningMaintainer extends Maintainer {
     protected void maintain() {
         if (! dynamicProvisioningEnabled.value()) return;
 
-        try (Mutex lock = nodeRepository().lockAllocation()) {
+        try (Mutex lock = nodeRepository().lockUnallocated()) {
             NodeList nodes = nodeRepository().list();
 
             updateProvisioningNodes(nodes, lock);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index dca5c092ad2..424d6409670 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -97,7 +97,7 @@ public class NodeFailer extends Maintainer {
         int throttledNodeFailures = 0;
 
         // Ready nodes
-        try (Mutex lock = nodeRepository().lockAllocation()) {
+        try (Mutex lock = nodeRepository().lockUnallocated()) {
             updateNodeLivenessEventsForReadyNodes(lock);
 
             for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
index cb9294b3485..ebd56229312 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.ParentHostUnavailableException;
-import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -90,6 +89,27 @@ class Activator {
         nodeRepository.deactivate(activeToRemove, transaction);
         nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes
         nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction);
+        unreserveParentsOf(reservedToActivate);
+    }
+
+    /** When a tenant node is activated on a host, we can open up that host for use by others */
+    private void unreserveParentsOf(List<Node> nodes) {
+        List<String> reservedParents = nodes.stream()
+                                            .filter(node -> node.parentHostname().isPresent())
+                                            .map(node -> nodeRepository.getNode(node.parentHostname().get()))
+                                            .filter(parent -> parent.isPresent())
+                                            .filter(parent -> parent.get().reservedTo().isPresent())
+                                            .map(parent -> parent.get().hostname())
+                                            .collect(Collectors.toList());
+        if (reservedParents.isEmpty()) return;
+
+        try (Mutex lock = nodeRepository.lockUnallocated()) {
+            List<Node> unreserved = reservedParents.stream()
+                                                   .map(hostname -> nodeRepository.getNode(hostname).get())
+                                                   .map(host -> host.withReservedTo(Optional.empty()))
+                                                   .collect(Collectors.toList());
+            nodeRepository.write(unreserved, lock);
+        }
     }
 
     /** Activate load balancers */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 4ecd10bdd86..74d3ac73641 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -73,7 +73,7 @@ public class GroupPreparer {
         try (Mutex lock = nodeRepository.lock(application)) {
 
             // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
-            try (Mutex allocationLock = nodeRepository.lockAllocation()) {
+            try (Mutex allocationLock = nodeRepository.lockUnallocated()) {
 
                 // Create a prioritized set of nodes
                 LockedNodeList nodeList = nodeRepository.list(allocationLock);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index aeb045f1b4b..2a8783748ef 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -127,12 +127,12 @@ public class MetricsReporterTest {
         Node container1 = Node.createDockerNode(Set.of("::2"), "container1",
                                                 "dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant);
         container1 = container1.with(allocation(Optional.of("app1"), container1).get());
-        nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation()));
+        nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated()));
 
         Node container2 = Node.createDockerNode(Set.of("::3"), "container2",
                                                 "dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant);
         container2 = container2.with(allocation(Optional.of("app2"), container2).get());
-        nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation()));
+        nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated()));
 
         Orchestrator orchestrator = mock(Orchestrator.class);
         ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index 6dfca4d2c04..a26e802dfe8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -247,7 +247,7 @@ public class LoadBalancerProvisionerTest {
     }
 
     private void assignIps(List<Node> nodes) {
-        try (var lock = tester.nodeRepository().lockAllocation()) {
+        try (var lock = tester.nodeRepository().lockUnallocated()) {
             for (int i = 0; i < nodes.size(); i++) {
                 tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock);
             }