diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-01-22 23:47:44 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-01-22 23:47:44 +0100 |
commit | 85c9eb688804ebf00a3593c04086d4eb9354225d (patch) | |
tree | ce0005966eeea3ed9982a2d7e456dc2319587870 | |
parent | 97c2dd0fb54bd2aed689e5b8b432588cc4fb6401 (diff) |
Unreserve hosts with allocations
7 files changed, 34 insertions, 14 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index f9ddde9be16..9279c4e2332 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -341,7 +341,7 @@ public class NodeRepository extends AbstractComponent { /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ public List<Node> addNodes(List<Node> nodes) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { var node = nodes.get(i); var message = "Cannot add " + node.hostname() + ": A node with this name already exists"; @@ -361,7 +361,7 @@ public class NodeRepository extends AbstractComponent { /** Sets a list of nodes ready and returns the nodes in the ready state */ public List<Node> setReady(List<Node> nodes, Agent agent, String reason) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> nodesWithResetFields = nodes.stream() .map(node -> { if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty) @@ -585,7 +585,7 @@ public class NodeRepository extends AbstractComponent { } public List<Node> removeRecursively(Node node, boolean force) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> removed = new ArrayList<>(); if (node.type().isDockerHost()) { @@ -713,7 +713,7 @@ public class NodeRepository extends AbstractComponent { // perform operation while holding locks List<Node> resultingNodes = new ArrayList<>(); - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (Node node : unallocatedNodes) resultingNodes.add(action.apply(node, lock)); } @@ -738,12 +738,12 @@ public class NodeRepository extends AbstractComponent { /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); } - /** Create a lock which provides exclusive rights to allocating nodes */ - public Mutex lockAllocation() { return db.lockInactive(); } + /** Create a lock which provides exclusive rights to modifying unallocated nodes */ + public Mutex lockUnallocated() { return db.lockInactive(); } /** Acquires the appropriate lock for this node */ public Mutex lock(Node node) { - return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockAllocation(); + return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 1c27f9d1713..bb6d53d3304 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -59,7 +59,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { protected void maintain() { if (! dynamicProvisioningEnabled.value()) return; - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); updateProvisioningNodes(nodes, lock); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index dca5c092ad2..424d6409670 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -97,7 +97,7 @@ public class NodeFailer extends Maintainer { int throttledNodeFailures = 0; // Ready nodes - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { updateNodeLivenessEventsForReadyNodes(lock); for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index cb9294b3485..ebd56229312 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; -import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -90,6 +89,27 @@ class Activator { nodeRepository.deactivate(activeToRemove, transaction); nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); + unreserveParentsOf(reservedToActivate); + } + + /** When a tenant node is activated on a host, we can open up that host for use by others */ + private void unreserveParentsOf(List<Node> nodes) { + List<String> reservedParents = nodes.stream() + .filter(node -> node.parentHostname().isPresent()) + .map(node -> nodeRepository.getNode(node.parentHostname().get())) + .filter(parent -> parent.isPresent()) + .filter(parent -> parent.get().reservedTo().isPresent()) + .map(parent -> parent.get().hostname()) + .collect(Collectors.toList()); + if (reservedParents.isEmpty()) return; + + try (Mutex lock = nodeRepository.lockUnallocated()) { + List<Node> unreserved = reservedParents.stream() + .map(hostname -> nodeRepository.getNode(hostname).get()) + .map(host -> host.withReservedTo(Optional.empty())) + .collect(Collectors.toList()); + nodeRepository.write(unreserved, lock); + } } /** Activate load balancers */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 4ecd10bdd86..74d3ac73641 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -73,7 +73,7 @@ public class GroupPreparer { try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others - try (Mutex allocationLock = nodeRepository.lockAllocation()) { + try (Mutex allocationLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes LockedNodeList nodeList = nodeRepository.list(allocationLock); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index aeb045f1b4b..2a8783748ef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -127,12 +127,12 @@ public class MetricsReporterTest { Node container1 = Node.createDockerNode(Set.of("::2"), "container1", "dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1"), container1).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated())); Node container2 = Node.createDockerNode(Set.of("::3"), "container2", "dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2"), container2).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated())); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 6dfca4d2c04..a26e802dfe8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -247,7 +247,7 @@ public class LoadBalancerProvisionerTest { } private void assignIps(List<Node> nodes) { - try (var lock = tester.nodeRepository().lockAllocation()) { + try (var lock = tester.nodeRepository().lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock); } |