From 37887a615a5c433a5c08dc786d8e8a2124f174f1 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 25 Jul 2017 13:42:05 +0200 Subject: Clean up old allocation code --- .../provision/provisioning/DockerAllocator.java | 153 --------------------- .../provisioning/DockerCapacityConstraints.java | 107 -------------- .../provision/provisioning/DockerHostCapacity.java | 25 +++- .../provision/provisioning/NodePrioritizer.java | 51 ++++++- .../provision/provisioning/NodePriority.java | 20 +-- .../provisioning/AllocationSimulator.java | 9 +- 6 files changed, 79 insertions(+), 286 deletions(-) delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java (limited to 'node-repository') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java deleted file mode 100644 index ef74fb8c5c7..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; - -/** - * Set of methods to allocate new docker nodes - *

- * The nodes are not added to the repository here - this is done by caller. - * - * @author smorgrav - */ -public class DockerAllocator { - - /** - * The docker container allocation algorithm - */ - static List allocateNewDockerNodes(NodeAllocation allocation, - NodeSpec requestedNodes, - List allNodes, - List nodesBefore, - NodeFlavors flavors, - Flavor flavor, - int nofSpares, - BiConsumer, String> recorder) { - // Try allocate new nodes with all constraints in place - List nodesWithHeadroomAndSpares = DockerCapacityConstraints.addHeadroomAndSpareNodes(allNodes, flavors, nofSpares); - recorder.accept(nodesWithHeadroomAndSpares, "Headroom and spares"); - List accepted = DockerAllocator.allocate(allocation, flavor, nodesWithHeadroomAndSpares); - - List allNodesIncludingAccepted = new ArrayList<>(allNodes); - allNodesIncludingAccepted.addAll(accepted); - recorder.accept(allNodesIncludingAccepted, "1st dynamic docker allocation - fullfilled: " + allocation.fullfilled()); - - // If still not fully allocated - try to allocate the remaining nodes with only hard constraints - if (!allocation.fullfilled()) { - List nodesWithSpares = DockerCapacityConstraints.addSpareNodes(allNodesIncludingAccepted, nofSpares); - recorder.accept(nodesWithSpares, "Spares only"); - - List acceptedWithHard = DockerAllocator.allocate(allocation, flavor, nodesWithSpares); - accepted.addAll(acceptedWithHard); - allNodesIncludingAccepted.addAll(acceptedWithHard); - recorder.accept(allNodesIncludingAccepted, "2nd dynamic docker allocation - fullfilled: " + allocation.fullfilled()); - - // If still not fully allocated and this is a replacement - drop all constraints - boolean isReplacement = DockerAllocator.isReplacement(requestedNodes, nodesBefore, allNodes); - if (!allocation.fullfilled() && isReplacement) { - List finalTry = DockerAllocator.allocate(allocation, flavor, allNodesIncludingAccepted); - accepted.addAll(finalTry); - allNodesIncludingAccepted.addAll(finalTry); - recorder.accept(allNodesIncludingAccepted, "Final dynamic docker alloction - fullfilled: " + allocation.fullfilled()); - } - } - - return accepted; - } - - /** - * Offer the node allocation a prioritized set of new nodes according to capacity constraints - * - * @param allocation The allocation we want to fulfill - * @param flavor Since we create nodes here we need to know the exact flavor - * @param nodes The nodes relevant for the allocation (all nodes from node repo give or take) - * @return Nodes accepted by the node allocation - these nodes does not exist in the noderepo yet. - * @see DockerHostCapacity - */ - public static List allocate(NodeAllocation allocation, Flavor flavor, List nodes) { - - DockerHostCapacity dockerCapacity = new DockerHostCapacity(nodes); - - // Get all active docker hosts with enough capacity and ip slots - sorted on free capacity - List dockerHosts = nodes.stream() - .filter(node -> node.type().equals(NodeType.host)) - .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) - .filter(dockerHost -> dockerCapacity.hasCapacity(dockerHost, flavor)) - .sorted(dockerCapacity::compare) - .collect(Collectors.toList()); - - // Create one node pr. docker host that we can offer to the allocation - List offers = new LinkedList<>(); - for (Node parentHost : dockerHosts) { - Set ipAddresses = DockerHostCapacity.findFreeIps(parentHost, nodes); - if (ipAddresses.isEmpty()) continue; - String ipAddress = ipAddresses.stream().findFirst().get(); - String hostname = lookupHostname(ipAddress); - if (hostname == null) continue; - Node node = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), - Collections.emptySet(), hostname, Optional.of(parentHost.hostname()), flavor, NodeType.tenant); - offers.add(node); - } - - return null;//allocation.offer(offers, false); - } - - /** - * From ipAddress - get hostname - * - * @return hostname or null if not able to do the loopup - */ - private static String lookupHostname(String ipAddress) { - try { - return InetAddress.getByName(ipAddress).getHostName(); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - return null; - } - - /** - * This is an heuristic way to find if new nodes are to replace failing nodes - * or are to expand the cluster. - * - * The current implementation does not account for failed nodes that are not in the application - * anymore. The consequence is that we will ignore the spare capacity constraints too often - in - * particular when the number of failed nodes (not in the application anymore) - * for the cluster equal to the upscaling of the cluster. - * - * The deployment algorithm will still try to allocate the the capacity outside the spare capacity if possible. - * - * TODO propagate this information either through the node object or from the configserver deployer - */ - private static boolean isReplacement(NodeSpec nodeSpec, List nodesBefore, List nodesReserved) { - int wantedCount = 0; - if (nodeSpec instanceof NodeSpec.CountNodeSpec) { - NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; - wantedCount = countSpec.getCount(); - } - - List failedNodes = new ArrayList<>(); - for (Node node : nodesBefore) { - if (node.state() == Node.State.failed) { - failedNodes.add(node); - } - } - - if (failedNodes.size() == 0) return false; - return (wantedCount <= nodesReserved.size() + failedNodes.size()); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java deleted file mode 100644 index 943b59bb2e1..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.cloud.config.ApplicationIdConfig; -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeType; -import com.yahoo.lang.MutableInteger; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; - -import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Enforce allocation constraints for docker by manipulating the NodeList we operate on. - * - * The constraints comes in two flavors: headroom and spare. - * - * Headroom is the number of docker nodes (of various flavors) we want to reserve for new applications. - * This is e.g. to make sure we don't smear out small docker flavors on all hosts - * starving allocations for bigger flavors. - * - * Spares is to make sure we have replacement for applications if one or more hosts go down. - * It is more important to safeguard already onboarded applications than accept new applications. - * - * For now - we will use spare also as a means to reserve capacity for future applications that - * have had a separate AI process. - * - * When using spares - we will relay on maintenance jobs to reclaim the spare capacity whenever the - * capacity has been recovered (e.g. when the dead docker host is replaced) - * - * @author smorgrav - */ -public class DockerCapacityConstraints { - - /** This is a static utility class */ - private DockerCapacityConstraints() {} - - public static List findSpareHosts(List nodes, int spares) { - DockerHostCapacity capacity = new DockerHostCapacity(nodes); - return nodes.stream() - .filter(node -> node.type().equals(NodeType.host)) - .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) - .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) - .sorted(capacity::compare) - .limit(spares) - .collect(Collectors.toList()); - } - - /** - * Spare nodes in first iteration is a node that fills up the two - * largest hosts (in terms of free capacity) - */ - public static List addSpareNodes(List nodes, int spares) { - List spareFlavors = findSpareHosts(nodes, spares).stream() - .map(dockerHost -> freeCapacityAsFlavor(dockerHost, nodes)) - .collect(Collectors.toList()); - - return addNodes(nodes, spareFlavors, "spare"); - } - - public static List addHeadroomAndSpareNodes(List nodes, NodeFlavors flavors, int nofSpares) { - List sparesAndHeadroom = addSpareNodes(nodes, nofSpares); - return addNodes(sparesAndHeadroom, flavors.getFlavors(), "headroom"); - } - - private static List addNodes(List nodes, List flavors, String id) { - List headroom = new ArrayList<>(nodes); - for (Flavor flavor : flavors) { - int headroomCount = flavor.getIdealHeadroom(); - if (headroomCount > 0) { - NodeAllocation allocation = createHeadRoomAllocation(flavor, headroomCount, id); - List acceptedNodes = DockerAllocator.allocate(allocation, flavor, headroom); - headroom.addAll(acceptedNodes); - } - } - return headroom; - } - - private static Flavor freeCapacityAsFlavor(Node host, List nodes) { - ResourceCapacity hostCapacity = new ResourceCapacity(host); - for (Node container : new NodeList(nodes).childNodes(host).asList()) { - hostCapacity.subtract(container); - } - return hostCapacity.asFlavor(); - } - - private static NodeAllocation createHeadRoomAllocation(Flavor flavor, int count, String id) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, - new ClusterSpec.Id(id), new Version()); - ApplicationId appId = new ApplicationId( - new ApplicationIdConfig( - new ApplicationIdConfig.Builder() - .tenant(id) - .application(id + "-" + flavor.name()) - .instance("temporarynode"))); - - return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor), - new MutableInteger(0), Clock.systemUTC()); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 9d49fdf0c49..e91c3eaa65f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -38,9 +38,21 @@ public class DockerHostCapacity { * Used in prioritizing hosts for allocation in descending order. */ int compare(Node hostA, Node hostB) { - int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true)); + int comp = freeCapacityOf(hostB, true, false).compare(freeCapacityOf(hostA, true, false)); if (comp == 0) { - comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false)); + comp = freeCapacityOf(hostB, false, false).compare(freeCapacityOf(hostA, false, false)); + if (comp == 0) { + // If resources are equal - we want to assign to the one with the most IPaddresses free + comp = freeIPs(hostB) - freeIPs(hostA); + } + } + return comp; + } + + int compareWithoutRetired(Node hostA, Node hostB) { + int comp = freeCapacityOf(hostB, true, true).compare(freeCapacityOf(hostA, true, true)); + if (comp == 0) { + comp = freeCapacityOf(hostB, false, true).compare(freeCapacityOf(hostA, false, true)); if (comp == 0) { // If resources are equal - we want to assign to the one with the most IPaddresses free comp = freeIPs(hostB) - freeIPs(hostA); @@ -54,7 +66,7 @@ public class DockerHostCapacity { * if we could allocate a flavor on the docker host. */ boolean hasCapacity(Node dockerHost, Flavor flavor) { - return freeCapacityOf(dockerHost, true).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; + return freeCapacityOf(dockerHost, true, false).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; } /** @@ -67,7 +79,7 @@ public class DockerHostCapacity { public ResourceCapacity getFreeCapacityTotal() { return allNodes.asList().stream() .filter(n -> n.type().equals(NodeType.host)) - .map(n -> freeCapacityOf(n, false)) + .map(n -> freeCapacityOf(n, false, false)) .reduce(new ResourceCapacity(), ResourceCapacity::add); } @@ -94,7 +106,7 @@ public class DockerHostCapacity { } private int canFitNumberOf(Node node, Flavor flavor) { - int capacityFactor = freeCapacityOf(node, false).freeCapacityInFlavorEquivalence(flavor); + int capacityFactor = freeCapacityOf(node, false, false).freeCapacityInFlavorEquivalence(flavor); int ips = freeIPs(node); return Math.min(capacityFactor, ips); } @@ -106,7 +118,7 @@ public class DockerHostCapacity { * * @return A default (empty) capacity if not a docker host, otherwise the free/unallocated/rest capacity */ - public ResourceCapacity freeCapacityOf(Node dockerHost, boolean headroomAsReservedCapacity) { + public ResourceCapacity freeCapacityOf(Node dockerHost, boolean headroomAsReservedCapacity, boolean retiredAsFreeCapacity) { // Only hosts have free capacity if (!dockerHost.type().equals(NodeType.host)) return new ResourceCapacity(); @@ -114,6 +126,7 @@ public class DockerHostCapacity { for (Node container : allNodes.childNodes(dockerHost).asList()) { if (headroomAsReservedCapacity || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals(HEADROOM_TENANT))) { + if (retiredAsFreeCapacity && container.allocation().get().membership().retired()) continue; hostCapacity.subtract(container); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index f76e5d766bf..bdc0d4a7937 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Builds up a priority queue of which nodes should be offered to the allocation. @@ -51,9 +52,8 @@ public class NodePrioritizer { this.clusterSpec = clusterSpec; this.appId = appId; - // Add spare and headroom allocations - spareHosts = DockerCapacityConstraints.findSpareHosts(allNodes, spares); - headroomViolatedHosts = new ArrayList<>(); + spareHosts = findSpareHosts(allNodes, spares); + headroomViolatedHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors); this.capacity = new DockerHostCapacity(allNodes); @@ -158,7 +158,7 @@ public class NodePrioritizer { if (pri.parent.isPresent()) { Node parent = pri.parent.get(); - pri.freeParentCapacity = capacity.freeCapacityOf(parent, true); + pri.freeParentCapacity = capacity.freeCapacityOf(parent, true, false); /** * To be conservative we have a restriction of how many nodes we can retire for each cluster, @@ -228,4 +228,47 @@ public class NodePrioritizer { .filter(n -> n.hostname().equals(node.parentHostname().orElse(" NOT A NODE"))) .findAny(); } + + private static List findSpareHosts(List nodes, int spares) { + DockerHostCapacity capacity = new DockerHostCapacity(nodes); + return nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) + .sorted(capacity::compareWithoutRetired) + .limit(spares) + .collect(Collectors.toList()); + } + + private static List findHeadroomHosts(List nodes, List spareNodes, NodeFlavors flavors) { + DockerHostCapacity capacity = new DockerHostCapacity(nodes); + List headroomNodes = new ArrayList<>(); + + List hostsSortedOnLeastCapacity = nodes.stream() + .filter(n -> !spareNodes.contains(n)) + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) + .sorted((a,b) -> capacity.compareWithoutRetired(b,a)) + .collect(Collectors.toList()); + + for (Flavor flavor : flavors.getFlavors()) { + for (int i = 0; i < flavor.getIdealHeadroom(); i++) { + Node lastNode = null; + for (Node potentialHeadroomHost : hostsSortedOnLeastCapacity) { + if (headroomNodes.contains(potentialHeadroomHost)) continue; + lastNode = potentialHeadroomHost; + if (capacity.hasCapacity(potentialHeadroomHost, flavor)) { + headroomNodes.add(potentialHeadroomHost); + continue; + } + } + if (lastNode != null) { + headroomNodes.add(lastNode); + } + } + } + + return headroomNodes; + } } \ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePriority.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePriority.java index f9e63e06b35..6efdb3c735a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePriority.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePriority.java @@ -60,14 +60,14 @@ public class NodePriority { if (n1.node.state().equals(Node.State.inactive) && !n2.node.state().equals(Node.State.inactive)) return -1; if (n2.node.state().equals(Node.State.inactive) && !n1.node.state().equals(Node.State.inactive)) return 1; - // Choose reserved nodes - if (n1.node.state().equals(Node.State.reserved) && !n2.node.state().equals(Node.State.reserved)) return -1; - if (n2.node.state().equals(Node.State.reserved) && !n1.node.state().equals(Node.State.reserved)) return 1; + // Choose reserved nodes from a previous allocation attempt (the exist in node repo) + if (isInNodeRepoAndReserved(n1) && isInNodeRepoAndReserved(n2)) return -1; + if (isInNodeRepoAndReserved(n2) && isInNodeRepoAndReserved(n1)) return 1; - // The node state has to be equal here + // The node state should be equal here if (!n1.node.state().equals(n2.node.state())) { throw new RuntimeException( - String.format("Nodes for allocation is not in expected state. Got %s and %s.", + String.format("Error during node priority comparison. Node states are not equal as expected. Got %s and %s.", n1.node.state(), n2.node.state())); } @@ -75,7 +75,7 @@ public class NodePriority { if (n1.preferredOnFlavor && !n2.preferredOnFlavor) return -1; if (n2.preferredOnFlavor && !n1.preferredOnFlavor) return 1; - // Choose docker node over non-docker node (this is to differentiate between docker replaces non-docker flavors) + // Choose docker node over non-docker node (is this to differentiate between docker replaces non-docker flavors?) if (n1.parent.isPresent() && !n2.parent.isPresent()) return -1; if (n2.parent.isPresent() && !n1.parent.isPresent()) return 1; @@ -87,8 +87,12 @@ public class NodePriority { if (n1.node.flavor().cost() < n2.node.flavor().cost()) return -1; if (n2.node.flavor().cost() < n1.node.flavor().cost()) return 1; - - // All else equal choose hostname lexically + // All else equal choose hostname alphabetically return n1.node.hostname().compareTo(n2.node.hostname()); } + + private static boolean isInNodeRepoAndReserved(NodePriority nodePri) { + if (nodePri.isNewNode) return false; + return nodePri.node.state().equals(Node.State.reserved); + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 985277d17ea..dc30f0ed1a8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -117,14 +117,7 @@ public class AllocationSimulator { NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor); NodeAllocation allocation = new NodeAllocation(app(id), cluster(), nodeSpec, new MutableInteger(0), Clock.systemUTC()); - List accepted = DockerAllocator.allocateNewDockerNodes(allocation, - nodeSpec, - new ArrayList<>(nodes.asList()), - new ArrayList<>(nodes.asList()), - flavors, - flavor, - 2, - (nodes, message)-> visualizer.addStep(nodes, id, message)); + List accepted = new ArrayList<>(); //TODO adpot the new allocation algoritm accepted.addAll(nodes.asList()); nodes = new NodeList(accepted); -- cgit v1.2.3