diff options
author | Torbjørn Smørgrav <smorgrav@users.noreply.github.com> | 2017-05-24 14:43:11 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-05-24 14:43:11 +0200 |
commit | 6b7beb419f8d22245ccbbb61b62ebfbf24c42eff (patch) | |
tree | 5010ae381685457f4f35487e5343d3854d8c1a93 /node-repository | |
parent | 330facf68ce39ad3fc4b74f11743bbd35cf24f26 (diff) | |
parent | e6e222824a554ce92c0a7f64de346deefe531090 (diff) |
Merge pull request #2392 from yahoo/smorgrav/dynamicdocker_sketching
Dynamic docker allocation
Diffstat (limited to 'node-repository')
18 files changed, 822 insertions, 45 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 09110c99fef..86a21806d26 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -51,10 +51,10 @@ public final class Node { Optional.empty(), History.empty(), type); } - /** Creates a node in the initial state (provisioned) */ + /** Creates a node in the initial state (reserved for docker containers, provisioned otherwise) */ public static Node create(String openStackId, Set<String> ipAddresses, Set<String> additionalIpAddresses, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { return new Node(openStackId, ipAddresses, additionalIpAddresses, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type); + Optional.empty(), History.empty(), type); } /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index ede9d818823..14d5549d0b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -241,12 +241,17 @@ public class NodeRepository extends AbstractComponent { // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */ - public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Optional<String> parentHostname, + public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Set<String> additionalIpAddresses, Optional<String> parentHostname, Flavor flavor, NodeType type) { if (ipAddresses.isEmpty()) { ipAddresses = nameResolver.getAllByNameOrThrow(hostname); } - return Node.create(openStackId, ImmutableSet.copyOf(ipAddresses), Collections.emptySet(), hostname, parentHostname, flavor, type); + return Node.create(openStackId, ImmutableSet.copyOf(ipAddresses), additionalIpAddresses, hostname, parentHostname, flavor, type); + } + + public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Optional<String> parentHostname, + Flavor flavor, NodeType type) { + return createNode(openStackId, hostname, ipAddresses, Collections.emptySet(), parentHostname, flavor, type); } public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java new file mode 100644 index 00000000000..a027b40ebad --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java @@ -0,0 +1,152 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * Set of methods to allocate new docker nodes + * <p> + * The nodes are not added to the repository here - this is done by caller. + * + * @author smorgrav + */ +public class DockerAllocator { + + /** + * The docker container allocation algorithm + */ + static List<Node> allocateNewDockerNodes(NodeAllocation allocation, + NodeSpec requestedNodes, + List<Node> allNodes, + List<Node> nodesBefore, + NodeFlavors flavors, + Flavor flavor, + int nofSpares, + BiConsumer<List<Node>, String> recorder) { + // Try allocate new nodes with all constraints in place + List<Node> nodesWithHeadroomAndSpares = DockerCapacityConstraints.addHeadroomAndSpareNodes(allNodes, flavors, nofSpares); + recorder.accept(nodesWithHeadroomAndSpares, "Headroom and spares"); + List<Node> accepted = DockerAllocator.allocate(allocation, flavor, nodesWithHeadroomAndSpares); + + List<Node> allNodesIncludingAccepted = new ArrayList<>(allNodes); + allNodesIncludingAccepted.addAll(accepted); + recorder.accept(allNodesIncludingAccepted, "1st dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated - try to allocate the remaining nodes with only hard constraints + if (!allocation.fullfilled()) { + List<Node> nodesWithSpares = DockerCapacityConstraints.addSpareNodes(allNodesIncludingAccepted, nofSpares); + recorder.accept(nodesWithSpares, "Spares only"); + + List<Node> acceptedWithHard = DockerAllocator.allocate(allocation, flavor, nodesWithSpares); + accepted.addAll(acceptedWithHard); + allNodesIncludingAccepted.addAll(acceptedWithHard); + recorder.accept(allNodesIncludingAccepted, "2nd dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated and this is a replacement - drop all constraints + boolean isReplacement = DockerAllocator.isReplacement(requestedNodes, nodesBefore, allNodes); + if (!allocation.fullfilled() && isReplacement) { + List<Node> finalTry = DockerAllocator.allocate(allocation, flavor, allNodesIncludingAccepted); + accepted.addAll(finalTry); + allNodesIncludingAccepted.addAll(finalTry); + recorder.accept(allNodesIncludingAccepted, "Final dynamic docker alloction - fullfilled: " + allocation.fullfilled()); + } + } + + return accepted; + } + + /** + * Offer the node allocation a prioritized set of new nodes according to capacity constraints + * + * @param allocation The allocation we want to fulfill + * @param flavor Since we create nodes here we need to know the exact flavor + * @param nodes The nodes relevant for the allocation (all nodes from node repo give or take) + * @return Nodes accepted by the node allocation - these nodes does not exist in the noderepo yet. + * @see DockerHostCapacity + */ + public static List<Node> allocate(NodeAllocation allocation, Flavor flavor, List<Node> nodes) { + + DockerHostCapacity dockerCapacity = new DockerHostCapacity(nodes); + + // Get all active docker hosts with enough capacity and ip slots - sorted on free capacity + List<Node> dockerHosts = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> dockerCapacity.hasCapacity(dockerHost, flavor)) + .sorted(dockerCapacity::compare) + .collect(Collectors.toList()); + + // Create one node pr. docker host that we can offer to the allocation + List<Node> offers = new LinkedList<>(); + for (Node parentHost : dockerHosts) { + Set<String> ipAddresses = DockerHostCapacity.findFreeIps(parentHost, nodes); + if (ipAddresses.isEmpty()) continue; + String ipAddress = ipAddresses.stream().findFirst().get(); + String hostname = lookupHostname(ipAddress); + if (hostname == null) continue; + Node node = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), + Collections.emptySet(), hostname, Optional.of(parentHost.hostname()), flavor, NodeType.tenant); + offers.add(node); + } + + return allocation.offer(offers, false); + } + + /** + * From ipAddress - get hostname + * + * @return hostname or null if not able to do the loopup + */ + private static String lookupHostname(String ipAddress) { + try { + return InetAddress.getByName(ipAddress).getHostName(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } + + /** + * This is an heuristic way to find if new nodes are to replace failing nodes + * or are to expand the cluster. + * + * The current implementation does not account for failed nodes that are not in the application + * anymore. The consequence is that we will ignore the spare capacity constraints too often - in + * particular when the number of failed nodes (not in the application anymore) + * for the cluster equal to the upscaling of the cluster. + * + * The deployment algorithm will still try to allocate the the capacity outside the spare capacity if possible. + * + * TODO propagate this information either through the node object or from the configserver deployer + */ + private static boolean isReplacement(NodeSpec nodeSpec, List<Node> nodesBefore, List<Node> nodesReserved) { + int wantedCount = 0; + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + wantedCount = countSpec.getCount(); + } + + List<Node> failedNodes = new ArrayList<>(); + for (Node node : nodesBefore) { + if (node.state() == Node.State.failed) { + failedNodes.add(node); + } + } + + if (failedNodes.size() == 0) return false; + return (wantedCount <= nodesReserved.size() + failedNodes.size()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java new file mode 100644 index 00000000000..d7cd8bb1e19 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Enforce allocation constraints for docker by manipulating the NodeList we operate on. + * + * The constraints comes in two flavors: headroom and spare. + * + * <b>Headroom</b> is the number of docker nodes (of various flavors) we want to reserve for new applications. + * This is e.g. to make sure we don't smear out small docker flavors on all hosts + * starving allocations for bigger flavors. + * + * <b>Spares</b> is to make sure we have replacement for applications if one or more hosts go down. + * It is more important to safeguard already onboarded applications than accept new applications. + * + * For now - we will use spare also as a means to reserve capacity for future applications that + * have had a separate AI process. + * + * When using spares - we will relay on maintenance jobs to reclaim the spare capacity whenever the + * capacity has been recovered (e.g. when the dead docker host is replaced) + * + * @author smorgrav + */ +public class DockerCapacityConstraints { + + /** This is a static utility class */ + private DockerCapacityConstraints() {} + + /** + * Spare nodes in first iteration is a node that fills up the two + * largest hosts (in terms of free capacity) + */ + public static List<Node> addSpareNodes(List<Node> nodes, int spares) { + DockerHostCapacity capacity = new DockerHostCapacity(nodes); + List<Flavor> spareFlavors = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) + .sorted(capacity::compare) + .limit(spares) + .map(dockerHost -> freeCapacityAsFlavor(dockerHost, nodes)) + .collect(Collectors.toList()); + + return addNodes(nodes, spareFlavors, "spare"); + } + + public static List<Node> addHeadroomAndSpareNodes(List<Node> nodes, NodeFlavors flavors, int nofSpares) { + List<Node> sparesAndHeadroom = addSpareNodes(nodes, nofSpares); + return addNodes(sparesAndHeadroom, flavors.getFlavors(), "headroom"); + } + + private static List<Node> addNodes(List<Node> nodes, List<Flavor> flavors, String id) { + List<Node> headroom = new ArrayList<>(nodes); + for (Flavor flavor : flavors) { + int headroomCount = flavor.getIdealHeadroom(); + if (headroomCount > 0) { + NodeAllocation allocation = createHeadRoomAllocation(flavor, headroomCount, id); + List<Node> acceptedNodes = DockerAllocator.allocate(allocation, flavor, headroom); + headroom.addAll(acceptedNodes); + } + } + return headroom; + } + + private static Flavor freeCapacityAsFlavor(Node host, List<Node> nodes) { + ResourceCapacity hostCapacity = new ResourceCapacity(host); + for (Node container : new NodeList(nodes).childNodes(host).asList()) { + hostCapacity.subtract(container); + } + return hostCapacity.asFlavor(); + } + + private static NodeAllocation createHeadRoomAllocation(Flavor flavor, int count, String id) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, + new ClusterSpec.Id(id), new Version()); + ApplicationId appId = new ApplicationId( + new ApplicationIdConfig( + new ApplicationIdConfig.Builder() + .tenant(id) + .application(id + "-" + flavor.name()) + .instance("temporarynode"))); + + return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor), + new MutableInteger(0), Clock.systemUTC()); + } +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 9ab5d54bb54..eeb3149ee53 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -19,6 +19,9 @@ import java.util.Set; */ public class DockerHostCapacity { + /** Tenant name for headroom nodes - only used internally */ + public static final String HEADROOM_TENANT = "-__!@#$$%THISisHEADroom"; + /** * An immutable list of nodes */ @@ -104,8 +107,7 @@ public class DockerHostCapacity { ResourceCapacity hostCapacity = new ResourceCapacity(dockerHost); for (Node container : allNodes.childNodes(dockerHost).asList()) { - // Until we have migrated we might have docker containers unallocated - TODO check off if headroom tenant is safe - if (includeHeadroom || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals("headroom"))) { + if (includeHeadroom || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals(HEADROOM_TENANT))) { hostCapacity.subtract(container); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 1c6bf8881c6..474b79af957 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.ComparisonChain; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; @@ -13,6 +15,8 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Clock; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.function.BiConsumer; /** * Performs preparation of node activation changes for a single host group in an application. @@ -41,14 +45,19 @@ class GroupPreparer { * This method will remove from this list if it finds it needs additional nodes * @param highestIndex the current highest node index among all active nodes in this cluster. * This method will increase this number when it allocates new nodes to the cluster. + * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers + * @param debugRecorder Debug facility to step through the allocation process after the fact * @return the list of nodes this cluster group will have allocated if activated */ // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - List<Node> surplusActiveNodes, MutableInteger highestIndex) { + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, + List<Node> surplusActiveNodes, MutableInteger highestIndex, int nofSpares, BiConsumer<List<Node>, String> debugRecorder) { try (Mutex lock = nodeRepository.lock(application)) { + + // A snapshot of nodes before we start the process - used to determind if this is a replacement + List<Node> nodesBefore = nodeRepository.getNodes(application, Node.State.values()); NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock); // Use active nodes @@ -71,9 +80,31 @@ class GroupPreparer { // Use new, ready nodes. Lock ready pool to ensure that nodes are not grabbed by others. try (Mutex readyLock = nodeRepository.lockUnallocated()) { + + // Check if we have ready nodes that we can allocate List<Node> readyNodes = nodeRepository.getNodes(requestedNodes.type(), Node.State.ready); accepted = allocation.offer(prioritizeNodes(readyNodes, requestedNodes), !canChangeGroup); allocation.update(nodeRepository.reserve(accepted)); + + if(nodeRepository.dynamicAllocationEnabled()) { + // Check if we have available capacity on docker hosts that we can allocate + if (!allocation.fullfilled()) { + // The new dynamic allocation method + Optional<Flavor> flavor = getFlavor(requestedNodes); + if (flavor.isPresent() && flavor.get().getType().equals(Flavor.Type.DOCKER_CONTAINER)) { + List<Node> allNodes = nodeRepository.getNodes(Node.State.values()); + NodeFlavors flavors = nodeRepository.getAvailableFlavors(); + accepted = DockerAllocator.allocateNewDockerNodes(allocation, requestedNodes, allNodes, + nodesBefore, flavors, flavor.get(), nofSpares, debugRecorder); + + // Add nodes to the node repository + if (allocation.fullfilled()) { + List<Node> nodesAddedToNodeRepo = nodeRepository.addDockerNodes(accepted); + allocation.update(nodesAddedToNodeRepo); + } + } + } + } } if (allocation.fullfilled()) @@ -83,7 +114,15 @@ class GroupPreparer { outOfCapacityDetails(allocation)); } } - + + private Optional<Flavor> getFlavor(NodeSpec nodeSpec) { + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + return Optional.of(countSpec.getFlavor()); + } + return Optional.empty(); + } + private String outOfCapacityDetails(NodeAllocation allocation) { if (allocation.wouldBeFulfilledWithClashingParentHost()) { return ": Not enough nodes available on separate physical hosts."; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 1feb12ba9ab..bac99b2bab4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -17,7 +18,6 @@ import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -39,24 +40,30 @@ import java.util.logging.Logger; public class NodeRepositoryProvisioner implements Provisioner { private static Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName()); + private static final int SPARE_CAPACITY_PROD = 2; + private static final int SPARE_CAPACITY_NONPROD = 0; private final NodeRepository nodeRepository; private final CapacityPolicies capacityPolicies; private final Zone zone; private final Preparer preparer; private final Activator activator; + private final BiConsumer<List<Node>, String> debugRecorder; @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone) { - this(nodeRepository, flavors, zone, Clock.systemUTC()); + this(nodeRepository, flavors, zone, Clock.systemUTC(), (x,y) -> {}); } - public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock) { + public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock, BiConsumer<List<Node>, String> debugRecorder) { this.nodeRepository = nodeRepository; this.capacityPolicies = new CapacityPolicies(zone, flavors); this.zone = zone; - this.preparer = new Preparer(nodeRepository, clock); + this.preparer = new Preparer(nodeRepository, clock, zone.environment().equals(Environment.prod) + ? SPARE_CAPACITY_PROD + : SPARE_CAPACITY_NONPROD); this.activator = new Activator(nodeRepository, clock); + this.debugRecorder = debugRecorder; } /** @@ -92,7 +99,7 @@ public class NodeRepositoryProvisioner implements Provisioner { effectiveGroups = 1; // type request with multiple groups is not supported } - return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups)); + return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups, debugRecorder)); } @Override @@ -121,5 +128,4 @@ public class NodeRepositoryProvisioner implements Provisioner { } return hosts; } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index cb6786270e7..ab4012b7d3e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -58,6 +58,12 @@ public interface NodeSpec { this.flavor = flavor; } + public Flavor getFlavor() { + return flavor; + } + + public int getCount() { return count; } + @Override public NodeType type() { return NodeType.tenant; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 57db9707049..dfd7be7804a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.List; import java.util.ListIterator; import java.util.Optional; +import java.util.function.BiConsumer; /** * Performs preparation of node activation changes for an application. @@ -25,10 +26,12 @@ class Preparer { private final NodeRepository nodeRepository; private final Clock clock; private final GroupPreparer groupPreparer; + private final int nofSpares; - public Preparer(NodeRepository nodeRepository, Clock clock) { + public Preparer(NodeRepository nodeRepository, Clock clock, int nofSpares) { this.nodeRepository = nodeRepository; this.clock = clock; + this.nofSpares = nofSpares; groupPreparer = new GroupPreparer(nodeRepository, clock); } @@ -40,7 +43,7 @@ class Preparer { // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups, BiConsumer<List<Node>, String> debugRecorder) { List<Node> surplusNodes = findNodesInRemovableGroups(application, cluster, wantedGroups); MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster)); @@ -48,7 +51,7 @@ class Preparer { for (int groupIndex = 0; groupIndex < wantedGroups; groupIndex++) { ClusterSpec clusterGroup = cluster.changeGroup(Optional.of(ClusterSpec.Group.from(groupIndex))); List<Node> accepted = groupPreparer.prepare(application, clusterGroup, - requestedNodes.fraction(wantedGroups), surplusNodes, highestIndex); + requestedNodes.fraction(wantedGroups), surplusNodes, highestIndex, nofSpares, debugRecorder); replace(acceptedNodes, accepted); } moveToActiveGroup(surplusNodes, wantedGroups, cluster.group()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 6aa8520b1f5..ed4f45ece63 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -85,7 +85,7 @@ public class FailedExpirerTest { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", flavorName); NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup()); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock, (x,y) -> {}); Flavor defaultFlavor = nodeFlavors.getFlavorOrThrow("default"); List<Node> hostNodes = new ArrayList<>(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index f78dc031b0d..1b9371d44ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -40,7 +40,7 @@ public class ReservationExpirerTest { NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup()); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock, (x,y) -> {}); List<Node> nodes = new ArrayList<>(2); nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java new file mode 100644 index 00000000000..7a24e4eaeaf --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -0,0 +1,154 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; + +import javax.swing.JFrame; +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + + +/** + * Graphically run allocation procedure to ease manual comprehension. + * + * Extremely useful when trying to understand test cases and build corner cases. + */ +public class AllocationSimulator { + + private AllocationVisualizer visualizer; + private NodeList nodes = new NodeList(new ArrayList<>()); + private NodeFlavors flavors; + + private AllocationSimulator(AllocationVisualizer visualizer) { + this.visualizer = visualizer; + + // + // Setup flavors + // + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 8., 8., 8, Flavor.Type.BARE_METAL).idealHeadroom(1); + b.addFlavor("host-small", 5., 5., 5, Flavor.Type.BARE_METAL).idealHeadroom(2); + b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + flavors = new NodeFlavors(b.build()); + + // + // Initiate nodes in system + // + List<Node> initialNodes = new ArrayList<>(); + initialNodes.add(host("host1", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host2", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host3", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host4", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host5", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host6", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host7", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host8", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host9", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host10", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(node("node1", flavors.getFlavorOrThrow("d-2"), Optional.of("host1"), Optional.of("test"))); + nodes = new NodeList(initialNodes); + + visualizer.addStep(nodes.asList(), "Initial state", ""); + } + + /* ------------ Create node and flavor methods ----------------*/ + + private Node host(String hostname, Flavor flavor) { + return node(hostname, flavor, Optional.empty(), Optional.empty()); + } + + private Node node(String hostname, Flavor flavor, Optional<String> parent, Optional<String> tenant) { + return new Node("fake", Collections.singleton("127.0.0.1"), + parent.isPresent() ? Collections.emptySet() : getAdditionalIP(), hostname, parent, flavor, Status.initial(), + parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant), History.empty(), parent.isPresent() ? NodeType.tenant : NodeType.host); + } + + private Set<String> getAdditionalIP() { + Set<String> h = new HashSet<String>(); + Collections.addAll(h, "::1", "::2", "::3", "::4", "::5", "::6", "::7", "::8"); + return h; + } + + private Optional<Allocation> allocation(Optional<String> tenant) { + if (tenant.isPresent()) { + Allocation allocation = new Allocation(app(tenant.get()), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + return Optional.of(allocation); + } + return Optional.empty(); + } + + private ApplicationId app(String tenant) { + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName("test") + .instanceName("default").build(); + } + + private ClusterSpec cluster() { + return ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), ClusterSpec.Group.from(1), Version.fromString("6.41")); + } + + /* ------------ Methods to add events to the system ----------------*/ + + public void addCluster(String task, int count, Flavor flavor, String id) { + NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor); + NodeAllocation allocation = new NodeAllocation(app(id), cluster(), nodeSpec, new MutableInteger(0), Clock.systemUTC()); + + List<Node> accepted = DockerAllocator.allocateNewDockerNodes(allocation, + nodeSpec, + new ArrayList<>(nodes.asList()), + new ArrayList<>(nodes.asList()), + flavors, + flavor, + 2, + (nodes, message)-> visualizer.addStep(nodes, id, message)); + + accepted.addAll(nodes.asList()); + nodes = new NodeList(accepted); + } + + + public static void main(String[] args) { + + AllocationVisualizer visualisator = new AllocationVisualizer(); + + javax.swing.SwingUtilities.invokeLater(() -> { + JFrame frame = new JFrame("Allocation Simulator"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setContentPane(visualisator); + frame.pack(); + frame.setVisible(true); + }); + + AllocationSimulator simulator = new AllocationSimulator(visualisator); + simulator.addCluster("App1 : 3 * d-1 nodes", 3, simulator.flavors.getFlavorOrThrow("d-1"), "App1"); + simulator.addCluster("App2 : 2 * d-2 nodes", 2, simulator.flavors.getFlavorOrThrow("d-2"), "App2"); + simulator.addCluster("App3 : 3 * d-2 nodes", 3, simulator.flavors.getFlavorOrThrow("d-2"), "App3"); + simulator.addCluster("App4 : 3 * d-3 nodes", 3, simulator.flavors.getFlavorOrThrow("d-3"), "App4"); + simulator.addCluster("App5 : 3 * d-3 nodes", 3, simulator.flavors.getFlavorOrThrow("d-3"), "App5"); + simulator.addCluster("App6 : 5 * d-2 nodes", 5, simulator.flavors.getFlavorOrThrow("d-2"), "App6"); + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java new file mode 100644 index 00000000000..80e9670546d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java @@ -0,0 +1,18 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.vespa.hosted.provision.NodeList; + +/** + * @author smorgrav + */ +public class AllocationSnapshot { + NodeList nodes; + String message; + String task; + + AllocationSnapshot(NodeList nodes, String task, String message) { + this.nodes = nodes; + this.message = message; + this.task = task; + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java new file mode 100644 index 00000000000..ba9a7f2924f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -0,0 +1,143 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import javax.swing.JButton; +import javax.swing.JFrame; +import javax.swing.JPanel; +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.Graphics; +import java.util.ArrayList; +import java.util.List; + +/** + * @author smorgrav + */ +public class AllocationVisualizer extends JPanel { + // Container box's width and height + private static final int BOX_WIDTH = 1024; + private static final int BOX_HEIGHT = 480; + + // node properties + private int nodeWidth = BOX_WIDTH / 15; + private int nodeHeight = nodeWidth / 2; + private int nodeSpacing = nodeWidth / 3; + + private final List<AllocationSnapshot> steps; + int step = 0; + + public AllocationVisualizer() { + this(new ArrayList<>()); + } + + public AllocationVisualizer(List<AllocationSnapshot> steps) { + this.steps = steps; + this.setPreferredSize(new Dimension(BOX_WIDTH, BOX_HEIGHT)); + + JButton back = new JButton("Back"); + back.addActionListener(e -> { + if (step > 0) step -= 1; + repaint(); + }); + JButton forward = new JButton("Forward"); + forward.addActionListener(e -> { + if (step < steps.size() - 1) step += 1; + repaint(); + }); + this.add(back); + this.add(forward); + } + + + public void addStep(List<Node> nodes, String task, String message) { + steps.add(new AllocationSnapshot(new NodeList(nodes), task, message)); + } + + @Override + public void paintComponent(Graphics g) { + super.paintComponent(g); + System.out.println("PAINTING"); + if (steps.size() == 0) return; + + int nodeX = 40; + int nodeY = BOX_HEIGHT - 20; //Start at the bottom + + // Draw the box + g.setColor(Color.WHITE); + g.fillRect(0, 0, BOX_WIDTH, BOX_HEIGHT); + + // Find number of docker hosts (to calculate start, and width of each) + // Draw the docker hosts - and color each container according to application + AllocationSnapshot simStep = steps.get(step); + NodeList hosts = simStep.nodes.nodeType(NodeType.host); + for (Node host : hosts.asList()) { + + // Paint the host + paintNode(host, g, nodeX, nodeY, true); + + // Paint containers + NodeList containers = simStep.nodes.childNodes(host); + for (Node container : containers.asList()) { + nodeY = paintNode(container, g, nodeX, nodeY, false); + } + + // Next host + nodeX += nodeWidth + nodeSpacing; + nodeY = BOX_HEIGHT - 20; + } + + // Display messages + g.setColor(Color.BLACK); + g.setFont(new Font("Courier New", Font.BOLD, 15)); + g.drawString(simStep.task, 20, 30); + g.drawString(simStep.message, 20, 50); + } + + private int paintNode(Node node, Graphics g, int x, int y, boolean isHost) { + + if (isHost) { + g.setColor(Color.GRAY); + for (int i = 0; i < node.flavor().getMinMainMemoryAvailableGb(); i++) { + g.fillRect(x, y - nodeHeight, nodeWidth, nodeHeight); + y = y - (nodeHeight + 2); + } + } else { + g.setColor(Color.YELLOW); + int multi = (int) node.flavor().getMinMainMemoryAvailableGb(); + int height = multi * nodeHeight + ((multi - 1) * 2); + g.fillRect(x, y - height, nodeWidth, height); + + // Write tenant name in allocation + String tenantName = node.allocation().get().owner().tenant().value(); + g.setColor(Color.BLACK); + g.setFont(new Font("Courier New", Font.PLAIN, 12)); + g.drawString(tenantName, x + nodeWidth / 2 - 20, y - height / 2); + + y = y - height - 2; + } + return y; + } + + public static void visualize(List<AllocationSnapshot> snaps) { + AllocationVisualizer visualisator = new AllocationVisualizer(snaps); + javax.swing.SwingUtilities.invokeLater(() -> { + JFrame frame = new JFrame("Allocation Simulator"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setContentPane(visualisator); + frame.pack(); + frame.setVisible(true); + }); + + while(true) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index 2b843929b6e..464dd83211a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -29,8 +29,6 @@ import static org.junit.Assert.assertFalse; */ public class DockerHostCapacityTest { - private static final String HEADROOM_TENANT = "-__!@#$$%THISisHEADroom"; - private DockerHostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; @@ -74,24 +72,17 @@ public class DockerHostCapacityTest { assertEquals(host1, nodes.get(1)); assertEquals(host2, nodes.get(2)); - // Add a headroom node to host1 and the host2 should be prioritized - Allocation allocation = new Allocation(app(HEADROOM_TENANT), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); - Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host1"), flavorDocker, NodeType.tenant); - nodeF.with(allocation); - nodes.add(nodeF); + // Replace a node for host 2 with a headroom node - host2 should then be prioritized + Allocation allocation = new Allocation(app(DockerHostCapacity.HEADROOM_TENANT), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host2"), flavorDocker, NodeType.tenant); + Node nodeFWithAllocation = nodeF.with(allocation); + nodes.add(nodeFWithAllocation); + nodes.remove(nodeC); capacity = new DockerHostCapacity(nodes); Collections.sort(nodes, capacity::compare); assertEquals(host3, nodes.get(0)); assertEquals(host2, nodes.get(1)); assertEquals(host1, nodes.get(2)); - - // Remove a node from host1 and it should be prioritized again - nodes.remove(nodeA); - capacity = new DockerHostCapacity(nodes); - Collections.sort(nodes, capacity::compare); - assertEquals(host3, nodes.get(0)); - assertEquals(host1, nodes.get(1)); - assertEquals(host2, nodes.get(2)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 89119441a93..24e9736906d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -14,7 +14,6 @@ import org.junit.Test; import java.util.HashSet; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java new file mode 100644 index 00000000000..4dff3cff60c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -0,0 +1,139 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.path.Path; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author mortent + */ +public class DynamicDockerProvisioningTest { + + @Test + public void spare_capacity_used_only_when_replacement() { + // Use spare capacity only when replacement (i.e one node is failed) + // Test should allocate as much capacity as possible, verify that it is not possible to allocate one more unit + // Verify that there is still capacity (available spare) + // Fail one node and redeploy, Verify that one less node is empty. + + // Setup test + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); + enableDynamicAllocation(tester); + ApplicationId application1 = tester.makeApplicationId(); + tester.makeReadyNodes(5, "host-small", NodeType.host, 32); + deployZoneApp(tester); + Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); + + // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + DockerHostCapacity capacity = new DockerHostCapacity(tester.nodeRepository().getNodes(Node.State.values())); + assertThat(capacity.freeCapacityInFlavorEquivalence(flavor), greaterThan(0)); + + List<Node> initialSpareCapacity = findSpareCapacity(tester); + assertThat(initialSpareCapacity.size(), is(2)); + + try { + hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 4, 1, flavor.canonicalName()); + fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); + } catch (OutOfCapacityException e) { + } + + tester.fail(hosts.get(0)); + hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + List<Node> finalSpareCapacity = findSpareCapacity(tester); + + assertThat(finalSpareCapacity.size(), is(1)); + + // Uncomment the statement below to walk through the allocation events visually + //AllocationVisualizer.visualize(tester.getAllocationSnapshots()); + } + + @Test + public void non_prod_do_not_have_spares() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig()); + enableDynamicAllocation(tester); + tester.makeReadyNodes(3, "host-small", NodeType.host, 32); + deployZoneApp(tester); + Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); + + ApplicationId application1 = tester.makeApplicationId(); + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + List<Node> initialSpareCapacity = findSpareCapacity(tester); + assertThat(initialSpareCapacity.size(), is(0)); + } + + private List<Node> findSpareCapacity(ProvisioningTester tester) { + List<Node> nodes = tester.nodeRepository().getNodes(Node.State.values()); + NodeList nl = new NodeList(nodes); + return nodes.stream() + .filter(n -> n.type() == NodeType.host) + .filter(n -> nl.childNodes(n).size() == 0) // Nodes without children + .collect(Collectors.toList()); + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); + b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); + b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + return b.build(); + } + + private List<HostSpec> deployZoneApp(ProvisioningTester tester) { + ApplicationId applicationId = tester.makeApplicationId(); + List<HostSpec> list = tester.prepare(applicationId, + ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from("node-admin"), + Version.fromString("6.42")), + Capacity.fromRequiredNodeType(NodeType.host), + 1); + tester.activate(applicationId, ImmutableSet.copyOf(list)); + return list; + } + + private void enableDynamicAllocation(ProvisioningTester tester) { + tester.getCurator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index bbf8967c254..5af13833a7a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -5,16 +5,18 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; @@ -23,8 +25,6 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; @@ -34,6 +34,7 @@ import java.io.IOException; import java.time.temporal.TemporalAmount; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -42,6 +43,7 @@ import java.util.Set; import java.util.UUID; import java.util.logging.Level; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -60,6 +62,7 @@ public class ProvisioningTester implements AutoCloseable { private final NodeRepositoryProvisioner provisioner; private final CapacityPolicies capacityPolicies; private final ProvisionLogger provisionLogger; + private final List<AllocationSnapshot> allocationSnapshots = new ArrayList<>(); public ProvisioningTester(Zone zone) { this(zone, createConfig()); @@ -75,7 +78,8 @@ public class ProvisioningTester implements AutoCloseable { this.clock = new ManualClock(); this.curator = curator; this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver); - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock, + (x,y) -> allocationSnapshots.add(new AllocationSnapshot(new NodeList(x), "Provision tester", y))); this.capacityPolicies = new CapacityPolicies(zone, nodeFlavors); this.provisionLogger = new NullProvisionLogger(); } @@ -111,6 +115,10 @@ public class ProvisioningTester implements AutoCloseable { //testingServer.close(); } + public List<AllocationSnapshot> getAllocationSnapshots() { + return allocationSnapshots; + } + public void advanceTime(TemporalAmount duration) { clock.advance(duration); } public NodeRepository nodeRepository() { return nodeRepository; } public ManualClock clock() { return clock; } @@ -213,13 +221,24 @@ public class ProvisioningTester implements AutoCloseable { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { + return makeReadyNodes(n, flavor, type, 0); + } + + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) { List<Node> nodes = new ArrayList<>(n); - for (int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { + Set<String> ips = IntStream.range(additionalIps * i, additionalIps * (i+1)) + .mapToObj(j -> String.format("127.0.0.%d", j)) + .collect(Collectors.toSet()); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - Optional.empty(), - nodeFlavors.getFlavorOrThrow(flavor), - type)); + UUID.randomUUID().toString(), + Collections.emptySet(), + ips, + Optional.empty(), + nodeFlavors.getFlavorOrThrow(flavor), + type)); + } nodes = nodeRepository.addNodes(nodes); nodes = nodeRepository.setDirty(nodes); return nodeRepository.setReady(nodes); |