diff options
author | Arnstein Ressem <aressem@gmail.com> | 2017-08-14 13:01:26 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-14 13:01:26 +0200 |
commit | 2b29283afac9713c23a8160c1f44374fdafede0c (patch) | |
tree | 2e98587efb4983eed6e98f6b2058a6fb856e774a /node-repository | |
parent | 3c745e5796d1d8b875ecd2ff68d677c228254b06 (diff) |
Revert "Allocation rewrite"
Diffstat (limited to 'node-repository')
26 files changed, 624 insertions, 1150 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java new file mode 100644 index 00000000000..fca15283e5d --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java @@ -0,0 +1,153 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * Set of methods to allocate new docker nodes + * <p> + * The nodes are not added to the repository here - this is done by caller. + * + * @author smorgrav + */ +public class DockerAllocator { + + /** + * The docker container allocation algorithm + */ + static List<Node> allocateNewDockerNodes(NodeAllocation allocation, + NodeSpec requestedNodes, + List<Node> allNodes, + List<Node> nodesBefore, + NodeFlavors flavors, + Flavor flavor, + int nofSpares, + BiConsumer<List<Node>, String> recorder) { + // Try allocate new nodes with all constraints in place + List<Node> nodesWithHeadroomAndSpares = DockerCapacityConstraints.addHeadroomAndSpareNodes(allNodes, flavors, nofSpares); + recorder.accept(nodesWithHeadroomAndSpares, "Headroom and spares"); + List<Node> accepted = DockerAllocator.allocate(allocation, flavor, nodesWithHeadroomAndSpares); + + List<Node> allNodesIncludingAccepted = new ArrayList<>(allNodes); + allNodesIncludingAccepted.addAll(accepted); + recorder.accept(allNodesIncludingAccepted, "1st dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated - try to allocate the remaining nodes with only hard constraints + if (!allocation.fullfilled()) { + List<Node> nodesWithSpares = DockerCapacityConstraints.addSpareNodes(allNodesIncludingAccepted, nofSpares); + recorder.accept(nodesWithSpares, "Spares only"); + + List<Node> acceptedWithHard = DockerAllocator.allocate(allocation, flavor, nodesWithSpares); + accepted.addAll(acceptedWithHard); + allNodesIncludingAccepted.addAll(acceptedWithHard); + recorder.accept(allNodesIncludingAccepted, "2nd dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated and this is a replacement - drop all constraints + boolean isReplacement = DockerAllocator.isReplacement(requestedNodes, nodesBefore, allNodes); + if (!allocation.fullfilled() && isReplacement) { + List<Node> finalTry = DockerAllocator.allocate(allocation, flavor, allNodesIncludingAccepted); + accepted.addAll(finalTry); + allNodesIncludingAccepted.addAll(finalTry); + recorder.accept(allNodesIncludingAccepted, "Final dynamic docker alloction - fullfilled: " + allocation.fullfilled()); + } + } + + return accepted; + } + + /** + * Offer the node allocation a prioritized set of new nodes according to capacity constraints + * + * @param allocation The allocation we want to fulfill + * @param flavor Since we create nodes here we need to know the exact flavor + * @param nodes The nodes relevant for the allocation (all nodes from node repo give or take) + * @return Nodes accepted by the node allocation - these nodes does not exist in the noderepo yet. + * @see DockerHostCapacity + */ + public static List<Node> allocate(NodeAllocation allocation, Flavor flavor, List<Node> nodes) { + + DockerHostCapacity dockerCapacity = new DockerHostCapacity(nodes); + + // Get all active docker hosts with enough capacity and ip slots - sorted on free capacity + List<Node> dockerHosts = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> dockerCapacity.hasCapacity(dockerHost, flavor)) + .sorted(dockerCapacity::compare) + .collect(Collectors.toList()); + + // Create one node pr. docker host that we can offer to the allocation + List<Node> offers = new LinkedList<>(); + for (Node parentHost : dockerHosts) { + Set<String> ipAddresses = DockerHostCapacity.findFreeIps(parentHost, nodes); + if (ipAddresses.isEmpty()) continue; + String ipAddress = ipAddresses.stream().findFirst().get(); + String hostname = lookupHostname(ipAddress); + if (hostname == null) continue; + Node node = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), + Collections.emptySet(), hostname, Optional.of(parentHost.hostname()), flavor, NodeType.tenant); + offers.add(node); + } + + return allocation.offer(offers, false); + } + + /** + * From ipAddress - get hostname + * + * @return hostname or null if not able to do the loopup + */ + private static String lookupHostname(String ipAddress) { + try { + return InetAddress.getByName(ipAddress).getHostName(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } + + /** + * This is an heuristic way to find if new nodes are to replace failing nodes + * or are to expand the cluster. + * + * The current implementation does not account for failed nodes that are not in the application + * anymore. The consequence is that we will ignore the spare capacity constraints too often - in + * particular when the number of failed nodes (not in the application anymore) + * for the cluster equal to the upscaling of the cluster. + * + * The deployment algorithm will still try to allocate the the capacity outside the spare capacity if possible. + * + * TODO propagate this information either through the node object or from the configserver deployer + */ + private static boolean isReplacement(NodeSpec nodeSpec, List<Node> nodesBefore, List<Node> nodesReserved) { + int wantedCount = 0; + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + wantedCount = countSpec.getCount(); + } + + List<Node> failedNodes = new ArrayList<>(); + for (Node node : nodesBefore) { + if (node.state() == Node.State.failed) { + failedNodes.add(node); + } + } + + if (failedNodes.size() == 0) return false; + return (wantedCount <= nodesReserved.size() + failedNodes.size()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java new file mode 100644 index 00000000000..3be6b1d6df3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java @@ -0,0 +1,102 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Enforce allocation constraints for docker by manipulating the NodeList we operate on. + * + * The constraints comes in two flavors: headroom and spare. + * + * <b>Headroom</b> is the number of docker nodes (of various flavors) we want to reserve for new applications. + * This is e.g. to make sure we don't smear out small docker flavors on all hosts + * starving allocations for bigger flavors. + * + * <b>Spares</b> is to make sure we have replacement for applications if one or more hosts go down. + * It is more important to safeguard already onboarded applications than accept new applications. + * + * For now - we will use spare also as a means to reserve capacity for future applications that + * have had a separate AI process. + * + * When using spares - we will relay on maintenance jobs to reclaim the spare capacity whenever the + * capacity has been recovered (e.g. when the dead docker host is replaced) + * + * @author smorgrav + */ +public class DockerCapacityConstraints { + + /** This is a static utility class */ + private DockerCapacityConstraints() {} + + /** + * Spare nodes in first iteration is a node that fills up the two + * largest hosts (in terms of free capacity) + */ + public static List<Node> addSpareNodes(List<Node> nodes, int spares) { + DockerHostCapacity capacity = new DockerHostCapacity(nodes); + List<Flavor> spareFlavors = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) + .sorted(capacity::compare) + .limit(spares) + .map(dockerHost -> freeCapacityAsFlavor(dockerHost, nodes)) + .collect(Collectors.toList()); + + return addNodes(nodes, spareFlavors, "spare"); + } + + public static List<Node> addHeadroomAndSpareNodes(List<Node> nodes, NodeFlavors flavors, int nofSpares) { + List<Node> sparesAndHeadroom = addSpareNodes(nodes, nofSpares); + return addNodes(sparesAndHeadroom, flavors.getFlavors(), "headroom"); + } + + private static List<Node> addNodes(List<Node> nodes, List<Flavor> flavors, String id) { + List<Node> headroom = new ArrayList<>(nodes); + for (Flavor flavor : flavors) { + int headroomCount = flavor.getIdealHeadroom(); + if (headroomCount > 0) { + NodeAllocation allocation = createHeadRoomAllocation(flavor, headroomCount, id); + List<Node> acceptedNodes = DockerAllocator.allocate(allocation, flavor, headroom); + headroom.addAll(acceptedNodes); + } + } + return headroom; + } + + private static Flavor freeCapacityAsFlavor(Node host, List<Node> nodes) { + ResourceCapacity hostCapacity = new ResourceCapacity(host); + for (Node container : new NodeList(nodes).childNodes(host).asList()) { + hostCapacity.subtract(container); + } + return hostCapacity.asFlavor(); + } + + private static NodeAllocation createHeadRoomAllocation(Flavor flavor, int count, String id) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, + new ClusterSpec.Id(id), new Version()); + ApplicationId appId = new ApplicationId( + new ApplicationIdConfig( + new ApplicationIdConfig.Builder() + .tenant(id) + .application(id + "-" + flavor.name()) + .instance("temporarynode"))); + + return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor), + new MutableInteger(0), Clock.systemUTC()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 77d91c7bea7..0d8a50df568 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -38,7 +38,7 @@ public class DockerHostCapacity { * Used in prioritizing hosts for allocation in <b>descending</b> order. */ int compare(Node hostA, Node hostB) { - int comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false)); + int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true)); if (comp == 0) { comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false)); if (comp == 0) { @@ -49,27 +49,11 @@ public class DockerHostCapacity { return comp; } - int compareWithoutInactive(Node hostA, Node hostB) { - int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true)); - if (comp == 0) { - comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true)); - if (comp == 0) { - // If resources are equal - we want to assign to the one with the most IPaddresses free - comp = freeIPs(hostB) - freeIPs(hostA); - } - } - return comp; - } - /** * Checks the node capacity and free ip addresses to see * if we could allocate a flavor on the docker host. */ boolean hasCapacity(Node dockerHost, Flavor flavor) { - return freeCapacityOf(dockerHost, false).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; - } - - boolean hasCapacityWhenRetiredAndInactiveNodesAreGone(Node dockerHost, Flavor flavor) { return freeCapacityOf(dockerHost, true).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; } @@ -117,34 +101,20 @@ public class DockerHostCapacity { /** * Calculate the remaining capacity for the dockerHost. - * @param dockerHost The host to find free capacity of. - * - * @return A default (empty) capacity if not a docker host, otherwise the free/unallocated/rest capacity */ - public ResourceCapacity freeCapacityOf(Node dockerHost, boolean treatInactiveOrRetiredAsUnusedCapacity) { + private ResourceCapacity freeCapacityOf(Node dockerHost, boolean includeHeadroom) { // Only hosts have free capacity if (!dockerHost.type().equals(NodeType.host)) return new ResourceCapacity(); ResourceCapacity hostCapacity = new ResourceCapacity(dockerHost); for (Node container : allNodes.childNodes(dockerHost).asList()) { - boolean isUsedCapacity = !(treatInactiveOrRetiredAsUnusedCapacity && isInactiveOrRetired(container)); - if (isUsedCapacity) { + if (includeHeadroom || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals(HEADROOM_TENANT))) { hostCapacity.subtract(container); } } return hostCapacity; } - private boolean isInactiveOrRetired(Node node) { - boolean isInactive = node.state().equals(Node.State.inactive); - boolean isRetired = false; - if (node.allocation().isPresent()) { - isRetired = node.allocation().get().membership().retired(); - } - - return isInactive || isRetired; - } - /** * Compare the additional ip addresses against the set of used addresses on * child nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index b52506c268c..cff62508ec6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -19,7 +19,7 @@ public class FlavorConfigBuilder { return new FlavorsConfig(builder); } - public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type, int headRoom) { + public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); flavor.description("Flavor-name-is-" + flavorName); @@ -27,15 +27,10 @@ public class FlavorConfigBuilder { flavor.minCpuCores(cpu); flavor.minMainMemoryAvailableGb(mem); flavor.environment(type.name()); - flavor.idealHeadroom(headRoom); builder.flavor(flavor); return flavor; } - public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { - return addFlavor(flavorName, cpu, mem, disk, type, 0); - } - public FlavorsConfig.Flavor.Builder addNonStockFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java index 5f81fed2a04..a7a0eadf515 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java @@ -29,7 +29,6 @@ import java.util.Set; * @author freva */ public class FlavorSpareChecker { - private final SpareNodesPolicy spareNodesPolicy; private final Map<Flavor, FlavorSpareCount> spareCountByFlavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java index 217f4999bfb..2a3f44df42f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java @@ -17,7 +17,6 @@ import java.util.stream.Collectors; * @author freva */ public class FlavorSpareCount { - private final Flavor flavor; private Set<FlavorSpareCount> possibleWantedFlavors; private Set<FlavorSpareCount> immediateReplacees; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index c5d60c30356..bf7942a1c56 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.google.common.collect.ComparisonChain; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; @@ -10,7 +13,9 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Clock; +import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.BiConsumer; /** @@ -23,6 +28,8 @@ class GroupPreparer { private final NodeRepository nodeRepository; private final Clock clock; + private static final boolean canChangeGroup = true; + public GroupPreparer(NodeRepository nodeRepository, Clock clock) { this.nodeRepository = nodeRepository; this.clock = clock; @@ -31,63 +38,123 @@ class GroupPreparer { /** * Ensure sufficient nodes are reserved or active for the given application, group and cluster * - * @param application the application we are allocating to - * @param cluster the cluster and group we are allocating to - * @param requestedNodes a specification of the requested nodes + * @param application the application we are allocating to + * @param cluster the cluster and group we are allocating to + * @param requestedNodes a specification of the requested nodes * @param surplusActiveNodes currently active nodes which are available to be assigned to this group. - * This method will remove from this list if it finds it needs additional nodes - * @param highestIndex the current highest node index among all active nodes in this cluster. - * This method will increase this number when it allocates new nodes to the cluster. - * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers - * @param debugRecorder Debug facility to step through the allocation process after the fact + * This method will remove from this list if it finds it needs additional nodes + * @param highestIndex the current highest node index among all active nodes in this cluster. + * This method will increase this number when it allocates new nodes to the cluster. + * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers + * @param debugRecorder Debug facility to step through the allocation process after the fact * @return the list of nodes this cluster group will have allocated if activated */ - // Note: This operation may make persisted changes to the set of reserved and inactive nodes, - // but it may not change the set of active nodes, as the active nodes must stay in sync with the - // active config model which is changed on activate + // Note: This operation may make persisted changes to the set of reserved and inactive nodes, + // but it may not change the set of active nodes, as the active nodes must stay in sync with the + // active config model which is changed on activate public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, MutableInteger highestIndex, int nofSpares, BiConsumer<List<Node>, String> debugRecorder) { try (Mutex lock = nodeRepository.lock(application)) { + // A snapshot of nodes before we start the process - used to determine if this is a replacement + List<Node> nodesBefore = nodeRepository.getNodes(application, Node.State.values()); + NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock); + + // Use active nodes + allocation.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); + + // Use active nodes from other groups that will otherwise be retired + List<Node> accepted = allocation.offer(prioritizeNodes(surplusActiveNodes, requestedNodes), canChangeGroup); + surplusActiveNodes.removeAll(accepted); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); + + // Use previously reserved nodes + allocation.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); + + // Use inactive nodes + accepted = allocation.offer(prioritizeNodes(nodeRepository.getNodes(application, Node.State.inactive), requestedNodes), !canChangeGroup); + allocation.update(nodeRepository.reserve(accepted)); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); + // Use new, ready nodes. Lock ready pool to ensure that nodes are not grabbed by others. try (Mutex readyLock = nodeRepository.lockUnallocated()) { - // Create a prioritized set of nodes - NodePrioritizer prioritizer = new NodePrioritizer(nodeRepository.getNodes(), - application, - cluster, - requestedNodes, - nodeRepository.getAvailableFlavors(), - nofSpares); - - prioritizer.addApplicationNodes(); - prioritizer.addSurplusNodes(surplusActiveNodes); - prioritizer.addReadyNodes(); - if (nodeRepository.dynamicAllocationEnabled()) - prioritizer.addNewDockerNodes(); - - // Allocate from the prioritized list - NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock); - allocation.offer(prioritizer.prioritize()); - if (! allocation.fullfilled()) - throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + - outOfCapacityDetails(allocation)); - - // Carry out and return allocation - nodeRepository.reserve(allocation.acceptedInactiveAndReadyNodes()); - nodeRepository.addDockerNodes(allocation.acceptedNewNodes()); - surplusActiveNodes.removeAll(allocation.acceptedSurplusNodes()); - return allocation.finalNodes(surplusActiveNodes); + // Check if we have ready nodes that we can allocate + List<Node> readyNodes = nodeRepository.getNodes(requestedNodes.type(), Node.State.ready); + accepted = allocation.offer(prioritizeNodes(readyNodes, requestedNodes), !canChangeGroup); + allocation.update(nodeRepository.reserve(accepted)); + + if (nodeRepository.dynamicAllocationEnabled()) { + // Check if we have available capacity on docker hosts that we can allocate + if (!allocation.fullfilled()) { + // The new dynamic allocation method + Optional<Flavor> flavor = getFlavor(requestedNodes); + if (flavor.isPresent() && flavor.get().getType().equals(Flavor.Type.DOCKER_CONTAINER)) { + List<Node> allNodes = nodeRepository.getNodes(Node.State.values()); + NodeFlavors flavors = nodeRepository.getAvailableFlavors(); + accepted = DockerAllocator.allocateNewDockerNodes(allocation, requestedNodes, allNodes, + nodesBefore, flavors, flavor.get(), nofSpares, debugRecorder); + + // Add nodes to the node repository + if (allocation.fullfilled()) { + List<Node> nodesAddedToNodeRepo = nodeRepository.addDockerNodes(accepted); + allocation.update(nodesAddedToNodeRepo); + } + } + } + } } + + if (allocation.fullfilled()) + return allocation.finalNodes(surplusActiveNodes); + else + throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + + outOfCapacityDetails(allocation)); } } + private Optional<Flavor> getFlavor(NodeSpec nodeSpec) { + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + return Optional.of(countSpec.getFlavor()); + } + return Optional.empty(); + } + private String outOfCapacityDetails(NodeAllocation allocation) { - if (allocation.wouldBeFulfilledWithClashingParentHost()) + if (allocation.wouldBeFulfilledWithClashingParentHost()) { return ": Not enough nodes available on separate physical hosts."; - if (allocation.wouldBeFulfilledWithRetiredNodes()) + } + if (allocation.wouldBeFulfilledWithRetiredNodes()) { return ": Not enough nodes available due to retirement."; - else - return "."; + } + return "."; + } + + /** + * Returns the node list in prioritized order, where the nodes we would most prefer the application + * to use comes first + */ + private List<Node> prioritizeNodes(List<Node> nodeList, NodeSpec nodeSpec) { + if ( nodeSpec.specifiesNonStockFlavor()) { // prefer exact flavor, docker hosts, lower cost, tie break by hostname + Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start() + .compareTrueFirst(nodeSpec.matchesExactly(n1.flavor()), nodeSpec.matchesExactly(n2.flavor())) + .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent()) + .compare(n1.flavor().cost(), n2.flavor().cost()) + .compare(n1.hostname(), n2.hostname()) + .result() + ); + } + else { // prefer docker hosts, lower cost, tie break by hostname + Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start() + .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent()) + .compare(n1.flavor().cost(), n2.flavor().cost()) + .compare(n1.hostname(), n2.hostname()) + .result() + ); + } + return nodeList; } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index b9f682bc79f..b8d0495095b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -21,10 +21,8 @@ import java.util.stream.Collectors; /** * Used to manage a list of nodes during the node reservation process * in order to fulfill the nodespec. - * - * @author bratseth */ -class NodeAllocation { +public class NodeAllocation { /** The application this list is for */ private final ApplicationId application; @@ -36,7 +34,7 @@ class NodeAllocation { private final NodeSpec requestedNodes; /** The nodes this has accepted so far */ - private final Set<PrioritizableNode> nodes = new LinkedHashSet<>(); + private final Set<Node> nodes = new LinkedHashSet<>(); /** The number of nodes in the accepted nodes which are of the requested flavor */ private int acceptedOfRequestedFlavor = 0; @@ -56,7 +54,7 @@ class NodeAllocation { /** Used to record event timestamps **/ private final Clock clock; - NodeAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex, Clock clock) { + public NodeAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex, Clock clock) { this.application = application; this.cluster = cluster; this.requestedNodes = requestedNodes; @@ -67,24 +65,23 @@ class NodeAllocation { /** * Offer some nodes to this. The nodes may have an allocation to a different application or cluster, * an allocation to this cluster, or no current allocation (in which case one is assigned). - * + * <p> * Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily * reject allocated nodes due to index duplicates. * - * @param nodesPrioritized the nodes which are potentially on offer. These may belong to a different application etc. + * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc. + * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary * @return the subset of offeredNodes which was accepted, with the correct allocation assigned */ - List<Node> offer(List<PrioritizableNode> nodesPrioritized) { + public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) { List<Node> accepted = new ArrayList<>(); - for (PrioritizableNode offeredPriority : nodesPrioritized) { - Node offered = offeredPriority.node; - + for (Node offered : offeredNodes) { if (offered.allocation().isPresent()) { boolean wantToRetireNode = false; ClusterMembership membership = offered.allocation().get().membership(); if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type - if ((! offeredPriority.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it + if ((! canChangeGroup || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) @@ -94,9 +91,8 @@ class NodeAllocation { if ( offered.flavor().isRetired()) wantToRetireNode = true; if ( offered.status().wantToRetire()) wantToRetireNode = true; - if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) { - accepted.add(acceptNode(offeredPriority, wantToRetireNode)); - } + if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) + accepted.add(acceptNode(offered, wantToRetireNode)); } else if (! saturated() && hasCompatibleFlavor(offered)) { if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) { @@ -109,18 +105,18 @@ class NodeAllocation { if (offered.status().wantToRetire()) { continue; } - offeredPriority.node = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant()); - accepted.add(acceptNode(offeredPriority, false)); + Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant()); + accepted.add(acceptNode(alloc, false)); } } return accepted; } - private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<PrioritizableNode> accepted, Node offered) { - for (PrioritizableNode acceptedNode : accepted) { - if (acceptedNode.node.parentHostname().isPresent() && offered.parentHostname().isPresent() && - acceptedNode.node.parentHostname().get().equals(offered.parentHostname().get())) { + private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) { + for (Node acceptedNode : accepted) { + if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() && + acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) { return true; } } @@ -154,29 +150,31 @@ class NodeAllocation { return requestedNodes.isCompatible(node.flavor()); } - private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire) { - Node node = prioritizableNode.node; + /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */ + public void update(List<Node> updatedNodes) { + nodes.removeAll(updatedNodes); + nodes.addAll(updatedNodes); + } + + private Node acceptNode(Node node, boolean wantToRetire) { if (! wantToRetire) { if ( ! node.state().equals(Node.State.active)) { // reactivated node - make sure its not retired node = node.unretire(); - prioritizableNode.node= node; } acceptedOfRequestedFlavor++; } else { ++wasRetiredJustNow; // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host node = node.retire(clock.instant()); - prioritizableNode.node= node; } if ( ! node.allocation().get().membership().cluster().equals(cluster)) { // group may be different node = setCluster(cluster, node); - prioritizableNode.node= node; } indexes.add(node.allocation().get().membership().index()); highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); - nodes.add(prioritizableNode); + nodes.add(node); return node; } @@ -186,20 +184,20 @@ class NodeAllocation { } /** Returns true if no more nodes are needed in this list */ - private boolean saturated() { + public boolean saturated() { return requestedNodes.saturatedBy(acceptedOfRequestedFlavor); } /** Returns true if the content of this list is sufficient to meet the request */ - boolean fullfilled() { + public boolean fullfilled() { return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor); } - boolean wouldBeFulfilledWithRetiredNodes() { + public boolean wouldBeFulfilledWithRetiredNodes() { return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow); } - boolean wouldBeFulfilledWithClashingParentHost() { + public boolean wouldBeFulfilledWithClashingParentHost() { return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost); } @@ -212,61 +210,42 @@ class NodeAllocation { * @param surplusNodes this will add nodes not any longer needed by this group to this list * @return the final list of nodes */ - List<Node> finalNodes(List<Node> surplusNodes) { - long currentRetired = nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); + public List<Node> finalNodes(List<Node> surplusNodes) { + long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count(); long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired; + List<Node> changedNodes = new ArrayList<>(); if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution - for (PrioritizableNode node : byDecreasingIndex(nodes)) { - if ( ! node.node.allocation().get().membership().retired() && node.node.state().equals(Node.State.active)) { - node.node = node.node.retire(Agent.application, clock.instant()); - surplusNodes.add(node.node); // offer this node to other groups + for (Node node : byDecreasingIndex(nodes)) { + if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) { + changedNodes.add(node.retire(Agent.application, clock.instant())); + surplusNodes.add(node); // offer this node to other groups if (--surplus == 0) break; } } } else if (surplus < 0) { // unretire until surplus is 0 - for (PrioritizableNode node : byIncreasingIndex(nodes)) { - if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node.node)) { - node.node = node.node.unretire(); + for (Node node : byIncreasingIndex(nodes)) { + if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) { + changedNodes.add(node.unretire()); if (++surplus == 0) break; } } } - - return nodes.stream().map(n -> n.node).collect(Collectors.toList()); - } - - List<Node> acceptedInactiveAndReadyNodes() { - return nodes.stream().map(n -> n.node) - .filter(n -> n.state().equals(Node.State.inactive) || n.state().equals(Node.State.ready)) - .collect(Collectors.toList()); - } - - List<Node> acceptedSurplusNodes() { - return nodes.stream() - .filter(n -> n.isSurplusNode) - .map(n -> n.node) - .collect(Collectors.toList()); - } - - List<Node> acceptedNewNodes() { - return nodes.stream() - .filter(n -> n.isNewNode) - .map(n -> n.node) - .collect(Collectors.toList()); + update(changedNodes); + return new ArrayList<>(nodes); } - private List<PrioritizableNode> byDecreasingIndex(Set<PrioritizableNode> nodes) { + private List<Node> byDecreasingIndex(Set<Node> nodes) { return nodes.stream().sorted(nodeIndexComparator().reversed()).collect(Collectors.toList()); } - private List<PrioritizableNode> byIncreasingIndex(Set<PrioritizableNode> nodes) { + private List<Node> byIncreasingIndex(Set<Node> nodes) { return nodes.stream().sorted(nodeIndexComparator()).collect(Collectors.toList()); } - private Comparator<PrioritizableNode> nodeIndexComparator() { - return Comparator.comparing((PrioritizableNode n) -> n.node.allocation().get().membership().index()); + private Comparator<Node> nodeIndexComparator() { + return Comparator.comparing((Node n) -> n.allocation().get().membership().index()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java deleted file mode 100644 index 24e33d31a7a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ /dev/null @@ -1,301 +0,0 @@ -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Builds up data structures necessary for node prioritization. It wraps each node - * up in a PrioritizableNode object with attributes used in sorting. - * - * The actual sorting/prioritization is implemented in the PrioritizableNode class as a compare method. - * - * @author smorgrav - */ -public class NodePrioritizer { - - private final Map<Node, PrioritizableNode> nodes = new HashMap<>(); - private final List<Node> allNodes; - private final DockerHostCapacity capacity; - private final NodeSpec requestedNodes; - private final ApplicationId appId; - private final ClusterSpec clusterSpec; - - private final boolean isAllocatingForReplacement; - private final Set<Node> spareHosts; - private final Map<Node, Boolean> headroomHosts; - private final boolean isDocker; - - NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, NodeFlavors nodeFlavors, int spares) { - this.allNodes = Collections.unmodifiableList(allNodes); - this.requestedNodes = nodeSpec; - this.clusterSpec = clusterSpec; - this.appId = appId; - - spareHosts = findSpareHosts(allNodes, spares); - headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors); - - this.capacity = new DockerHostCapacity(allNodes); - - long nofFailedNodes = allNodes.stream() - .filter(node -> node.state().equals(Node.State.failed)) - .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(appId)) - .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) - .count(); - - long nofNodesInCluster = allNodes.stream() - .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(appId)) - .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) - .count(); - - isAllocatingForReplacement = isReplacement(nofNodesInCluster, nofFailedNodes); - isDocker = isDocker(); - } - - /** - * From ipAddress - get hostname - * - * @return hostname or null if not able to do the loopup - */ - private static String lookupHostname(String ipAddress) { - try { - return InetAddress.getByName(ipAddress).getHostName(); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - return null; - } - - /** - * Spare hosts are the two hosts in the system with the most free capacity. - * - * We do not count retired or inactive nodes as used capacity (as they could have been - * moved to create space for the spare node in the first place). - */ - private static Set<Node> findSpareHosts(List<Node> nodes, int spares) { - DockerHostCapacity capacity = new DockerHostCapacity(new ArrayList<>(nodes)); - return nodes.stream() - .filter(node -> node.type().equals(NodeType.host)) - .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) - .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) - .sorted(capacity::compareWithoutInactive) - .limit(spares) - .collect(Collectors.toSet()); - } - - /** - * Headroom are the nodes with the least but sufficient space for the requested headroom. - * - * If not enough headroom - the headroom violating hosts are the once that are closest to fulfull - * a headroom request. - */ - private static Map<Node, Boolean> findHeadroomHosts(List<Node> nodes, Set<Node> spareNodes, NodeFlavors flavors) { - DockerHostCapacity capacity = new DockerHostCapacity(nodes); - Map<Node, Boolean> headroomNodesToViolation = new HashMap<>(); - - List<Node> hostsSortedOnLeastCapacity = nodes.stream() - .filter(n -> !spareNodes.contains(n)) - .filter(node -> node.type().equals(NodeType.host)) - .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) - .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) - .sorted((a, b) -> capacity.compareWithoutInactive(b, a)) - .collect(Collectors.toList()); - - for (Flavor flavor : flavors.getFlavors().stream().filter(f -> f.getIdealHeadroom() > 0).collect(Collectors.toList())) { - Set<Node> tempHeadroom = new HashSet<>(); - Set<Node> notEnoughCapacity = new HashSet<>(); - for (Node host : hostsSortedOnLeastCapacity) { - if (headroomNodesToViolation.containsKey(host)) continue; - if (capacity.hasCapacityWhenRetiredAndInactiveNodesAreGone(host, flavor)) { - headroomNodesToViolation.put(host, false); - tempHeadroom.add(host); - } else { - notEnoughCapacity.add(host); - } - - if (tempHeadroom.size() == flavor.getIdealHeadroom()) { - continue; - } - } - - // Now check if we have enough headroom - if not choose the nodes that almost has it - if (tempHeadroom.size() < flavor.getIdealHeadroom()) { - List<Node> violations = notEnoughCapacity.stream() - .sorted((a, b) -> capacity.compare(b, a)) - .limit(flavor.getIdealHeadroom() - tempHeadroom.size()) - .collect(Collectors.toList()); - - for (Node nodeViolatingHeadrom : violations) { - headroomNodesToViolation.put(nodeViolatingHeadrom, true); - } - - } - } - - return headroomNodesToViolation; - } - - /** - * @return The list of nodes sorted by PrioritizableNode::compare - */ - List<PrioritizableNode> prioritize() { - List<PrioritizableNode> priorityList = new ArrayList<>(nodes.values()); - Collections.sort(priorityList); - return priorityList; - } - - /** - * Add nodes that have been previously reserved to the same application from - * an earlier downsizing of a cluster - */ - void addSurplusNodes(List<Node> surplusNodes) { - for (Node node : surplusNodes) { - PrioritizableNode nodePri = toNodePriority(node, true, false); - if (!nodePri.violatesSpares || isAllocatingForReplacement) { - nodes.put(node, nodePri); - } - } - } - - /** - * Add a node on each docker host with enough capacity for the requested flavor - */ - void addNewDockerNodes() { - if (!isDocker) return; - DockerHostCapacity capacity = new DockerHostCapacity(allNodes); - - for (Node node : allNodes) { - if (node.type() == NodeType.host) { - boolean conflictingCluster = false; - NodeList list = new NodeList(allNodes); - NodeList childrenWithSameApp = list.childNodes(node).owner(appId); - for (Node child : childrenWithSameApp.asList()) { - // Look for nodes from the same cluster - if (child.allocation().get().membership().cluster().id().equals(clusterSpec.id())) { - conflictingCluster = true; - break; - } - } - - if (!conflictingCluster && capacity.hasCapacity(node, getFlavor())) { - Set<String> ipAddresses = DockerHostCapacity.findFreeIps(node, allNodes); - if (ipAddresses.isEmpty()) continue; - String ipAddress = ipAddresses.stream().findFirst().get(); - String hostname = lookupHostname(ipAddress); - if (hostname == null) continue; - Node newNode = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), - Collections.emptySet(), hostname, Optional.of(node.hostname()), getFlavor(), NodeType.tenant); - PrioritizableNode nodePri = toNodePriority(newNode, false, true); - if (!nodePri.violatesSpares || isAllocatingForReplacement) { - nodes.put(newNode, nodePri); - } - } - } - } - } - - /** - * Add existing nodes allocated to the application - */ - void addApplicationNodes() { - List<Node.State> legalStates = Arrays.asList(Node.State.active, Node.State.inactive, Node.State.reserved); - allNodes.stream() - .filter(node -> node.type().equals(requestedNodes.type())) - .filter(node -> legalStates.contains(node.state())) - .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(appId)) - .map(node -> toNodePriority(node, false, false)) - .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); - } - - /** - * Add nodes already provisioned, but not allocatied to any application - */ - void addReadyNodes() { - allNodes.stream() - .filter(node -> node.type().equals(requestedNodes.type())) - .filter(node -> node.state().equals(Node.State.ready)) - .map(node -> toNodePriority(node, false, false)) - .filter(n -> !n.violatesSpares || isAllocatingForReplacement) - .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); - } - - /** - * Convert a list of nodes to a list of node priorities. This includes finding, calculating - * parameters to the priority sorting procedure. - */ - private PrioritizableNode toNodePriority(Node node, boolean isSurplusNode, boolean isNewNode) { - PrioritizableNode pri = new PrioritizableNode(); - pri.node = node; - pri.isSurplusNode = isSurplusNode; - pri.isNewNode = isNewNode; - pri.preferredOnFlavor = requestedNodes.specifiesNonStockFlavor() && node.flavor().equals(getFlavor()); - pri.parent = findParentNode(node); - - if (pri.parent.isPresent()) { - Node parent = pri.parent.get(); - pri.freeParentCapacity = capacity.freeCapacityOf(parent, false); - - if (spareHosts.contains(parent)) { - pri.violatesSpares = true; - } - - if (headroomHosts.containsKey(parent)) { - pri.violatesHeadroom = headroomHosts.get(parent); - } - } - - return pri; - } - - private boolean isReplacement(long nofNodesInCluster, long nodeFailedNodes) { - if (nodeFailedNodes == 0) return false; - - int wantedCount = 0; - if (requestedNodes instanceof NodeSpec.CountNodeSpec) { - NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes; - wantedCount = countSpec.getCount(); - } - - return (wantedCount > nofNodesInCluster - nodeFailedNodes); - } - - private Flavor getFlavor() { - if (requestedNodes instanceof NodeSpec.CountNodeSpec) { - NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes; - return countSpec.getFlavor(); - } - return null; - } - - private boolean isDocker() { - Flavor flavor = getFlavor(); - return (flavor != null) && flavor.getType().equals(Flavor.Type.DOCKER_CONTAINER); - } - - private Optional<Node> findParentNode(Node node) { - if (!node.parentHostname().isPresent()) return Optional.empty(); - return allNodes.stream() - .filter(n -> n.hostname().equals(node.parentHostname().orElse(" NOT A NODE"))) - .findAny(); - } -}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index a0f9452ef9f..639d6ea17f4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -88,12 +88,7 @@ class Preparer { } } } - - /** - * Nodes are immutable so when changing attributes to the node we create a new instance. - * - * This method is used to both add new nodes and replaces old node references with the new references. - */ + private List<Node> replace(List<Node> list, List<Node> changed) { list.removeAll(changed); list.addAll(changed); @@ -126,4 +121,5 @@ class Preparer { } return retired; } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java deleted file mode 100644 index 219778ab7a0..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.Optional; - -/** - * A node with additional information required to prioritize it for allocation. - * - * @author smorgrav - */ -class PrioritizableNode implements Comparable<PrioritizableNode> { - - Node node; - - /** The free capacity excluding headroom, including retired allocations */ - ResourceCapacity freeParentCapacity = new ResourceCapacity(); - - /** The parent host (docker or hypervisor) */ - Optional<Node> parent = Optional.empty(); - - /** True if the node is allocated to a host that should be dedicated as a spare */ - boolean violatesSpares; - - /** True if the node is allocated on slots that should be dedicated to headroom */ - boolean violatesHeadroom; - - /** True if this is a node that has been retired earlier in the allocation process */ - boolean isSurplusNode; - - /** This node does not exist in the node repository yet */ - boolean isNewNode; - - /** True if exact flavor is specified by the allocation request and this node has this flavor */ - boolean preferredOnFlavor; - - /** - * Compares two prioritizable nodes - * - * @return negative if first priority is higher than second node - */ - @Override - public int compareTo(PrioritizableNode other) { - // First always pick nodes without violation above nodes with violations - if (!this.violatesSpares && other.violatesSpares) return -1; - if (!other.violatesSpares && this.violatesSpares) return 1; - if (!this.violatesHeadroom && other.violatesHeadroom) return -1; - if (!other.violatesHeadroom && this.violatesHeadroom) return 1; - - // Choose active nodes - if (this.node.state().equals(Node.State.active) && !other.node.state().equals(Node.State.active)) return -1; - if (other.node.state().equals(Node.State.active) && !this.node.state().equals(Node.State.active)) return 1; - - // Choose active node that is not retired first (surplus is active but retired) - if (!this.isSurplusNode && other.isSurplusNode) return -1; - if (!other.isSurplusNode && this.isSurplusNode) return 1; - - // Choose inactive nodes - if (this.node.state().equals(Node.State.inactive) && !other.node.state().equals(Node.State.inactive)) return -1; - if (other.node.state().equals(Node.State.inactive) && !this.node.state().equals(Node.State.inactive)) return 1; - - // Choose reserved nodes from a previous allocation attempt (the exist in node repo) - if (isInNodeRepoAndReserved(this) && !isInNodeRepoAndReserved(other)) return -1; - if (isInNodeRepoAndReserved(other) && !isInNodeRepoAndReserved(this)) return 1; - - // Choose ready nodes - if (this.node.state().equals(Node.State.ready) && !other.node.state().equals(Node.State.ready)) return -1; - if (other.node.state().equals(Node.State.ready) && !this.node.state().equals(Node.State.ready)) return 1; - - // The node state should be equal here - if (!this.node.state().equals(other.node.state())) { - throw new RuntimeException( - String.format("Error during node priority comparison. Node states are not equal as expected. Got %s and %s.", - this.node.state(), other.node.state())); - } - - // Choose exact flavor - if (this.preferredOnFlavor && !other.preferredOnFlavor) return -1; - if (other.preferredOnFlavor && !this.preferredOnFlavor) return 1; - - // Choose docker node over non-docker node (is this to differentiate between docker replaces non-docker flavors?) - if (this.parent.isPresent() && !other.parent.isPresent()) return -1; - if (other.parent.isPresent() && !this.parent.isPresent()) return 1; - - // Choose the node with parent node with the least capacity (TODO parameterize this as this is pretty much the core of the algorithm) - int freeCapacity = this.freeParentCapacity.compare(other.freeParentCapacity); - if (freeCapacity != 0) return freeCapacity; - - // Choose cheapest node - if (this.node.flavor().cost() < other.node.flavor().cost()) return -1; - if (other.node.flavor().cost() < this.node.flavor().cost()) return 1; - - // All else equal choose hostname alphabetically - return this.node.hostname().compareTo(other.node.hostname()); - } - - private static boolean isInNodeRepoAndReserved(PrioritizableNode nodePri) { - if (nodePri.isNewNode) return false; - return nodePri.node.state().equals(Node.State.reserved); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java index fdec29d5b97..e56409d47b9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java @@ -91,5 +91,4 @@ public class ResourceCapacity { b.addFlavor("spareflavor", cpu, memory, disk, Flavor.Type.DOCKER_CONTAINER).idealHeadroom(1); return new Flavor(b.build().flavor(0)); } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0885b941401..ad66f86de69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -51,11 +51,6 @@ public class MockNodeRepository extends NodeRepository { populate(); } - @Override - public boolean dynamicAllocationEnabled() { - return true; - } - private void populate() { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); @@ -65,22 +60,18 @@ public class MockNodeRepository extends NodeRepository { Collections.sort(ipAddressesForAllHost); final HashSet<String> ipAddresses = new HashSet<>(ipAddressesForAllHost); - final List<String> additionalIpAddressesForAllHost = Arrays.asList("::2", "::3", "::4"); - Collections.sort(additionalIpAddressesForAllHost); - final HashSet<String> additionalIpAddresses = new HashSet<>(additionalIpAddressesForAllHost); - nodes.add(createNode("node1", "host1.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); nodes.add(createNode("node2", "host2.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); nodes.add(createNode("node3", "host3.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("expensive"), NodeType.tenant)); - Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost1.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant); + // TODO: Use docker flavor + Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost4"), flavors.getFlavorOrThrow("default"), NodeType.tenant); node4 = node4.with(node4.status().withVespaVersion(new Version("6.41.0"))); nodes.add(node4); - Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("dockerhost2.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant); + Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("parent1.yahoo.com"), flavors.getFlavorOrThrow("default"), NodeType.tenant); nodes.add(node5.with(node5.status().withVespaVersion(new Version("1.2.3")))); - nodes.add(createNode("node6", "host6.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); nodes.add(createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); // 8 and 9 are added by web service calls @@ -92,13 +83,7 @@ public class MockNodeRepository extends NodeRepository { nodes.add(node10); nodes.add(createNode("node55", "host55.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - - /** Setup docker hosts (two of these will be reserved for spares */ - nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("parent1", "parent1.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.host)); nodes = addNodes(nodes); nodes.remove(6); @@ -108,12 +93,6 @@ public class MockNodeRepository extends NodeRepository { fail("host5.yahoo.com", Agent.system, "Failing to unit test"); setDirty("host55.yahoo.com"); - ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp")); - ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("node-admin"), - Version.fromString("6.42")); - activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner); - ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42")); provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null); @@ -124,7 +103,7 @@ public class MockNodeRepository extends NodeRepository { ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3")); ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Version.fromString("6.42")); - activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2, "docker"), 1, null), app3, provisioner); + activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2), 1, null), app3, provisioner); } private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { @@ -132,4 +111,5 @@ public class MockNodeRepository extends NodeRepository { provisioner.activate(transaction, application, hosts); transaction.commit(); } -}
\ No newline at end of file + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index dc30f0ed1a8..985277d17ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -117,7 +117,14 @@ public class AllocationSimulator { NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor); NodeAllocation allocation = new NodeAllocation(app(id), cluster(), nodeSpec, new MutableInteger(0), Clock.systemUTC()); - List<Node> accepted = new ArrayList<>(); //TODO adpot the new allocation algoritm + List<Node> accepted = DockerAllocator.allocateNewDockerNodes(allocation, + nodeSpec, + new ArrayList<>(nodes.asList()), + new ArrayList<>(nodes.asList()), + flavors, + flavor, + 2, + (nodes, message)-> visualizer.addStep(nodes, id, message)); accepted.addAll(nodes.asList()); nodes = new NodeList(accepted); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index 55e1ff8de9f..eae846abb2a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -1,11 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; import org.junit.Before; import org.junit.Test; @@ -63,11 +67,22 @@ public class DockerHostCapacityTest { @Test public void compare_used_to_sort_in_decending_order() { assertEquals(host1, nodes.get(0)); //Make sure it is unsorted here - Collections.sort(nodes, capacity::compare); assertEquals(host3, nodes.get(0)); assertEquals(host1, nodes.get(1)); assertEquals(host2, nodes.get(2)); + + // Replace a node for host 2 with a headroom node - host2 should then be prioritized + Allocation allocation = new Allocation(app(DockerHostCapacity.HEADROOM_TENANT), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host2"), flavorDocker, NodeType.tenant); + Node nodeFWithAllocation = nodeF.with(allocation); + nodes.add(nodeFWithAllocation); + nodes.remove(nodeC); + capacity = new DockerHostCapacity(nodes); + Collections.sort(nodes, capacity::compare); + assertEquals(host3, nodes.get(0)); + assertEquals(host2, nodes.get(1)); + assertEquals(host1, nodes.get(2)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java index 8254cd23030..a5d72947fcc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -16,27 +15,15 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.path.Path; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.node.Agent; -import org.junit.Assert; import org.junit.Test; -import java.time.Instant; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -45,218 +32,6 @@ import static org.junit.Assert.fail; */ public class DynamicDockerProvisioningTest { - /** - * Test reloaction of nodes that violates headroom. - * - * Setup 4 docker hosts and allocate one container on each (from two different applications) - * No spares - only headroom (4xd-2) - * - * One application is now violating headroom and need relocation - * - * Initial allocation of app 1 and 2 --> final allocation (headroom marked as H): - * - * | H | H | H | H | | | | | | - * | H | H | H1a | H1b | --> | | | | | - * | | | 2a | 2b | | 1a | 1b | 2a | 2b | - * - */ - @Test - public void relocate_nodes_from_headroom_hosts() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "a1"); - ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - addAndAssignNode(application1, "1a", dockerHosts.get(2).hostname(), flavor, 0, tester); - addAndAssignNode(application1, "1b", dockerHosts.get(3).hostname(), flavor, 1, tester); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "a2"); - ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester); - addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester); - - // Redeploy one of the applications - deployapp(application1, clusterSpec1, flavor, tester, 2); - - // Assert that the nodes are spread across all hosts (to allow headroom) - Set<String> hostsWithChildren = new HashSet<>(); - for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) { - if (!isInactiveOrRetired(node)) { - hostsWithChildren.add(node.parentHostname().get()); - } - } - Assert.assertEquals(4, hostsWithChildren.size()); - } - - /** - * Test relocation of nodes from spare hosts. - * - * Setup 4 docker hosts and allocate one container on each (from two different applications) - * No headroom defined - only 2 spares. - * - * Check that it relocates containers away from the 2 spares - * - * Initial allocation of app 1 and 2 --> final allocation: - * - * | | | | | | | | | | - * | | | | | --> | 2a | 2b | | | - * | 1a | 1b | 2a | 2b | | 1a | 1b | | | - * - */ - @Test - public void relocate_nodes_from_spare_hosts() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "a1"); - ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester); - addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "a2"); - ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester); - addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester); - - // Redeploy both applications (to be agnostic on which hosts are picked as spares) - deployapp(application1, clusterSpec1, flavor, tester, 2); - deployapp(application2, clusterSpec2, flavor, tester, 2); - - // Assert that we have two spare nodes (two hosts that are don't have allocations) - Set<String> hostsWithChildren = new HashSet<>(); - for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) { - if (!isInactiveOrRetired(node)) { - hostsWithChildren.add(node.parentHostname().get()); - } - } - Assert.assertEquals(2, hostsWithChildren.size()); - } - - /** - * Test an allocation workflow: - * - * 5 Hosts of capacity 3 (2 spares) - * - Allocate app with 3 nodes - * - Allocate app with 2 nodes - * - Fail host and check redistribution - */ - @Test - public void reloacte_failed_nodes() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); - tester.makeReadyNodes(5, "host", "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "a1"); - ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - deployapp(application1, clusterSpec1, flavor, tester, 3); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "a2"); - ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - deployapp(application2, clusterSpec2, flavor, tester, 2); - - // Application 3 - ApplicationId application3 = makeApplicationId("t3", "a3"); - ClusterSpec clusterSpec3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - deployapp(application3, clusterSpec3, flavor, tester, 2); - - // App 2 and 3 should have been allocated to the same nodes - fail on of the parent hosts from there - String parent = tester.nodeRepository().getNodes(application2).stream().findAny().get().parentHostname().get(); - tester.nodeRepository().failRecursively(parent, Agent.system, "Testing"); - - // Redeploy all applications - deployapp(application1, clusterSpec1, flavor, tester, 3); - deployapp(application2, clusterSpec2, flavor, tester, 2); - deployapp(application3, clusterSpec3, flavor, tester, 2); - - Map<Integer, Integer> numberOfChildrenStat = new HashMap<>(); - for (Node node : dockerHosts) { - int nofChildren = tester.nodeRepository().getChildNodes(node.hostname()).size(); - if (!numberOfChildrenStat.containsKey(nofChildren)) { - numberOfChildrenStat.put(nofChildren, 0); - } - numberOfChildrenStat.put(nofChildren, numberOfChildrenStat.get(nofChildren) + 1); - } - - assertEquals(3l, (long)numberOfChildrenStat.get(3)); - assertEquals(1l, (long)numberOfChildrenStat.get(0)); - assertEquals(1l, (long)numberOfChildrenStat.get(1)); - } - - /** - * Test redeployment of nodes that violates spare headroom - but without alternatives - * - * Setup 2 docker hosts and allocate one app with a container on each - * No headroom defined - only 2 spares. - * - * Initial allocation of app 1 --> final allocation: - * - * | | | | | | - * | | | --> | | | - * | 1a | 1b | | 1a | 1b | - * - */ - @Test - public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); - tester.makeReadyNodes(2, "host", "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "a1"); - ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); - addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester); - addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester); - - // Redeploy both applications (to be agnostic on which hosts are picked as spares) - deployapp(application1, clusterSpec1, flavor, tester, 2); - - // Assert that we have two spare nodes (two hosts that are don't have allocations) - Set<String> hostsWithChildren = new HashSet<>(); - for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) { - if (!isInactiveOrRetired(node)) { - hostsWithChildren.add(node.parentHostname().get()); - } - } - Assert.assertEquals(2, hostsWithChildren.size()); - } - - @Test(expected = OutOfCapacityException.class) - public void multiple_groups_are_on_separate_parent_hosts() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); - tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - deployZoneApp(tester); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - //Deploy an application of 6 nodes of 3 nodes in each cluster. We only have 3 docker hosts available - ApplicationId application1 = tester.makeApplicationId(); - tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), - 6, 2, flavor.canonicalName()); - - fail("Two groups have been allocated to the same parent host"); - } - @Test public void spare_capacity_used_only_when_replacement() { // Use spare capacity only when replacement (i.e one node is failed) @@ -301,6 +76,9 @@ public class DynamicDockerProvisioningTest { List<Node> finalSpareCapacity = findSpareCapacity(tester); assertThat(finalSpareCapacity.size(), is(1)); + + // Uncomment the statement below to walk through the allocation events visually + //AllocationVisualizer.visualize(tester.getAllocationSnapshots()); } @Test @@ -321,29 +99,6 @@ public class DynamicDockerProvisioningTest { assertThat(initialSpareCapacity.size(), is(0)); } - private ApplicationId makeApplicationId(String tenant, String appName) { - return ApplicationId.from(tenant, appName, "default"); - } - - private void deployapp(ApplicationId id, ClusterSpec spec, Flavor flavor, ProvisioningTester tester, int nodecount) { - List<HostSpec> hostSpec = tester.prepare(id, spec, nodecount,1, flavor.canonicalName()); - tester.activate(id, new HashSet<>(hostSpec)); - } - - private Node addAndAssignNode(ApplicationId id, String hostname, String parentHostname, Flavor flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), flavor, NodeType.tenant); - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")).changeGroup(Optional.of(ClusterSpec.Group.from(0))); - ClusterMembership clusterMembership1 = ClusterMembership.from(clusterSpec,index); - Node node1aAllocation = node1a.allocate(id,clusterMembership1, Instant.now()); - - tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation)); - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator())); - tester.nodeRepository().activate(Collections.singletonList(node1aAllocation), transaction); - transaction.commit(); - - return node1aAllocation; - } - private List<Node> findSpareCapacity(ProvisioningTester tester) { List<Node> nodes = tester.nodeRepository().getNodes(Node.State.values()); NodeList nl = new NodeList(nodes); @@ -353,22 +108,6 @@ public class DynamicDockerProvisioningTest { .collect(Collectors.toList()); } - private FlavorsConfig flavorsConfig(boolean includeHeadroom) { - FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); - b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); - b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); - b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); - if (includeHeadroom) { - b.addFlavor("d-2-4", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER, 4); - } - b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); - b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); - b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); - b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); - return b.build(); - } - private FlavorsConfig flavorsConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); @@ -397,14 +136,4 @@ public class DynamicDockerProvisioningTest { private void enableDynamicAllocation(ProvisioningTester tester) { tester.getCurator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); } - - private boolean isInactiveOrRetired(Node node) { - boolean isInactive = node.state().equals(Node.State.inactive); - boolean isRetired = false; - if (node.allocation().isPresent()) { - isRetired = node.allocation().get().membership().retired(); - } - - return isInactive || isRetired; - } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index bd50a9fc581..f576f11cd64 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -226,10 +226,6 @@ public class ProvisioningTester implements AutoCloseable { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) { - return makeReadyNodes(n, UUID.randomUUID().toString(), flavor, type, additionalIps); - } - - public List<Node> makeReadyNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { Set<String> ips = IntStream.range(additionalIps * i, additionalIps * (i+1)) @@ -237,7 +233,7 @@ public class ProvisioningTester implements AutoCloseable { .collect(Collectors.toSet()); nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), - prefix + i, + UUID.randomUUID().toString(), Collections.emptySet(), ips, Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 8a74f3aff24..dd2b64a7854 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -42,7 +41,6 @@ public class RestApiTest { /** This test gives examples of all the requests that can be made to nodes/v2 */ @Test - @Ignore /** TODO re-enable this and verify correctness */ public void test_requests() throws Exception { // GET assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); @@ -312,33 +310,33 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/" + hostname, new byte[0], Request.Method.PUT), "{\"message\":\"Moved foo.yahoo.com to ready\"}"); - Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[.*" + + Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" + "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," + "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," + "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}" + - ".*],\"trustedNetworks\":\\[\\]}"); + "],\"trustedNetworks\":\\[\\]}"); assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/" + hostname), responsePattern); } @Test public void acl_request_by_config_server() throws Exception { - Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[.*" + + Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" + "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," + "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," + "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}" + - ".*],\"trustedNetworks\":\\[\\]}"); + "],\"trustedNetworks\":\\[\\]}"); assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/cfg1"), responsePattern); } @Test public void acl_request_by_docker_host() throws Exception { Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" + - "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}," + - "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}," + - "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}]," + + "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," + + "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," + + "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}]," + "\"trustedNetworks\":\\[" + - "\\{\"network\":\"172.17.0.0/16\",\"trustedBy\":\"dockerhost1.yahoo.com\"}]}"); - assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/dockerhost1.yahoo.com"), responsePattern); + "\\{\"network\":\"172.17.0.0/16\",\"trustedBy\":\"parent1.yahoo.com\"}]}"); + assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/parent1.yahoo.com"), responsePattern); } @Test @@ -349,8 +347,8 @@ public class RestApiTest { "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"host1.yahoo.com\"}," + "\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," + "\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}," + - "\\{\"hostname\":\"host10.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," + - "\\{\"hostname\":\"host10.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}" + + "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," + + "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}" + "],\"trustedNetworks\":\\[\\]}"); assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/host1.yahoo.com"), responsePattern); } @@ -372,8 +370,7 @@ public class RestApiTest { "{\"message\":\"Moved host1.yahoo.com to failed\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", new byte[0], Request.Method.PUT), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set failed node host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' ready. It is not dirty.\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set failed node host1.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' ready. It is not dirty.\"}"); // (... while dirty then ready works (the ready move will be initiated by node maintenance)) assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com", @@ -389,7 +386,7 @@ public class RestApiTest { "{\"message\":\"Moved host2.yahoo.com to parked\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", new byte[0], Request.Method.PUT), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set parked node host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' ready. It is not dirty.\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set parked node host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/1' ready. It is not dirty.\"}"); // (... while dirty then ready works (the ready move will be initiated by node maintenance)) assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", new byte[0], Request.Method.PUT), @@ -446,18 +443,18 @@ public class RestApiTest { @Test public void test_hardware_patching_of_docker_host() throws Exception { assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(false)); - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(false)); + assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com"), Optional.of(false)); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com", Utf8.toBytes("{" + "\"hardwareFailureType\": \"memory_mcelog\"" + "}" ), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); + "{\"message\":\"Updated parent1.yahoo.com\"}"); assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(true)); - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(true)); + assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com"), Optional.of(true)); } @Test @@ -509,8 +506,7 @@ public class RestApiTest { private JDisc container; @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } + public void startContainer() { container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } @After public void stopContainer() { container.close(); } @@ -562,10 +558,10 @@ public class RestApiTest { private void assertHardwareFailure(Request request, Optional<Boolean> expectedHardwareFailure) throws CharacterCodingException { Response response = container.handleRequest(request); + assertEquals(response.getStatus(), 200); String json = response.getBodyAsString(); Optional<Boolean> actualHardwareFailure = getHardwareFailure(json); assertEquals(expectedHardwareFailure, actualHardwareFailure); - assertEquals(200, response.getStatus()); } /** Asserts a particular response and 200 as response status */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index 075ce1693cb..9c299cb6fe8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -1,55 +1,41 @@ { "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com", "id": "host1.yahoo.com", - "state": "reserved", + "state": "active", "type": "tenant", "hostname": "host1.yahoo.com", "openStackId": "node1", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb": 400.0, - "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", - "minCpuCores": 2.0, - "fastDisk": true, - "environment": "BARE_METAL", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" }, "membership": { - "clustertype": "container", - "clusterid": "id1", + "clustertype": "content", + "clusterid": "id2", "group": "0", "index": 0, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion":"6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 120d6286634..f788b5c6e59 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -8,12 +8,12 @@ "openStackId": "node10", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb": 400.0, - "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", - "minCpuCores": 2.0, - "fastDisk": true, - "environment": "BARE_METAL", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "owner": { "tenant": "tenant1", "application": "application1", @@ -23,12 +23,12 @@ "clustertype": "container", "clusterid": "id1", "group": "0", - "index": 1, + "index": 0, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, @@ -37,24 +37,10 @@ "hostedVersion": "5.104.142", "convergedStateVersion": "5.104.142", "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 52864fc165c..6a4522bf0a4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -7,12 +7,12 @@ "openStackId": "node2", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb": 400.0, - "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", - "minCpuCores": 2.0, - "fastDisk": true, - "environment": "BARE_METAL", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "owner": { "tenant": "tenant2", "application": "application2", @@ -22,39 +22,20 @@ "clustertype": "content", "clusterid": "id2", "group": "0", - "index": 0, + "index": 1, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion":"6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json index 7782cf15e50..f05e2cd578a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -1,32 +1,39 @@ { "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com", "id": "host3.yahoo.com", - "state": "ready", + "state": "active", "type": "tenant", "hostname": "host3.yahoo.com", "openStackId": "node3", - "flavor": "expensive", - "canonicalFlavor": "default", - "description": "Flavor-name-is-expensive", - "cost": 200, - "fastDisk": true, - "environment": "BARE_METAL", + "flavor":"expensive", + "canonicalFlavor":"default", + "description":"Flavor-name-is-expensive", + "cost":200, + "fastDisk":true, + "environment":"BARE_METAL", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion":"6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 10b5689f8ce..efdde53ffb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -1,34 +1,34 @@ { "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", - "state": "active", + "state": "reserved", "type": "tenant", "hostname": "host4.yahoo.com", - "parentHostname": "dockerhost1.yahoo.com", + "parentHostname":"dockerhost4", "openStackId": "node4", - "flavor": "docker", - "canonicalFlavor": "docker", - "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, - "fastDisk": true, - "environment": "DOCKER_CONTAINER", + "flavor": "default", + "canonicalFlavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "owner": { - "tenant": "tenant3", - "application": "application3", - "instance": "instance3" + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" }, "membership": { - "clustertype": "content", - "clusterid": "id3", + "clustertype": "container", + "clusterid": "id1", "group": "0", - "index": 0, + "index": 1, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, @@ -37,29 +37,10 @@ "hostedVersion": "6.41.0", "convergedStateVersion": "6.41.0", "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json index bf81509b79a..0d0fda0b594 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json @@ -4,37 +4,23 @@ "state": "failed", "type": "tenant", "hostname": "host5.yahoo.com", - "parentHostname": "dockerhost2.yahoo.com", + "parentHostname":"parent1.yahoo.com", "openStackId": "node5", - "flavor": "docker", - "canonicalFlavor": "docker", - "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, - "fastDisk": true, - "environment": "DOCKER_CONTAINER", + "flavor": "default", + "canonicalFlavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 1, "hardwareFailure": false, "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "failed", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index 1fc001fa224..35805e3b20f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -4,16 +4,16 @@ "state": "failed", "type": "tenant", "hostname": "host5.yahoo.com", - "parentHostname": "dockerhost2.yahoo.com", + "parentHostname":"parent1.yahoo.com", "openStackId": "node5", - "flavor": "docker", - "canonicalFlavor": "docker", - "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, - "fastDisk": true, - "environment": "DOCKER_CONTAINER", + "flavor": "default", + "canonicalFlavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "rebootGeneration": 1, "currentRebootGeneration": 0, "vespaVersion": "1.2.3", @@ -21,24 +21,10 @@ "hostedVersion": "1.2.3", "convergedStateVersion": "1.2.3", "failCount": 1, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "failed", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 750ebbd695e..25a9b8554d8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -7,54 +7,35 @@ "openStackId": "node6", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb": 400.0, - "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", - "minCpuCores": 2.0, - "fastDisk": true, - "environment": "BARE_METAL", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "fastDisk":true, + "environment":"BARE_METAL", "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" }, "membership": { "clustertype": "content", - "clusterid": "id2", + "clusterid": "id3", "group": "0", - "index": 1, + "index": 0, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", + "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion":"6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure": false, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::1", - "127.0.0.1" - ], - "additionalIpAddresses": [] -}
\ No newline at end of file + "hardwareFailure" : false, + "wantToRetire" : false, + "wantToDeprovision" : false, + "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], + "ipAddresses":["::1", "127.0.0.1"], + "additionalIpAddresses":[] +} |