diff options
Diffstat (limited to 'node-repository/src/main')
11 files changed, 718 insertions, 450 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceException.java deleted file mode 100644 index e5ab519ab94..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceException.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.lb; - -import com.yahoo.config.provision.TransientException; - -/** - * Transient exception thrown on behalf of a {@link LoadBalancerService}. - * - * @author mpolden - */ -public class LoadBalancerServiceException extends TransientException { - - public LoadBalancerServiceException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java new file mode 100644 index 00000000000..48f846d5e7f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -0,0 +1,526 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Allocation; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class CapacityChecker { + private List<Node> hosts; + Map<String, Node> nodeMap; + private Map<Node, List<Node>> nodeChildren; + private Map<Node, AllocationResources> availableResources; + + public AllocationHistory allocationHistory = null; + + public CapacityChecker(NodeRepository nodeRepository) { + this.hosts = getHosts(nodeRepository); + List<Node> tenants = getTenants(nodeRepository, hosts); + nodeMap = constructHostnameToNodeMap(hosts); + this.nodeChildren = constructNodeChildrenMap(tenants, hosts, nodeMap); + this.availableResources = constructAvailableResourcesMap(hosts, nodeChildren); + } + + public List<Node> getHosts() { + return hosts; + } + + public Optional<HostFailurePath> worstCaseHostLossLeadingToFailure() { + Map<Node, Integer> timesNodeCanBeRemoved = computeMaximalRepeatedRemovals(hosts, nodeChildren, availableResources); + return greedyHeuristicFindFailurePath(timesNodeCanBeRemoved, hosts, nodeChildren, availableResources); + } + + protected List<Node> findOvercommittedHosts() { + return findOvercommittedNodes(availableResources); + } + + public List<Node> nodesFromHostnames(List<String> hostnames) { + List<Node> nodes = hostnames.stream() + .filter(h -> nodeMap.containsKey(h)) + .map(h -> nodeMap.get(h)) + .collect(Collectors.toList()); + if (nodes.size() != hostnames.size()) { + Set<String> notFoundNodes = new HashSet<>(hostnames); + notFoundNodes.removeAll(nodes.stream().map(Node::hostname).collect(Collectors.toList())); + throw new IllegalArgumentException(String.format("Host(s) not found: [ %s ]", + String.join(", ", notFoundNodes))); + } + + return nodes; + } + + public Optional<HostFailurePath> findHostRemovalFailure(List<Node> hostsToRemove) { + var removal = findHostRemovalFailure(hostsToRemove, hosts, nodeChildren, availableResources); + if (removal.isEmpty()) return Optional.empty(); + HostFailurePath failurePath = new HostFailurePath(); + failurePath.hostsCausingFailure = hostsToRemove; + failurePath.failureReason = removal.get(); + return Optional.of(failurePath); + } + + // We only care about nodes in one of these states. + private static Node.State[] relevantNodeStates = { + Node.State.active, + Node.State.inactive, + Node.State.dirty, + Node.State.provisioned, + Node.State.ready, + Node.State.reserved + }; + + private List<Node> getHosts(NodeRepository nodeRepository) { + return nodeRepository.getNodes(NodeType.host, relevantNodeStates); + } + + private List<Node> getTenants(NodeRepository nodeRepository, List<Node> hosts) { + var parentNames = hosts.stream().map(Node::hostname).collect(Collectors.toSet()); + return nodeRepository.getNodes(NodeType.tenant, relevantNodeStates).stream() + .filter(t -> parentNames.contains(t.parentHostname().orElse(""))) + .collect(Collectors.toList()); + } + + private Optional<HostFailurePath> greedyHeuristicFindFailurePath(Map<Node, Integer> heuristic, List<Node> hosts, + Map<Node, List<Node>> nodeChildren, + Map<Node, AllocationResources> availableResources) { + if (hosts.size() == 0) return Optional.empty(); + + List<Node> parentRemovalPriorityList = heuristic.entrySet().stream() + .sorted(Comparator.comparingInt(Map.Entry::getValue)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + for (int i = 1; i <= parentRemovalPriorityList.size(); i++) { + List<Node> hostsToRemove = parentRemovalPriorityList.subList(0, i); + var hostRemovalFailure = findHostRemovalFailure(hostsToRemove, hosts, nodeChildren, availableResources); + if (hostRemovalFailure.isPresent()) { + HostFailurePath failurePath = new HostFailurePath(); + failurePath.hostsCausingFailure = hostsToRemove; + failurePath.failureReason = hostRemovalFailure.get(); + return Optional.of(failurePath); + } + } + + throw new IllegalStateException("No path to failure found. This should be impossible!"); + } + + private Map<String, Node> constructHostnameToNodeMap(List<Node> nodes) { + return nodes.stream().collect(Collectors.toMap(Node::hostname, n -> n)); + } + + private Map<Node, List<Node>> constructNodeChildrenMap(List<Node> tenants, List<Node> hosts, Map<String, Node> hostnameToNode) { + Map<Node, List<Node>> nodeChildren = tenants.stream() + .filter(n -> n.parentHostname().isPresent()) + .filter(n -> hostnameToNode.containsKey(n.parentHostname().get())) + .collect(Collectors.groupingBy( + n -> hostnameToNode.get(n.parentHostname().orElseThrow()))); + + for (var host : hosts) nodeChildren.putIfAbsent(host, List.of()); + + return nodeChildren; + } + + private Map<Node, AllocationResources> constructAvailableResourcesMap(List<Node> hosts, Map<Node, List<Node>> nodeChildren) { + Map<Node, AllocationResources> availableResources = new HashMap<>(); + for (var host : hosts) { + NodeResources hostResources = host.flavor().resources(); + int occupiedIps = 0; + Set<String> ipPool = host.ipAddressPool().asSet(); + for (var child : nodeChildren.get(host)) { + hostResources = hostResources.subtract(child.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any)); + occupiedIps += child.ipAddresses().stream().filter(ipPool::contains).count(); + } + availableResources.put(host, new AllocationResources(hostResources, host.ipAddressPool().asSet().size() - occupiedIps)); + } + + return availableResources; + } + + /** + * Computes a heuristic for each host, with a lower score indicating a higher perceived likelihood that removing + * the host causes an unrecoverable state + */ + private Map<Node, Integer> computeMaximalRepeatedRemovals(List<Node> hosts, Map<Node, List<Node>> nodeChildren, + Map<Node, AllocationResources> availableResources) { + Map<Node, Integer> timesNodeCanBeRemoved = hosts.stream().collect(Collectors.toMap( + Function.identity(), + _x -> Integer.MAX_VALUE + )); + for (Node host : hosts) { + List<Node> children = nodeChildren.get(host); + if (children.size() == 0) continue; + Map<Node, AllocationResources> resourceMap = new HashMap<>(availableResources); + Map<Node, List<Allocation>> containedAllocations = collateAllocations(nodeChildren); + + int timesHostCanBeRemoved = 0; + Optional<Node> unallocatedNode; + while (timesHostCanBeRemoved < 1000) { // Arbritrary upper bound + unallocatedNode = tryAllocateNodes(nodeChildren.get(host), hosts, resourceMap, containedAllocations); + if (unallocatedNode.isEmpty()) { + timesHostCanBeRemoved++; + } else break; + } + timesNodeCanBeRemoved.put(host, timesHostCanBeRemoved); + } + + return timesNodeCanBeRemoved; + } + + private List<Node> findOvercommittedNodes(Map<Node, AllocationResources> availableResources) { + List<Node> overcommittedNodes = new ArrayList<>(); + for (var entry : availableResources.entrySet()) { + var resources = entry.getValue().nodeResources; + if (resources.vcpu() < 0 || resources.memoryGb() < 0 || resources.diskGb() < 0) { + overcommittedNodes.add(entry.getKey()); + } + } + return overcommittedNodes; + } + + private Map<Node, List<Allocation>> collateAllocations(Map<Node, List<Node>> nodeChildren) { + return nodeChildren.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream() + .map(Node::allocation).flatMap(Optional::stream) + .collect(Collectors.toList()) + )); + } + + /** + * Tests whether it's possible to remove the provided hosts. + * Does not mutate any input variable. + * @return Empty optional if removal is possible, information on what caused the failure otherwise + */ + private Optional<HostRemovalFailure> findHostRemovalFailure(List<Node> hostsToRemove, List<Node> allHosts, + Map<Node, List<Node>> nodechildren, + Map<Node, AllocationResources> availableResources) { + var containedAllocations = collateAllocations(nodechildren); + var resourceMap = new HashMap<>(availableResources); + List<Node> validAllocationTargets = allHosts.stream() + .filter(h -> !hostsToRemove.contains(h)) + .collect(Collectors.toList()); + if (validAllocationTargets.size() == 0) { + return Optional.of(HostRemovalFailure.none()); + } + + allocationHistory = new AllocationHistory(); + for (var host : hostsToRemove) { + Optional<Node> unallocatedNode = tryAllocateNodes(nodechildren.get(host), + validAllocationTargets, resourceMap, containedAllocations, true); + + if (unallocatedNode.isPresent()) { + AllocationFailureReasonList failures = collateAllocationFailures(unallocatedNode.get(), + validAllocationTargets, resourceMap, containedAllocations); + return Optional.of(HostRemovalFailure.create(host, unallocatedNode.get(), failures)); + } + } + return Optional.empty(); + } + + /** + * Attempts to allocate the listed nodes to a new host, mutating availableResources and containedAllocations, + * optionally returning the first node to fail, if one does. + * */ + private Optional<Node> tryAllocateNodes(List<Node> nodes, List<Node> hosts, + Map<Node, AllocationResources> availableResources, + Map<Node, List<Allocation>> containedAllocations) { + return tryAllocateNodes(nodes, hosts, availableResources, containedAllocations, false); + } + private Optional<Node> tryAllocateNodes(List<Node> nodes, List<Node> hosts, + Map<Node, AllocationResources> availableResources, + Map<Node, List<Allocation>> containedAllocations, boolean withHistory) { + for (var node : nodes) { + var newParent = tryAllocateNode(node, hosts, availableResources, containedAllocations); + if (newParent.isEmpty()) { + if (withHistory) allocationHistory.addEntry(node, null, 0); + return Optional.of(node); + } + if (withHistory) { + long eligibleParents = + hosts.stream().filter(h -> + !violatesParentHostPolicy(node, h, containedAllocations) + && availableResources.get(h).satisfies(AllocationResources.from(node.flavor().resources()))).count(); + allocationHistory.addEntry(node, newParent.get(), eligibleParents + 1); + } + } + return Optional.empty(); + } + + /** + * @return The parent to which the node was allocated, if it was successfully allocated. + */ + private Optional<Node> tryAllocateNode(Node node, List<Node> hosts, + Map<Node, AllocationResources> availableResources, + Map<Node, List<Allocation>> containedAllocations) { + AllocationResources requiredNodeResources = AllocationResources.from(node.flavor().resources()); + for (var host : hosts) { + var availableHostResources = availableResources.get(host); + if (violatesParentHostPolicy(node, host, containedAllocations)) { + continue; + } + if (availableHostResources.satisfies(requiredNodeResources)) { + availableResources.put(host, availableHostResources.subtract(requiredNodeResources)); + if (node.allocation().isPresent()) { + containedAllocations.get(host).add(node.allocation().get()); + } + return Optional.of(host); + } + } + + return Optional.empty(); + } + + private static boolean violatesParentHostPolicy(Node node, Node host, Map<Node, List<Allocation>> containedAllocations) { + if (node.allocation().isEmpty()) return false; + Allocation nodeAllocation = node.allocation().get(); + for (var allocation : containedAllocations.get(host)) { + if (allocation.membership().cluster().equalsIgnoringGroupAndVespaVersion(nodeAllocation.membership().cluster()) + && allocation.owner().equals(nodeAllocation.owner())) { + return true; + } + } + return false; + } + + private AllocationFailureReasonList collateAllocationFailures(Node node, List<Node> hosts, + Map<Node, AllocationResources> availableResources, + Map<Node, List<Allocation>> containedAllocations) { + List<AllocationFailureReason> allocationFailureReasons = new ArrayList<>(); + for (var host : hosts) { + AllocationFailureReason reason = new AllocationFailureReason(host); + var availableHostResources = availableResources.get(host); + reason.violatesParentHostPolicy = violatesParentHostPolicy(node, host, containedAllocations); + + NodeResources l = availableHostResources.nodeResources; + NodeResources r = node.flavor().resources(); + if (l.vcpu() < r.vcpu()) { reason.insufficientVcpu = true; } + if (l.memoryGb() < r.memoryGb()) { reason.insufficientMemoryGb = true; } + if (l.diskGb() < r.diskGb()) { reason.insufficientDiskGb = true; } + if (r.diskSpeed() != NodeResources.DiskSpeed.any && r.diskSpeed() != l.diskSpeed()) + { reason.incompatibleDiskSpeed = true; } + if (availableHostResources.availableIPs < 1) { reason.insufficientAvailableIPs = true; } + + allocationFailureReasons.add(reason); + } + + return new AllocationFailureReasonList(allocationFailureReasons); + } + + /** + * Contains the list of hosts that, upon being removed, caused an unrecoverable state, + * as well as the specific host and tenant which caused it. + */ + public static class HostFailurePath { + public List<Node> hostsCausingFailure; + public HostRemovalFailure failureReason; + } + + /** + * Data class used for detailing why removing the given tenant from the given host was unsuccessful. + * A failure might not be caused by failing to allocate a specific tenant, in which case the fields + * will be empty. + */ + public static class HostRemovalFailure { + public Optional<Node> host; + public Optional<Node> tenant; + public AllocationFailureReasonList allocationFailures; + + public static HostRemovalFailure none() { + return new HostRemovalFailure( + Optional.empty(), + Optional.empty(), + new AllocationFailureReasonList(List.of())); + } + + public static HostRemovalFailure create(Node host, Node tenant, AllocationFailureReasonList failureReasons) { + return new HostRemovalFailure( + Optional.of(host), + Optional.of(tenant), + failureReasons); + } + + private HostRemovalFailure(Optional<Node> host, Optional<Node> tenant, AllocationFailureReasonList allocationFailures) { + this.host = host; + this.tenant = tenant; + this.allocationFailures = allocationFailures; + } + + @Override + public String toString() { + if (host.isEmpty() || tenant.isEmpty()) return "No removal candidates exists."; + return String.format( + "Failure to remove host %s" + + "\n\tNo new host found for tenant %s:" + + "\n\t\tSingular Reasons: %s" + + "\n\t\tTotal Reasons: %s", + this.host.get().hostname(), + this.tenant.get().hostname(), + this.allocationFailures.singularReasonFailures().toString(), + this.allocationFailures.toString() + ); + } + } + + /** + * Used to describe the resources required for a tenant, and available to a host. + */ + private static class AllocationResources { + NodeResources nodeResources; + int availableIPs; + + public static AllocationResources from(NodeResources nodeResources) { + return new AllocationResources(nodeResources, 1); + } + + public AllocationResources(NodeResources nodeResources, int availableIPs) { + this.nodeResources = nodeResources; + this.availableIPs = availableIPs; + } + + public boolean satisfies(AllocationResources other) { + if (!this.nodeResources.satisfies(other.nodeResources)) return false; + return this.availableIPs >= other.availableIPs; + } + + public AllocationResources subtract(AllocationResources other) { + return new AllocationResources(this.nodeResources.subtract(other.nodeResources), this.availableIPs - other.availableIPs); + } + } + + /** + * Keeps track of the reason why a host rejected an allocation. + */ + private static class AllocationFailureReason { + Node host; + public AllocationFailureReason (Node host) { + this.host = host; + } + public boolean insufficientVcpu = false; + public boolean insufficientMemoryGb = false; + public boolean insufficientDiskGb = false; + public boolean incompatibleDiskSpeed = false; + public boolean insufficientAvailableIPs = false; + public boolean violatesParentHostPolicy = false; + + public int numberOfReasons() { + int n = 0; + if (insufficientVcpu) n++; + if (insufficientMemoryGb) n++; + if (insufficientDiskGb) n++; + if (incompatibleDiskSpeed) n++; + if (insufficientAvailableIPs) n++; + if (violatesParentHostPolicy) n++; + return n; + } + + @Override + public String toString() { + List<String> reasons = new ArrayList<>(); + if (insufficientVcpu) reasons.add("insufficientVcpu"); + if (insufficientMemoryGb) reasons.add("insufficientMemoryGb"); + if (insufficientDiskGb) reasons.add("insufficientDiskGb"); + if (incompatibleDiskSpeed) reasons.add("incompatibleDiskSpeed"); + if (insufficientAvailableIPs) reasons.add("insufficientAvailableIPs"); + if (violatesParentHostPolicy) reasons.add("violatesParentHostPolicy"); + + return String.format("[%s]", String.join(", ", reasons)); + } + } + + /** + * Provides convenient methods for tallying failures. + */ + public static class AllocationFailureReasonList { + private List<AllocationFailureReason> allocationFailureReasons; + public AllocationFailureReasonList(List<AllocationFailureReason> allocationFailureReasons) { + this.allocationFailureReasons = allocationFailureReasons; + } + + public long insufficientVcpu() { return allocationFailureReasons.stream().filter(r -> r.insufficientVcpu).count(); } + public long insufficientMemoryGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientMemoryGb).count(); } + public long insufficientDiskGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientDiskGb).count(); } + public long incompatibleDiskSpeed() { return allocationFailureReasons.stream().filter(r -> r.incompatibleDiskSpeed).count(); } + public long insufficientAvailableIps() { return allocationFailureReasons.stream().filter(r -> r.insufficientAvailableIPs).count(); } + public long violatesParentHostPolicy() { return allocationFailureReasons.stream().filter(r -> r.violatesParentHostPolicy).count(); } + + public AllocationFailureReasonList singularReasonFailures() { + return new AllocationFailureReasonList(allocationFailureReasons.stream() + .filter(reason -> reason.numberOfReasons() == 1).collect(Collectors.toList())); + } + public AllocationFailureReasonList multipleReasonFailures() { + return new AllocationFailureReasonList(allocationFailureReasons.stream() + .filter(reason -> reason.numberOfReasons() > 1).collect(Collectors.toList())); + } + public long size() { + return allocationFailureReasons.size(); + } + @Override + public String toString() { + return String.format("CPU (%3d), Memory (%3d), Disk size (%3d), Disk speed (%3d), IP (%3d), Parent-Host Policy (%3d)", + insufficientVcpu(), insufficientMemoryGb(), insufficientDiskGb(), + incompatibleDiskSpeed(), insufficientAvailableIps(), violatesParentHostPolicy()); + } + } + + public static class AllocationHistory { + public static class Entry { + public Node tenant; + public Node newParent; + public long eligibleParents; + + public Entry(Node tenant, Node newParent, long eligibleParents) { + this.tenant = tenant; + this.newParent = newParent; + this.eligibleParents = eligibleParents; + } + + @Override + public String toString() { + return String.format("%-20s %-65s -> %15s [%3d valid]", + tenant.hostname().replaceFirst("\\..+", ""), + tenant.flavor().resources(), + newParent == null ? "x" : newParent.hostname().replaceFirst("\\..+", ""), + this.eligibleParents + ); + } + } + + public List<Entry> historyEntries; + + public AllocationHistory() { + this.historyEntries = new ArrayList<>(); + } + + public void addEntry(Node tenant, Node newParent, long eligibleParents) { + this.historyEntries.add(new Entry(tenant, newParent, eligibleParents)); + } + + public Set<String> oldParents() { + Set<String> oldParents = new HashSet<>(); + for (var entry : historyEntries) + entry.tenant.parentHostname().ifPresent(oldParents::add); + return oldParents; + } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + + String currentParent = ""; + for (var entry : historyEntries) { + String parentName = entry.tenant.parentHostname().orElseThrow(); + if (!parentName.equals(currentParent)) { + currentParent = parentName; + out.append(parentName).append("\n"); + } + out.append(entry.toString()).append("\n"); + } + + return out.toString(); + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index 44d43081ef2..3c47e418b94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -1,23 +1,15 @@ package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.logging.Logger; import java.util.stream.Collectors; -import com.yahoo.vespa.hosted.provision.node.Allocation; - import java.util.*; -import java.util.function.Function; /** * Performs analysis on the node repository to produce metrics that pertain to the capacity of the node repository. @@ -29,7 +21,6 @@ import java.util.function.Function; * @author mgimle */ public class CapacityReportMaintainer extends Maintainer { - private final Metric metric; private final NodeRepository nodeRepository; private static final Logger log = Logger.getLogger(CapacityReportMaintainer.class.getName()); @@ -44,403 +35,20 @@ public class CapacityReportMaintainer extends Maintainer { @Override protected void maintain() { - metric.set("overcommittedHosts", countOvercommittedHosts(), null); - - Optional<HostFailurePath> failurePath = worstCaseHostLossLeadingToFailure(); - if (failurePath.isPresent()) { - int worstCaseHostLoss = failurePath.get().hostsCausingFailure.size(); - metric.set("spareHostCapacity", worstCaseHostLoss - 1, null); - } - } - - protected Optional<HostFailurePath> worstCaseHostLossLeadingToFailure() { - List<Node> hosts = getHosts(); - List<Node> tenants = getTenants(hosts); - Map<String, Node> nodeMap = constructHostnameToNodeMap(hosts); - Map<Node, List<Node>> nodeChildren = constructNodeChildrenMap(tenants, hosts, nodeMap); - Map<Node, AllocationResources> availableResources = constructAvailableResourcesMap(hosts, nodeChildren); - - Map<Node, Integer> timesNodeCanBeRemoved = computeMaximalRepeatedRemovals(hosts, nodeChildren, availableResources); - return greedyHeuristicFindFailurePath(timesNodeCanBeRemoved, hosts, nodeChildren, availableResources); - } - - // We only care about nodes in one of these states. - private Node.State[] relevantNodeStates = { - Node.State.active, - Node.State.inactive, - Node.State.dirty, - Node.State.provisioned, - Node.State.ready, - Node.State.reserved - }; - - private List<Node> getHosts() { - return nodeRepository.getNodes(NodeType.host, relevantNodeStates); - } - - private List<Node> getTenants(List<Node> hosts) { - var parentNames = hosts.stream().map(Node::hostname).collect(Collectors.toSet()); - return nodeRepository.getNodes(NodeType.tenant, relevantNodeStates).stream() - .filter(t -> parentNames.contains(t.parentHostname().orElse(""))) - .collect(Collectors.toList()); - } - - private Optional<HostFailurePath> greedyHeuristicFindFailurePath(Map<Node, Integer> heuristic, List<Node> hosts, - Map<Node, List<Node>> nodeChildren, - Map<Node, AllocationResources> availableResources) { - if (hosts.size() == 0) return Optional.empty(); - List<Node> parentRemovalPriorityList = heuristic.entrySet().stream() - .sorted(Comparator.comparingInt(Map.Entry::getValue)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - for (int i = 1; i <= parentRemovalPriorityList.size(); i++) { - List<Node> hostsToRemove = parentRemovalPriorityList.subList(0, i); - var hostRemovalFailure = findHostRemovalFailure(hostsToRemove, hosts, nodeChildren, availableResources); - if (hostRemovalFailure.isPresent()) { - HostFailurePath failurePath = new HostFailurePath(); - failurePath.hostsCausingFailure = hostsToRemove; - failurePath.failureReason = hostRemovalFailure.get(); - return Optional.of(failurePath); + if (!nodeRepository.zone().cloud().value().equals("aws")) { + CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); + List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); + if (overcommittedHosts.size() != 0) { + log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), + overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); } - } - - throw new IllegalStateException("No path to failure found. This should be impossible!"); - } - - protected int countOvercommittedHosts() { - List<Node> hosts = getHosts(); - List<Node> tenants = getTenants(hosts); - var nodeMap = constructHostnameToNodeMap(hosts); - var nodeChildren = constructNodeChildrenMap(tenants, hosts, nodeMap); - var availableResources = constructAvailableResourcesMap(hosts, nodeChildren); - - List<Node> overcommittedNodes = findOvercommittedNodes(availableResources); - if (overcommittedNodes.size() != 0) { - log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedNodes.size(), - overcommittedNodes.stream().map(Node::hostname).collect(Collectors.joining(", ")))); - } - return overcommittedNodes.size(); - } - - private Map<String, Node> constructHostnameToNodeMap(List<Node> nodes) { - return nodes.stream().collect(Collectors.toMap(Node::hostname, n -> n)); - } - - private Map<Node, List<Node>> constructNodeChildrenMap(List<Node> tenants, List<Node> hosts, Map<String, Node> hostnameToNode) { - Map<Node, List<Node>> nodeChildren = tenants.stream() - .filter(n -> n.parentHostname().isPresent()) - .filter(n -> hostnameToNode.containsKey(n.parentHostname().get())) - .collect(Collectors.groupingBy( - n -> hostnameToNode.get(n.parentHostname().orElseThrow()))); - - for (var host : hosts) nodeChildren.putIfAbsent(host, List.of()); - - return nodeChildren; - } - - private Map<Node, AllocationResources> constructAvailableResourcesMap(List<Node> hosts, Map<Node, List<Node>> nodeChildren) { - Map<Node, AllocationResources> availableResources = new HashMap<>(); - for (var host : hosts) { - NodeResources hostResources = host.flavor().resources(); - int occupiedIps = 0; - Set<String> ipPool = host.ipAddressPool().asSet(); - for (var child : nodeChildren.get(host)) { - hostResources = hostResources.subtract(child.flavor().resources()); - occupiedIps += child.ipAddresses().stream().filter(ipPool::contains).count(); - } - availableResources.put(host, new AllocationResources(hostResources, host.ipAddressPool().asSet().size() - occupiedIps)); - } - - return availableResources; - } - - /** - * Computes a heuristic for each host, with a lower score indicating a higher perceived likelihood that removing - * the host causes an unrecoverable state - */ - private Map<Node, Integer> computeMaximalRepeatedRemovals(List<Node> hosts, Map<Node, List<Node>> nodeChildren, - Map<Node, AllocationResources> availableResources) { - Map<Node, Integer> timesNodeCanBeRemoved = hosts.stream().collect(Collectors.toMap( - Function.identity(), - _x -> Integer.MAX_VALUE - )); - for (Node host : hosts) { - List<Node> children = nodeChildren.get(host); - if (children.size() == 0) continue; - Map<Node, AllocationResources> resourceMap = new HashMap<>(availableResources); - Map<Node, List<Allocation>> containedAllocations = collateAllocations(nodeChildren); - - int timesHostCanBeRemoved = 0; - Optional<Node> unallocatedTenant; - while (timesHostCanBeRemoved < 1000) { // Arbritrary upper bound - unallocatedTenant = tryAllocateNodes(nodeChildren.get(host), hosts, resourceMap, containedAllocations); - if (unallocatedTenant.isEmpty()) { - timesHostCanBeRemoved++; - } else break; - } - timesNodeCanBeRemoved.put(host, timesHostCanBeRemoved); - } - - return timesNodeCanBeRemoved; - } - - private List<Node> findOvercommittedNodes(Map<Node, AllocationResources> availableResources) { - List<Node> overcommittedNodes = new ArrayList<>(); - for (var entry : availableResources.entrySet()) { - var resources = entry.getValue().nodeResources; - if (resources.vcpu() < 0 || resources.memoryGb() < 0 || resources.diskGb() < 0) { - overcommittedNodes.add(entry.getKey()); - } - } - return overcommittedNodes; - } - - private Map<Node, List<Allocation>> collateAllocations(Map<Node, List<Node>> nodeChildren) { - return nodeChildren.entrySet().stream().collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().stream() - .map(Node::allocation).flatMap(Optional::stream) - .collect(Collectors.toList()) - )); - } - - /** - * Tests whether it's possible to remove the provided hosts. - * Does not mutate any input variable. - * @return Empty optional if removal is possible, information on what caused the failure otherwise - */ - private Optional<HostRemovalFailure> findHostRemovalFailure(List<Node> hostsToRemove, List<Node> allHosts, - Map<Node, List<Node>> nodechildren, - Map<Node, AllocationResources> availableResources) { - var containedAllocations = collateAllocations(nodechildren); - var resourceMap = new HashMap<>(availableResources); - List<Node> validAllocationTargets = allHosts.stream() - .filter(h -> !hostsToRemove.contains(h)) - .collect(Collectors.toList()); - if (validAllocationTargets.size() == 0) { - return Optional.of(HostRemovalFailure.none()); - } - - for (var host : hostsToRemove) { - Optional<Node> unallocatedNode = tryAllocateNodes(nodechildren.get(host), - validAllocationTargets, resourceMap, containedAllocations); - - if (unallocatedNode.isPresent()) { - AllocationFailureReasonList failures = collateAllocationFailures(unallocatedNode.get(), - validAllocationTargets, resourceMap, containedAllocations); - return Optional.of(HostRemovalFailure.create(host, unallocatedNode.get(), failures)); - } - } - return Optional.empty(); - } + metric.set("overcommittedHosts", overcommittedHosts.size(), null); - /** - * Attempts to allocate the listed nodes to a new host, mutating availableResources and containedAllocations, - * optionally returning the first node to fail, if one does. - * */ - private Optional<Node> tryAllocateNodes(List<Node> nodes, List<Node> hosts, - Map<Node, AllocationResources> availableResources, - Map<Node, List<Allocation>> containedAllocations) { - for (var node : nodes) { - if (!tryAllocateNode(node, hosts, availableResources, containedAllocations)) { - return Optional.of(node); + Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); + if (failurePath.isPresent()) { + int worstCaseHostLoss = failurePath.get().hostsCausingFailure.size(); + metric.set("spareHostCapacity", worstCaseHostLoss - 1, null); } } - return Optional.empty(); - } - - private boolean tryAllocateNode(Node node, List<Node> hosts, - Map<Node, AllocationResources> availableResources, - Map<Node, List<Allocation>> containedAllocations) { - AllocationResources requiredNodeResources = AllocationResources.from(node.flavor().resources()); - for (var host : hosts) { - var availableHostResources = availableResources.get(host); - if (violatesParentHostPolicy(node, host, containedAllocations)) { - continue; - } - if (availableHostResources.satisfies(requiredNodeResources)) { - availableResources.put(host, availableHostResources.subtract(requiredNodeResources)); - if (node.allocation().isPresent()) { - containedAllocations.get(host).add(node.allocation().get()); - } - return true; - } - } - - return false; - } - - private boolean violatesParentHostPolicy(Node node, Node host, Map<Node, List<Allocation>> containedAllocations) { - if (node.allocation().isEmpty()) return false; - Allocation nodeAllocation = node.allocation().get(); - for (var allocation : containedAllocations.get(host)) { - if (allocation.membership().cluster().equalsIgnoringGroupAndVespaVersion(nodeAllocation.membership().cluster()) - && allocation.owner().equals(nodeAllocation.owner())) { - return true; - } - } - return false; - } - - private AllocationFailureReasonList collateAllocationFailures(Node node, List<Node> hosts, - Map<Node, AllocationResources> availableResources, - Map<Node, List<Allocation>> containedAllocations) { - List<AllocationFailureReason> allocationFailureReasons = new ArrayList<>(); - for (var host : hosts) { - AllocationFailureReason reason = new AllocationFailureReason(host); - var availableHostResources = availableResources.get(host); - reason.violatesParentHostPolicy = violatesParentHostPolicy(node, host, containedAllocations); - - NodeResources l = availableHostResources.nodeResources; - NodeResources r = node.flavor().resources(); - if (l.vcpu() < r.vcpu()) { reason.insufficientVcpu = true; } - if (l.memoryGb() < r.memoryGb()) { reason.insufficientMemoryGb = true; } - if (l.diskGb() < r.diskGb()) { reason.insufficientDiskGb = true; } - if (r.diskSpeed() != NodeResources.DiskSpeed.any && r.diskSpeed() != l.diskSpeed()) - { reason.incompatibleDiskSpeed = true; } - if (availableHostResources.availableIPs < 1) { reason.insufficientAvailableIPs = true; } - - allocationFailureReasons.add(reason); - } - - return new AllocationFailureReasonList(allocationFailureReasons); - } - - /** - * Contains the list of hosts that, upon being removed, caused an unrecoverable state, - * as well as the specific host and tenant which caused it. - */ - public static class HostFailurePath { - List<Node> hostsCausingFailure; - HostRemovalFailure failureReason; - } - - /** - * Data class used for detailing why removing the given tenant from the given host was unsuccessful. - * A failure might not be caused by failing to allocate a specific tenant, in which case the fields - * will be empty. - */ - public static class HostRemovalFailure { - Optional<Node> host; - Optional<Node> tenant; - AllocationFailureReasonList failureReasons; - public static HostRemovalFailure none() { - return new HostRemovalFailure( - Optional.empty(), - Optional.empty(), - new AllocationFailureReasonList(List.of())); - } - public static HostRemovalFailure create(Node host, Node tenant, AllocationFailureReasonList failureReasons) { - return new HostRemovalFailure( - Optional.of(host), - Optional.of(tenant), - failureReasons); - } - private HostRemovalFailure(Optional<Node> host, Optional<Node> tenant, AllocationFailureReasonList failureReasons) { - this.host = host; - this.tenant = tenant; - this.failureReasons = failureReasons; - } - } - - /** - * Used to describe the resources required for a tenant, and available to a host. - */ - private static class AllocationResources { - NodeResources nodeResources; - int availableIPs; - - public static AllocationResources from(NodeResources nodeResources) { - return new AllocationResources(nodeResources, 1); - } - - public AllocationResources(NodeResources nodeResources, int availableIPs) { - this.nodeResources = nodeResources; - this.availableIPs = availableIPs; - } - - public boolean satisfies(AllocationResources other) { - if (!this.nodeResources.satisfies(other.nodeResources)) return false; - return this.availableIPs >= other.availableIPs; - } - - public AllocationResources subtract(AllocationResources other) { - return new AllocationResources(this.nodeResources.subtract(other.nodeResources), this.availableIPs - other.availableIPs); - } - } - - /** - * Keeps track of the reason why a host rejected an allocation. - */ - private class AllocationFailureReason { - Node host; - public AllocationFailureReason (Node host) { - this.host = host; - } - public boolean insufficientVcpu = false; - public boolean insufficientMemoryGb = false; - public boolean insufficientDiskGb = false; - public boolean incompatibleDiskSpeed = false; - public boolean insufficientAvailableIPs = false; - public boolean violatesParentHostPolicy = false; - - public int numberOfReasons() { - int n = 0; - if (insufficientVcpu) n++; - if (insufficientMemoryGb) n++; - if (insufficientDiskGb) n++; - if (incompatibleDiskSpeed) n++; - if (insufficientAvailableIPs) n++; - if (violatesParentHostPolicy) n++; - return n; - } - - @Override - public String toString() { - List<String> reasons = new ArrayList<>(); - if (insufficientVcpu) reasons.add("insufficientVcpu"); - if (insufficientMemoryGb) reasons.add("insufficientMemoryGb"); - if (insufficientDiskGb) reasons.add("insufficientDiskGb"); - if (incompatibleDiskSpeed) reasons.add("incompatibleDiskSpeed"); - if (insufficientAvailableIPs) reasons.add("insufficientAvailableIPs"); - if (violatesParentHostPolicy) reasons.add("violatesParentHostPolicy"); - - return String.format("[%s]", String.join(", ", reasons)); - } - } - - /** - * Provides convenient methods for tallying failures. - */ - public static class AllocationFailureReasonList { - private List<AllocationFailureReason> allocationFailureReasons; - public AllocationFailureReasonList(List<AllocationFailureReason> allocationFailureReasons) { - this.allocationFailureReasons = allocationFailureReasons; - } - - long insufficientVcpu() { return allocationFailureReasons.stream().filter(r -> r.insufficientVcpu).count(); } - long insufficientMemoryGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientMemoryGb).count(); } - long insufficientDiskGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientDiskGb).count(); } - long incompatibleDiskSpeed() { return allocationFailureReasons.stream().filter(r -> r.incompatibleDiskSpeed).count(); } - long insufficientAvailableIps() { return allocationFailureReasons.stream().filter(r -> r.insufficientAvailableIPs).count(); } - long violatesParentHostPolicy() { return allocationFailureReasons.stream().filter(r -> r.violatesParentHostPolicy).count(); } - - public AllocationFailureReasonList singularReasonFailures() { - return new AllocationFailureReasonList(allocationFailureReasons.stream() - .filter(reason -> reason.numberOfReasons() == 1).collect(Collectors.toList())); - } - public AllocationFailureReasonList multipleReasonFailures() { - return new AllocationFailureReasonList(allocationFailureReasons.stream() - .filter(reason -> reason.numberOfReasons() > 1).collect(Collectors.toList())); - } - public long size() { - return allocationFailureReasons.size(); - } - @Override - public String toString() { - return String.format("CPU (%3d), Memory (%3d), Disk size (%3d), Disk speed (%3d), IP (%3d), Parent-Host Policy (%3d)", - insufficientVcpu(), insufficientMemoryGb(), insufficientDiskGb(), - incompatibleDiskSpeed(), insufficientAvailableIps(), violatesParentHostPolicy()); - } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index f661977d933..bb1ff637f08 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -82,7 +82,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { new HostProvisionMaintainer(nodeRepository, durationFromEnv("host_provisioner_interval").orElse(defaults.hostProvisionerInterval), hostProvisioner, flagSource)); hostDeprovisionMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> new HostDeprovisionMaintainer(nodeRepository, durationFromEnv("host_deprovisioner_interval").orElse(defaults.hostDeprovisionerInterval), hostProvisioner, flagSource)); - capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, durationFromEnv("alert_interval").orElse(defaults.nodeAlerterInterval)); + capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, durationFromEnv("capacity_report_interval").orElse(defaults.capacityReportInterval)); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); @@ -143,7 +143,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration dirtyExpiry; private final Duration provisionedExpiry; private final Duration rebootInterval; - private final Duration nodeAlerterInterval; + private final Duration capacityReportInterval; private final Duration metricsInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; @@ -162,7 +162,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failedExpirerInterval = Duration.ofMinutes(10); provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); - nodeAlerterInterval = Duration.ofHours(1); + capacityReportInterval = Duration.ofHours(1); metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index 812c370df5f..f46e2f501bc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -7,5 +7,5 @@ package com.yahoo.vespa.hosted.provision.node; * @author bratseth */ public enum Agent { - system, application, operator, NodeRetirer, NodeFailer + system, application, operator, NodeFailer } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index cf6531c0748..6198183be89 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -385,7 +385,6 @@ public class NodeSerializer { case "application" : return Agent.application; case "system" : return Agent.system; case "operator" : return Agent.operator; - case "NodeRetirer" : return Agent.system; // TODO: Remove after 7.67 case "NodeFailer" : return Agent.NodeFailer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); @@ -395,7 +394,6 @@ public class NodeSerializer { case application : return "application"; case system : return "system"; case operator : return "operator"; - case NodeRetirer : return "system"; // TODO: Remove after 7.67 case NodeFailer : return "NodeFailer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 94db765c08a..77ca4b01cf2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.NodeFlavors; @@ -64,29 +63,20 @@ public class CapacityPolicies { if (requestedResources.isEmpty()) return defaultNodeResources(cluster.type()); - // Flavor is specified and is allocateByLegacyName: Handle legacy flavor specs - if (zone.system() == SystemName.cd) - return flavors.exists(requestedResources.get().legacyName().get()) ? requestedResources.get() - : defaultNodeResources(cluster.type()); - else { - switch (zone.environment()) { - case dev: case test: case staging: return defaultNodeResources(cluster.type()); - default: - flavors.getFlavorOrThrow(requestedResources.get().legacyName().get()); // verify existence - // Return this spec containing the legacy flavor name, not the flavor's capacity object - // which describes the flavors capacity, as the point of legacy allocation is to match - // by name, not by resources - return requestedResources.get(); - } + switch (zone.environment()) { + case dev: case test: case staging: return defaultNodeResources(cluster.type()); + default: + flavors.getFlavorOrThrow(requestedResources.get().legacyName().get()); // verify existence + // Return this spec containing the legacy flavor name, not the flavor's capacity object + // which describes the flavors capacity, as the point of legacy allocation is to match + // by name, not by resources + return requestedResources.get(); } } private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) - return new NodeResources(0.5, 3, 50); - - if (zone.system().isCd() && zone.environment().isTest()) - new NodeResources(4, 4, 50); + return nodeResourcesForAdminCluster(); return new NodeResources(1.5, 8, 50); } @@ -114,4 +104,9 @@ public class CapacityPolicies { return nodeCount; } + private NodeResources nodeResourcesForAdminCluster() { + double memoryInGb = (zone.system().isCd() ? 2 : 3); + return new NodeResources(0.5, memoryInGb, 50); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 4f0081b6a7f..ea30fba9798 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceException; +import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.IP; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java new file mode 100644 index 00000000000..7b0eb38b628 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java @@ -0,0 +1,161 @@ +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.CapacityChecker; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +public class HostCapacityResponse extends HttpResponse { + private final StringBuilder text; + private final Slime slime; + private final CapacityChecker capacityChecker; + private final boolean json; + + public HostCapacityResponse(NodeRepository nodeRepository, HttpRequest request) { + super(200); + capacityChecker = new CapacityChecker(nodeRepository); + + json = request.getBooleanProperty("json"); + String hostsJson = request.getProperty("hosts"); + + text = new StringBuilder(); + slime = new Slime(); + Cursor root = slime.setObject(); + + if (hostsJson != null) { + List<Node> hosts = parseHostList(hostsJson); + hostRemovalResponse(root, hosts); + } else { + zoneFailureReponse(root); + } + } + + private List<Node> parseHostList(String hosts) { + List<String> hostNames = Arrays.asList(hosts.split(",")); + try { + return capacityChecker.nodesFromHostnames(hostNames); + } catch (IllegalArgumentException e) { + throw new NotFoundException(e.getMessage()); + } + } + + private void hostRemovalResponse(Cursor root, List<Node> hosts) { + var failure = capacityChecker.findHostRemovalFailure(hosts); + if (failure.isPresent() && failure.get().failureReason.allocationFailures.size() == 0) { + root.setBool("removalPossible", false); + error(root, "Removing all hosts is trivially impossible."); + } else { + if (json) hostLossPossibleToSlime(root, failure, hosts); + else hostLossPossibleToText(failure, hosts); + } + } + + private void zoneFailureReponse(Cursor root) { + var failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); + if (failurePath.isPresent()) { + if (json) zoneFailurePathToSlime(root, failurePath.get()); + else zoneFailurePathToText(failurePath.get()); + } else { + error(root, "Node repository contained no hosts."); + } + } + + private void error(Cursor root, String errorMessage) { + if (json) root.setString("error", errorMessage); + else text.append(errorMessage); + } + + private void hostLossPossibleToText(Optional<CapacityChecker.HostFailurePath> failure, List<Node> hostsToRemove) { + text.append(String.format("Attempting to remove %d hosts: ", hostsToRemove.size())); + CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; + if (failure.isEmpty()) { + text.append("OK\n\n"); + text.append(history); + if (history.oldParents().size() != hostsToRemove.size()) { + long emptyHostCount = hostsToRemove.size() - history.oldParents().size(); + text.append(String.format("\nTrivially removed %d empty host%s.", emptyHostCount, emptyHostCount > 1 ? "s" : "")); + } + } else { + text.append("FAILURE\n\n"); + text.append(history).append("\n"); + text.append(failure.get().failureReason).append("\n\n"); + } + } + + private void zoneFailurePathToText(CapacityChecker.HostFailurePath failurePath) { + text.append(String.format("Found %d hosts. Failure upon trying to remove %d hosts:\n\n", + capacityChecker.getHosts().size(), + failurePath.hostsCausingFailure.size())); + text.append(capacityChecker.allocationHistory).append("\n"); + text.append(failurePath.failureReason); + } + + private void hostLossPossibleToSlime(Cursor root, Optional<CapacityChecker.HostFailurePath> failure, List<Node> hostsToRemove) { + var hosts = root.setArray("hostsToRemove"); + hostsToRemove.forEach(h -> hosts.addString(h.hostname())); + CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; + root.setBool("removalPossible", failure.isEmpty()); + var arr = root.setArray("history"); + for (var entry : history.historyEntries) { + var object = arr.addObject(); + object.setString("tenant", entry.tenant.hostname()); + if (entry.newParent != null) { + object.setString("newParent", entry.newParent.hostname()); + } + object.setLong("eligibleParents", entry.eligibleParents); + } + } + + private void zoneFailurePathToSlime(Cursor object, CapacityChecker.HostFailurePath failurePath) { + object.setLong("totalHosts", capacityChecker.getHosts().size()); + object.setLong("couldLoseHosts", failurePath.hostsCausingFailure.size()); + failurePath.failureReason.host.ifPresent(host -> + object.setString("failedTenantParent", host.hostname()) + ); + failurePath.failureReason.tenant.ifPresent(tenant -> { + object.setString("failedTenant", tenant.hostname()); + object.setString("failedTenantResources", tenant.flavor().resources().toString()); + tenant.allocation().ifPresent(allocation -> + object.setString("failedTenantAllocation", allocation.toString()) + ); + var explanation = object.setObject("hostCandidateRejectionReasons"); + allocationFailureReasonListToSlime(explanation.setObject("singularReasonFailures"), + failurePath.failureReason.allocationFailures.singularReasonFailures()); + allocationFailureReasonListToSlime(explanation.setObject("totalFailures"), + failurePath.failureReason.allocationFailures); + }); + var details = object.setObject("details"); + hostLossPossibleToSlime(details, Optional.of(failurePath), failurePath.hostsCausingFailure); + } + + private void allocationFailureReasonListToSlime(Cursor root, CapacityChecker.AllocationFailureReasonList allocationFailureReasonList) { + root.setLong("insufficientVcpu", allocationFailureReasonList.insufficientVcpu()); + root.setLong("insufficientMemoryGb", allocationFailureReasonList.insufficientMemoryGb()); + root.setLong("insufficientDiskGb", allocationFailureReasonList.insufficientDiskGb()); + root.setLong("incompatibleDiskSpeed", allocationFailureReasonList.incompatibleDiskSpeed()); + root.setLong("insufficientAvailableIps", allocationFailureReasonList.insufficientAvailableIps()); + root.setLong("violatesParentHostPolicy", allocationFailureReasonList.violatesParentHostPolicy()); + } + + @Override + public void render(OutputStream stream) throws IOException { + if (json) new JsonFormat(true).encode(stream, slime); + else stream.write(text.toString().getBytes()); + } + + @Override + public String getContentType() { + return json ? "application/json" : "text/plain"; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index bfbf7775031..9f8f4a804d1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -77,10 +77,6 @@ public class LoadBalancersResponse extends HttpResponse { realObject.setString("ipAddress", real.ipAddress()); realObject.setLong("port", real.port()); }); - - // TODO(mpolden): The following fields preserves API compatibility. These can be removed once clients stop expecting them - lbObject.setArray("rotations"); - lbObject.setBool("inactive", lb.state() == LoadBalancer.State.inactive); }); new JsonFormat(true).encode(stream, slime); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 22318f1ddb4..b2f0998189d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -102,6 +102,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); + if (path.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); throw new NotFoundException("Nothing at path '" + path + "'"); } |