diff options
author | Jon Bratseth <bratseth@gmail.com> | 2020-04-27 13:53:18 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2020-04-27 13:53:18 +0200 |
commit | 009f5ff4e1e6c9f5d7e27f639a953ea4b3db6647 (patch) | |
tree | 6e88ebd02512312951a6b4e491fbf68c31be4210 /node-repository | |
parent | 92e1b817ac7dc6d22d121cf0aa444a72a8f5d71c (diff) |
Extract toAllocatableResources
Diffstat (limited to 'node-repository')
2 files changed, 64 insertions, 57 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 2414bd95b85..721a1af23b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -1,14 +1,20 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.List; +import java.util.Optional; /** * @author bratseth @@ -121,4 +127,56 @@ public class AllocatableClusterResources { (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : ""); } + /** + * Returns the smallest allocatable node resources larger than the given node resources, + * or empty if none available. + */ + public static Optional<AllocatableClusterResources> from(ClusterResources resources, + ClusterSpec.Type clusterType, + Optional<Cluster> limits, + NodeRepository nodeRepository, + HostResourcesCalculator resourcesCalculator) { + NodeResources nodeResources = resources.nodeResources(); + if (limits.isPresent()) + nodeResources = limits.get().capAtLimits(nodeResources); + nodeResources = new NodeResourceLimits(nodeRepository.zone()).enlargeToLegal(nodeResources, clusterType); + + if (allowsHostSharing(nodeRepository.zone().cloud())) { + // return the requested resources, or empty if they cannot fit on existing hosts + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { + if (flavor.resources().satisfies(nodeResources)) + return Optional.of(new AllocatableClusterResources(resources.with(nodeResources), + nodeResources, + resources.nodeResources(), + clusterType)); + } + return Optional.empty(); + } + else { + // return the cheapest flavor satisfying the target resources, if any + Optional<AllocatableClusterResources> best = Optional.empty(); + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { + if ( ! flavor.resources().satisfies(nodeResources)) continue; + + if (flavor.resources().storageType() == NodeResources.StorageType.remote) + flavor = flavor.with(FlavorOverrides.ofDisk(nodeResources.diskGb())); + var candidate = new AllocatableClusterResources(resources.with(flavor.resources()), + flavor, + resources.nodeResources(), + clusterType, + resourcesCalculator); + + if (best.isEmpty() || candidate.cost() <= best.get().cost()) + best = Optional.of(candidate); + } + return best; + } + } + + // TODO: Put this in zone config instead? + private static boolean allowsHostSharing(CloudName cloudName) { + if (cloudName.value().equals("aws")) return false; + return true; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 6dca9c9a796..4ef5d11d3f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -44,7 +45,6 @@ public class Autoscaler { private final HostResourcesCalculator resourcesCalculator; private final NodeMetricsDb metricsDb; private final NodeRepository nodeRepository; - private final NodeResourceLimits nodeResourceLimits; public Autoscaler(HostResourcesCalculator resourcesCalculator, NodeMetricsDb metricsDb, @@ -52,7 +52,6 @@ public class Autoscaler { this.resourcesCalculator = resourcesCalculator; this.metricsDb = metricsDb; this.nodeRepository = nodeRepository; - this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone()); } /** @@ -107,10 +106,11 @@ public class Autoscaler { Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster, respectLimits); i.hasNext(); ) { - Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(), - currentAllocation.clusterType(), - cluster, - respectLimits); + var allocatableResources = AllocatableClusterResources.from(i.next(), + currentAllocation.clusterType(), + respectLimits ? Optional.of(cluster) : Optional.empty(), + nodeRepository, + resourcesCalculator); if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; @@ -135,51 +135,6 @@ public class Autoscaler { } /** - * Returns the smallest allocatable node resources larger than the given node resources, - * or empty if none available. - */ - private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources, - ClusterSpec.Type clusterType, - Cluster cluster, - boolean respectLimits) { - NodeResources nodeResources = resources.nodeResources(); - if (respectLimits) - nodeResources = cluster.capAtLimits(nodeResources); - nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits - - if (allowsHostSharing(nodeRepository.zone().cloud())) { - // return the requested resources, or empty if they cannot fit on existing hosts - for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - if (flavor.resources().satisfies(nodeResources)) - return Optional.of(new AllocatableClusterResources(resources.with(nodeResources), - nodeResources, - resources.nodeResources(), - clusterType)); - } - return Optional.empty(); - } - else { - // return the cheapest flavor satisfying the target resources, if any - Optional<AllocatableClusterResources> best = Optional.empty(); - for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - if ( ! flavor.resources().satisfies(nodeResources)) continue; - - if (flavor.resources().storageType() == NodeResources.StorageType.remote) - flavor = flavor.with(FlavorOverrides.ofDisk(nodeResources.diskGb())); - var candidate = new AllocatableClusterResources(resources.with(flavor.resources()), - flavor, - resources.nodeResources(), - clusterType, - resourcesCalculator); - - if (best.isEmpty() || candidate.cost() <= best.get().cost()) - best = Optional.of(candidate); - } - return best; - } - } - - /** * Returns the average load of this resource in the measurement window, * or empty if we are not in a position to make decisions from these measurements at this time. */ @@ -200,12 +155,6 @@ public class Autoscaler { return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account } - // TODO: Put this in zone config instead? - private boolean allowsHostSharing(CloudName cloudName) { - if (cloudName.value().equals("aws")) return false; - return true; - } - public static boolean unstable(List<Node> nodes) { return nodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || |