diff options
Diffstat (limited to 'node-repository/src/main/java')
8 files changed, 68 insertions, 35 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 321c5632302..d5dd2138a4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -83,7 +83,7 @@ public final class Node { this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) - requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); + requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 750b22484c9..f1be485d368 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -43,10 +44,14 @@ public class Autoscaler { private static final double memoryUnitCost = 1.2; private static final double diskUnitCost = 0.045; + private final HostResourcesCalculator hostResourcesCalculator; private final NodeMetricsDb metricsDb; private final NodeRepository nodeRepository; - public Autoscaler(NodeMetricsDb metricsDb, NodeRepository nodeRepository) { + public Autoscaler(HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + NodeRepository nodeRepository) { + this.hostResourcesCalculator = hostResourcesCalculator; this.metricsDb = metricsDb; this.nodeRepository = nodeRepository; } @@ -56,7 +61,6 @@ public class Autoscaler { node.allocation().get().membership().retired() || node.allocation().get().isRemovable())) return Optional.empty(); // Don't autoscale clusters that are in flux - ClusterResources currentAllocation = new ClusterResources(clusterNodes); Optional<Double> totalCpuSpent = averageUseOf(Resource.cpu, applicationId, cluster, clusterNodes); Optional<Double> totalMemorySpent = averageUseOf(Resource.memory, applicationId, cluster, clusterNodes); @@ -74,24 +78,23 @@ public class Autoscaler { private Optional<ClusterResources> findBestAllocation(double totalCpu, double totalMemory, double totalDisk, ClusterResources currentAllocation) { - Optional<ClusterResources> bestAllocation = Optional.empty(); + Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty(); for (ResourceIterator i = new ResourceIterator(totalCpu, totalMemory, totalDisk, currentAllocation); i.hasNext(); ) { ClusterResources allocation = i.next(); - Optional<NodeResources> allocatableResources = toAllocatableResources(allocation.resources()); + Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation); if (allocatableResources.isEmpty()) continue; - ClusterResources effectiveAllocation = allocation.with(allocatableResources.get()); - if (bestAllocation.isEmpty() || effectiveAllocation.cost() < bestAllocation.get().cost()) - bestAllocation = Optional.of(effectiveAllocation); + if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) + bestAllocation = allocatableResources; } - return bestAllocation; + return bestAllocation.map(a -> a.clusterResources()); } private boolean isSimilar(ClusterResources a1, ClusterResources a2) { if (a1.nodes() != a2.nodes()) return false; // A full node is always a significant difference - return isSimilar(a1.resources().vcpu(), a2.resources().vcpu()) && - isSimilar(a1.resources().memoryGb(), a2.resources().memoryGb()) && - isSimilar(a1.resources().diskGb(), a2.resources().diskGb()); + return isSimilar(a1.nodeResources().vcpu(), a2.nodeResources().vcpu()) && + isSimilar(a1.nodeResources().memoryGb(), a2.nodeResources().memoryGb()) && + isSimilar(a1.nodeResources().diskGb(), a2.nodeResources().diskGb()); } private boolean isSimilar(double r1, double r2) { @@ -102,11 +105,13 @@ public class Autoscaler { * Returns the smallest allocatable node resources larger than the given node resources, * or empty if none available. */ - private Optional<NodeResources> toAllocatableResources(NodeResources nodeResources) { + private Optional<ClusterResourcesWithCost> toAllocatableResources(ClusterResources resources) { if (allowsHostSharing(nodeRepository.zone().cloud())) { // Return the requested resources, or empty if they cannot fit on existing hosts for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) - if (flavor.resources().satisfies(nodeResources)) return Optional.of(nodeResources); + if (flavor.resources().satisfies(resources.nodeResources())) + return Optional.of(new ClusterResourcesWithCost(resources, + costOf(resources.nodeResources()) * resources.nodes())); return Optional.empty(); } else { @@ -114,14 +119,17 @@ public class Autoscaler { double bestCost = Double.MAX_VALUE; Optional<Flavor> bestFlavor = Optional.empty(); for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - // TODO: Use effective not advertised flavor resources - if ( ! flavor.resources().satisfies(nodeResources)) continue; + if ( ! flavor.resources().satisfies(resources.nodeResources())) continue; if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) { bestFlavor = Optional.of(flavor); - bestCost = costOf(flavor.resources()); + bestCost = costOf(flavor); } } - return bestFlavor.map(flavor -> flavor.resources()); + if (bestFlavor.isEmpty()) + return Optional.empty(); + else + return Optional.of(new ClusterResourcesWithCost(resources.with(bestFlavor.get().resources()), + bestCost * resources.nodes())); } } @@ -154,6 +162,11 @@ public class Autoscaler { return true; } + private double costOf(Flavor flavor) { + NodeResources chargedResources = hostResourcesCalculator.availableCapacityOf(flavor.name(), flavor.resources()); + return costOf(chargedResources); + } + static double costOf(NodeResources resources) { return resources.vcpu() * cpuUnitCost + resources.memoryGb() * memoryUnitCost + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java index 3fdf3c87601..4f66c8bb8f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java @@ -17,7 +17,7 @@ public class ClusterResources { private final int groups; /** The resources of each node in the cluster */ - private final NodeResources resources; + private final NodeResources nodeResources; public ClusterResources(List<Node> nodes) { this(nodes.size(), @@ -25,25 +25,21 @@ public class ClusterResources { nodes.get(0).flavor().resources()); } - public ClusterResources(int nodes, int groups, NodeResources resources) { + public ClusterResources(int nodes, int groups, NodeResources nodeResources) { this.nodes = nodes; this.groups = groups; - this.resources = resources; + this.nodeResources = nodeResources; } /** Returns the total number of allocated nodes (over all groups) */ public int nodes() { return nodes; } public int groups() { return groups; } - public NodeResources resources() { return resources; } + public NodeResources nodeResources() { return nodeResources; } public ClusterResources with(NodeResources resources) { return new ClusterResources(nodes, groups, resources); } - public double cost() { - return Autoscaler.costOf(resources) * nodes; - } - @Override public boolean equals(Object o) { if (o == this) return true; @@ -52,18 +48,18 @@ public class ClusterResources { ClusterResources other = (ClusterResources)o; if (other.nodes != this.nodes) return false; if (other.groups != this.groups) return false; - if (other.resources != this.resources) return false; + if (other.nodeResources != this.nodeResources) return false; return true; } @Override public int hashCode() { - return Objects.hash(nodes, groups, resources); + return Objects.hash(nodes, groups, nodeResources); } @Override public String toString() { - return "cluster resources: " + nodes + " * " + resources + (groups > 1 ? " in " + groups + " groups" : ""); + return "cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : ""); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java new file mode 100644 index 00000000000..c581d2304fb --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java @@ -0,0 +1,21 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +/** + * @author bratseth + */ +public class ClusterResourcesWithCost { + + private final ClusterResources resources; + private final double cost; + + public ClusterResourcesWithCost(ClusterResources resources, double cost) { + this.resources = resources; + this.cost = cost; + } + + public ClusterResources clusterResources() { return resources;} + + public double cost() { return cost; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index 2d3d7c83d4f..2dd1ae4b5a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -30,7 +30,7 @@ public class ResourceIterator { // ceil: If the division does not produce a whole number we assume some node is missing groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); - resourcesPrototype = currentAllocation.resources(); + resourcesPrototype = currentAllocation.nodeResources(); // What number of nodes is it effective to add or remove at the time from this cluster? // This is the group size, since we (for now) assume the group size is decided by someone wiser than us diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 2279d7a9eeb..30c25b3771e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -25,9 +26,12 @@ public class AutoscalingMaintainer extends Maintainer { private final Autoscaler autoscaler; - public AutoscalingMaintainer(NodeRepository nodeRepository, NodeMetricsDb metricsDb, Duration interval) { + public AutoscalingMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Duration interval) { super(nodeRepository, interval); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); } @Override @@ -42,7 +46,7 @@ public class AutoscalingMaintainer extends Maintainer { Optional<ClusterResources> target = autoscaler.autoscale(applicationId, clusterSpec, clusterNodes); target.ifPresent(t -> log.info("Autoscale: Application " + applicationId + " cluster " + clusterSpec + " from " + applicationNodes.size() + " * " + applicationNodes.get(0).flavor().resources() + - " to " + t.nodes() + " * " + t.resources())); + " to " + t.nodes() + " * " + t.nodeResources())); }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index a49049f8b04..9b06a3161e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -91,7 +91,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); - autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, defaults.autoscalingInterval); + autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.autoscalingInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 5753bbb3c5a..af6fa8edf64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -64,7 +64,6 @@ public class GroupPreparer { .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); - try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others |