diff options
Diffstat (limited to 'node-repository/src/main/java/com/yahoo/vespa')
10 files changed, 258 insertions, 131 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 1ca81df824b..796bc2eeb92 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -208,6 +208,16 @@ public class Cluster { return minimum(ClusterModel.minScalingDuration(clusterSpec), totalDuration.dividedBy(completedEventCount)); } + /** The predicted time this cluster will stay in each resource configuration (including the scaling duration). */ + public Duration allocationDuration(ClusterSpec clusterSpec) { + if (scalingEvents.size() < 2) return Duration.ofHours(12); // Default + + long totalDurationMs = 0; + for (int i = 1; i < scalingEvents().size(); i++) + totalDurationMs += scalingEvents().get(i).at().toEpochMilli() - scalingEvents().get(i - 1).at().toEpochMilli(); + return Duration.ofMillis(totalDurationMs / (scalingEvents.size() - 1)); + } + private static Duration minimum(Duration smallestAllowed, Duration duration) { if (duration.minus(smallestAllowed).isNegative()) return smallestAllowed; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index c19d76efb35..8069c9c089b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -10,13 +10,14 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import java.time.Duration; import java.util.List; import java.util.Optional; /** * @author bratseth */ -public class AllocatableClusterResources { +public class AllocatableResources { /** The node count in the cluster */ private final int nodes; @@ -32,9 +33,9 @@ public class AllocatableClusterResources { private final double fulfilment; /** Fake allocatable resources from requested capacity */ - public AllocatableClusterResources(ClusterResources requested, - ClusterSpec clusterSpec, - NodeRepository nodeRepository) { + public AllocatableResources(ClusterResources requested, + ClusterSpec clusterSpec, + NodeRepository nodeRepository) { this.nodes = requested.nodes(); this.groups = requested.groups(); this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec), false); @@ -43,7 +44,7 @@ public class AllocatableClusterResources { this.fulfilment = 1; } - public AllocatableClusterResources(NodeList nodes, NodeRepository nodeRepository) { + public AllocatableResources(NodeList nodes, NodeRepository nodeRepository) { this.nodes = nodes.size(); this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); this.realResources = averageRealResourcesOf(nodes.asList(), nodeRepository); // Average since we average metrics over nodes @@ -52,10 +53,10 @@ public class AllocatableClusterResources { this.fulfilment = 1; } - public AllocatableClusterResources(ClusterResources realResources, - NodeResources advertisedResources, - ClusterResources idealResources, - ClusterSpec clusterSpec) { + public AllocatableResources(ClusterResources realResources, + NodeResources advertisedResources, + ClusterResources idealResources, + ClusterSpec clusterSpec) { this.nodes = realResources.nodes(); this.groups = realResources.groups(); this.realResources = realResources.nodeResources(); @@ -64,12 +65,12 @@ public class AllocatableClusterResources { this.fulfilment = fulfilment(realResources, idealResources); } - private AllocatableClusterResources(int nodes, - int groups, - NodeResources realResources, - NodeResources advertisedResources, - ClusterSpec clusterSpec, - double fulfilment) { + private AllocatableResources(int nodes, + int groups, + NodeResources realResources, + NodeResources advertisedResources, + ClusterSpec clusterSpec, + double fulfilment) { this.nodes = nodes; this.groups = groups; this.realResources = realResources; @@ -79,16 +80,16 @@ public class AllocatableClusterResources { } /** Returns this with the redundant node or group removed from counts. */ - public AllocatableClusterResources withoutRedundancy() { + public AllocatableResources withoutRedundancy() { int groupSize = nodes / groups; int nodesAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? nodes - 1 : nodes - groupSize) : nodes; int groupsAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? 1 : groups - 1) : groups; - return new AllocatableClusterResources(nodesAdjustedForRedundancy, - groupsAdjustedForRedundancy, - realResources, - advertisedResources, - clusterSpec, - fulfilment); + return new AllocatableResources(nodesAdjustedForRedundancy, + groupsAdjustedForRedundancy, + realResources, + advertisedResources, + clusterSpec, + fulfilment); } /** @@ -112,6 +113,7 @@ public class AllocatableClusterResources { public ClusterSpec clusterSpec() { return clusterSpec; } + /** Returns the standard cost of these resources, in dollars per hour */ public double cost() { return nodes * advertisedResources.cost(); } /** @@ -128,11 +130,22 @@ public class AllocatableClusterResources { return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3; } - public boolean preferableTo(AllocatableClusterResources other) { - if (this.fulfilment < 1 || other.fulfilment < 1) // always fulfil as much as possible - return this.fulfilment > other.fulfilment; + public boolean preferableTo(AllocatableResources other, ClusterModel model) { + if (other.fulfilment() < 1 || this.fulfilment() < 1) // always fulfil as much as possible + return this.fulfilment() > other.fulfilment(); - return this.cost() < other.cost(); // otherwise, prefer lower cost + return this.cost() * toHours(model.allocationDuration()) + this.costChangingFrom(model) + < + other.cost() * toHours(model.allocationDuration()) + other.costChangingFrom(model); + } + + private double toHours(Duration duration) { + return duration.toMillis() / 3600000.0; + } + + /** The estimated cost of changing from the given current resources to this. */ + public double costChangingFrom(ClusterModel model) { + return new ResourceChange(model, this).cost(); } @Override @@ -154,12 +167,13 @@ public class AllocatableClusterResources { .withBandwidthGbps(sum.bandwidthGbps() / nodes.size()); } - public static Optional<AllocatableClusterResources> from(ClusterResources wantedResources, - ApplicationId applicationId, - ClusterSpec clusterSpec, - Limits applicationLimits, - List<NodeResources> availableRealHostResources, - NodeRepository nodeRepository) { + public static Optional<AllocatableResources> from(ClusterResources wantedResources, + ApplicationId applicationId, + ClusterSpec clusterSpec, + Limits applicationLimits, + List<NodeResources> availableRealHostResources, + ClusterModel model, + NodeRepository nodeRepository) { var systemLimits = nodeRepository.nodeResourceLimits(); boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); if (! exclusive) { @@ -193,8 +207,8 @@ public class AllocatableClusterResources { } else { // Return the cheapest flavor satisfying the requested resources, if any NodeResources cappedWantedResources = applicationLimits.cap(wantedResources.nodeResources()); - Optional<AllocatableClusterResources> best = Optional.empty(); - Optional<AllocatableClusterResources> bestDisregardingDiskLimit = Optional.empty(); + Optional<AllocatableResources> best = Optional.empty(); + Optional<AllocatableResources> bestDisregardingDiskLimit = Optional.empty(); for (Flavor flavor : nodeRepository.flavors().getFlavors()) { // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); @@ -216,18 +230,18 @@ public class AllocatableClusterResources { if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec)) continue; - var candidate = new AllocatableClusterResources(wantedResources.with(realResources), - advertisedResources, - wantedResources, - clusterSpec); + var candidate = new AllocatableResources(wantedResources.with(realResources), + advertisedResources, + wantedResources, + clusterSpec); if ( ! systemLimits.isWithinAdvertisedDiskLimits(advertisedResources, clusterSpec)) { // TODO: Remove when disk limit is enforced - if (bestDisregardingDiskLimit.isEmpty() || candidate.preferableTo(bestDisregardingDiskLimit.get())) { + if (bestDisregardingDiskLimit.isEmpty() || candidate.preferableTo(bestDisregardingDiskLimit.get(), model)) { bestDisregardingDiskLimit = Optional.of(candidate); } continue; } - if (best.isEmpty() || candidate.preferableTo(best.get())) { + if (best.isEmpty() || candidate.preferableTo(best.get(), model)) { best = Optional.of(candidate); } } @@ -237,13 +251,13 @@ public class AllocatableClusterResources { } } - private static AllocatableClusterResources calculateAllocatableResources(ClusterResources wantedResources, - NodeRepository nodeRepository, - ApplicationId applicationId, - ClusterSpec clusterSpec, - Limits applicationLimits, - boolean exclusive, - boolean bestCase) { + private static AllocatableResources calculateAllocatableResources(ClusterResources wantedResources, + NodeRepository nodeRepository, + ApplicationId applicationId, + ClusterSpec clusterSpec, + Limits applicationLimits, + boolean exclusive, + boolean bestCase) { var systemLimits = nodeRepository.nodeResourceLimits(); var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive, bestCase); advertisedResources = systemLimits.enlargeToLegal(advertisedResources, applicationId, clusterSpec, exclusive, true); // Ask for something legal @@ -255,10 +269,10 @@ public class AllocatableClusterResources { advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); } - return new AllocatableClusterResources(wantedResources.with(realResources), - advertisedResources, - wantedResources, - clusterSpec); + return new AllocatableResources(wantedResources.with(realResources), + advertisedResources, + wantedResources, + clusterSpec); } /** Returns true if the given resources could be allocated on any of the given host flavors */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 42bb16005ee..f650d8ec269 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.Optional; @@ -35,21 +34,20 @@ public class AllocationOptimizer { * @return the best allocation, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits */ - public Optional<AllocatableClusterResources> findBestAllocation(Load loadAdjustment, - AllocatableClusterResources current, - ClusterModel clusterModel, - Limits limits) { + public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment, + ClusterModel model, + Limits limits) { if (limits.isEmpty()) limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), IntRange.empty()); else - limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec(), nodeRepository, clusterModel.application().id()); - Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); + limits = atLeast(minimumNodes, limits).fullySpecified(model.current().clusterSpec(), nodeRepository, model.application().id()); + Optional<AllocatableResources> bestAllocation = Optional.empty(); var availableRealHostResources = nodeRepository.zone().cloud().dynamicProvisioning() ? nodeRepository.flavors().getFlavors().stream().map(flavor -> flavor.resources()).toList() : nodeRepository.nodes().list().hosts().stream().map(host -> host.flavor().resources()) - .map(hostResources -> maxResourcesOf(hostResources, clusterModel)) + .map(hostResources -> maxResourcesOf(hostResources, model)) .toList(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { @@ -58,15 +56,16 @@ public class AllocationOptimizer { var resources = new ClusterResources(nodes, groups, nodeResourcesWith(nodes, groups, - limits, loadAdjustment, current, clusterModel)); - var allocatableResources = AllocatableClusterResources.from(resources, - clusterModel.application().id(), - current.clusterSpec(), - limits, - availableRealHostResources, - nodeRepository); + limits, loadAdjustment, model)); + var allocatableResources = AllocatableResources.from(resources, + model.application().id(), + model.current().clusterSpec(), + limits, + availableRealHostResources, + model, + nodeRepository); if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get(), model)) bestAllocation = allocatableResources; } } @@ -74,8 +73,8 @@ public class AllocationOptimizer { } /** Returns the max resources of a host one node may allocate. */ - private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel clusterModel) { - if (nodeRepository.exclusiveAllocation(clusterModel.clusterSpec())) return hostResources; + private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel model) { + if (nodeRepository.exclusiveAllocation(model.clusterSpec())) return hostResources; // static, shared hosts: Allocate at most half of the host cpu to simplify management return hostResources.withVcpu(hostResources.vcpu() / 2); } @@ -88,9 +87,8 @@ public class AllocationOptimizer { int groups, Limits limits, Load loadAdjustment, - AllocatableClusterResources current, - ClusterModel clusterModel) { - var loadWithTarget = clusterModel.loadAdjustmentWith(nodes, groups, loadAdjustment); + ClusterModel model) { + var loadWithTarget = model.loadAdjustmentWith(nodes, groups, loadAdjustment); // Leave some headroom above the ideal allocation to avoid immediately needing to scale back up if (loadAdjustment.cpu() < 1 && (1.0 - loadWithTarget.cpu()) < headroomRequiredToScaleDown) @@ -100,11 +98,11 @@ public class AllocationOptimizer { if (loadAdjustment.disk() < 1 && (1.0 - loadWithTarget.disk()) < headroomRequiredToScaleDown) loadAdjustment = loadAdjustment.withDisk(Math.min(1.0, loadAdjustment.disk() * (1.0 + headroomRequiredToScaleDown))); - loadWithTarget = clusterModel.loadAdjustmentWith(nodes, groups, loadAdjustment); + loadWithTarget = model.loadAdjustmentWith(nodes, groups, loadAdjustment); - var scaled = loadWithTarget.scaled(current.realResources().nodeResources()); + var scaled = loadWithTarget.scaled(model.current().realResources().nodeResources()); var nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() - ? current.advertisedResources().nodeResources() + ? model.current().advertisedResources().nodeResources() : limits.min().nodeResources(); // min=max for non-scaled return nonScaled.withVcpu(scaled.vcpu()).withMemoryGb(scaled.memoryGb()).withDiskGb(scaled.diskGb()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 32b59319a88..b5f86be68f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -54,40 +54,40 @@ public class Autoscaler { } private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { - ClusterModel clusterModel = new ClusterModel(nodeRepository, - application, - clusterNodes.not().retired().clusterSpec(), - cluster, - clusterNodes, - nodeRepository.metricsDb(), - nodeRepository.clock()); - if (clusterModel.isEmpty()) return Autoscaling.empty(); + var model = new ClusterModel(nodeRepository, + application, + clusterNodes.not().retired().clusterSpec(), + cluster, + clusterNodes, + new AllocatableResources(clusterNodes.not().retired(), nodeRepository), + nodeRepository.metricsDb(), + nodeRepository.clock()); + if (model.isEmpty()) return Autoscaling.empty(); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) - return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", clusterModel); + return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", model); - if ( ! clusterModel.isStable(nodeRepository)) - return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", clusterModel); + if ( ! model.isStable(nodeRepository)) + return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", model); - var current = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository); - var loadAdjustment = clusterModel.loadAdjustment(); + var loadAdjustment = model.loadAdjustment(); // Ensure we only scale down if we'll have enough headroom to not scale up again given a small load increase - var target = allocationOptimizer.findBestAllocation(loadAdjustment, current, clusterModel, limits); + var target = allocationOptimizer.findBestAllocation(loadAdjustment, model, limits); if (target.isEmpty()) - return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", clusterModel); + return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", model); - if (! worthRescaling(current.realResources(), target.get().realResources())) { + if (! worthRescaling(model.current().realResources(), target.get().realResources())) { if (target.get().fulfilment() < 0.9999999) - return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", clusterModel); - else if ( ! clusterModel.safeToScaleDown() && clusterModel.idealLoad().any(v -> v < 1.0)) - return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", clusterModel); + return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", model); + else if ( ! model.safeToScaleDown() && model.idealLoad().any(v -> v < 1.0)) + return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", model); else - return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", clusterModel); + return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", model); } - return Autoscaling.scaleTo(target.get().advertisedResources(), clusterModel); + return Autoscaling.scaleTo(target.get().advertisedResources(), model); } /** Returns true if it is worthwhile to make the given resource change, false if it is too insignificant */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 0c86108b36c..fad280d6c29 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -120,25 +120,25 @@ public class Autoscaling { } /** Creates an autoscaling conclusion which does not change the current allocation for a specified reason. */ - public static Autoscaling dontScale(Status status, String description, ClusterModel clusterModel) { + public static Autoscaling dontScale(Status status, String description, ClusterModel model) { return new Autoscaling(status, description, Optional.empty(), - clusterModel.at(), - clusterModel.peakLoad(), - clusterModel.idealLoad(), - clusterModel.metrics()); + model.at(), + model.peakLoad(), + model.idealLoad(), + model.metrics()); } /** Creates an autoscaling conclusion to scale. */ - public static Autoscaling scaleTo(ClusterResources target, ClusterModel clusterModel) { + public static Autoscaling scaleTo(ClusterResources target, ClusterModel model) { return new Autoscaling(Status.rescaling, "Rescaling initiated due to load changes", Optional.of(target), - clusterModel.at(), - clusterModel.peakLoad(), - clusterModel.idealLoad(), - clusterModel.metrics()); + model.at(), + model.peakLoad(), + model.idealLoad(), + model.metrics()); } public enum Status { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 0d64d4fbb10..7c2f3a563fb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -50,6 +50,7 @@ public class ClusterModel { private final Application application; private final ClusterSpec clusterSpec; private final Cluster cluster; + private final AllocatableResources current; private final CpuModel cpu = new CpuModel(); private final MemoryModel memory = new MemoryModel(); @@ -63,6 +64,7 @@ public class ClusterModel { private final Clock clock; private final Duration scalingDuration; + private final Duration allocationDuration; private final ClusterTimeseries clusterTimeseries; private final ClusterNodesTimeseries nodeTimeseries; private final Instant at; @@ -77,6 +79,7 @@ public class ClusterModel { ClusterSpec clusterSpec, Cluster cluster, NodeList clusterNodes, + AllocatableResources current, MetricsDb metricsDb, Clock clock) { this.nodeRepository = nodeRepository; @@ -84,8 +87,10 @@ public class ClusterModel { this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = clusterNodes; + this.current = current; this.clock = clock; this.scalingDuration = cluster.scalingDuration(clusterSpec); + this.allocationDuration = cluster.allocationDuration(clusterSpec); this.clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id()); this.nodeTimeseries = new ClusterNodesTimeseries(scalingDuration(), cluster, nodes, metricsDb); this.at = clock.instant(); @@ -95,8 +100,10 @@ public class ClusterModel { Application application, ClusterSpec clusterSpec, Cluster cluster, + AllocatableResources current, Clock clock, Duration scalingDuration, + Duration allocationDuration, ClusterTimeseries clusterTimeseries, ClusterNodesTimeseries nodeTimeseries) { this.nodeRepository = nodeRepository; @@ -104,9 +111,11 @@ public class ClusterModel { this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = NodeList.of(); + this.current = current; this.clock = clock; this.scalingDuration = scalingDuration; + this.allocationDuration = allocationDuration; this.clusterTimeseries = clusterTimeseries; this.nodeTimeseries = nodeTimeseries; this.at = clock.instant(); @@ -114,6 +123,7 @@ public class ClusterModel { public Application application() { return application; } public ClusterSpec clusterSpec() { return clusterSpec; } + public AllocatableResources current() { return current; } private ClusterNodesTimeseries nodeTimeseries() { return nodeTimeseries; } private ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } @@ -127,6 +137,23 @@ public class ClusterModel { /** Returns the predicted duration of a rescaling of this cluster */ public Duration scalingDuration() { return scalingDuration; } + /** + * Returns the predicted duration of a resource change in this cluster, + * until we, or the application , will change it again. + */ + public Duration allocationDuration() { return allocationDuration; } + + /** Returns the predicted duration of data redistribution in this cluster. */ + public Duration redistributionDuration() { + if (! clusterSpec.type().isContent()) return Duration.ofMinutes(0); + return scalingDuration(); // TODO: Estimate separately + } + + /** Returns the predicted duration of replacing all the nodes in this cluster. */ + public Duration nodeReplacementDuration() { + return Duration.ofMinutes(5); // TODO: Estimate? + } + /** Returns the average of the peak load measurement in each dimension, from each node. */ public Load peakLoad() { return nodeTimeseries().peakLoad(); @@ -137,6 +164,10 @@ public class ClusterModel { return loadWith(nodeCount(), groupCount()); } + public boolean isExclusive() { + return nodeRepository.exclusiveAllocation(clusterSpec); + } + /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ public Load loadAdjustment() { if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java new file mode 100644 index 00000000000..c0cbb40d992 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; + +import java.time.Duration; + +/** + * A resource change. + * + * @author bratseth + */ +public class ResourceChange { + + private final AllocatableResources from, to; + private final ClusterModel model; + + public ResourceChange(ClusterModel model, AllocatableResources to) { + this.from = model.current(); + this.to = to; + this.model = model; + } + + /** Returns the estimated total cost of this resource change (coming in addition to the "to" resource cost). */ + public double cost() { + if (requiresRedistribution()) return toHours(model.redistributionDuration()) * from.cost(); + if (requiresNodeReplacement()) return toHours(model.nodeReplacementDuration()) * from.cost(); + return 0; + } + + private boolean requiresRedistribution() { + if ( ! model.clusterSpec().type().isContent()) return false; + if (from.nodes() != to.nodes()) return true; + if (from.groups() != to.groups()) return true; + if (requiresNodeReplacement()) return true; + return false; + } + + /** Returns true if the *existing* nodes of this needs to be replaced in this change. */ + private boolean requiresNodeReplacement() { + var fromNodes = from.advertisedResources().nodeResources(); + var toNodes = to.advertisedResources().nodeResources(); + + if (model.isExclusive()) { + return ! fromNodes.equals(toNodes); + } + else { + if ( ! fromNodes.justNonNumbers().equalsWhereSpecified(toNodes.justNonNumbers())) return true; + if ( ! canInPlaceResize()) return true; + return false; + } + } + + private double toHours(Duration duration) { + return duration.toMillis() / 3600000.0; + } + + private boolean canInPlaceResize() { + return canInPlaceResize(from.nodes(), from.advertisedResources().nodeResources(), + to.nodes(), to.advertisedResources().nodeResources(), + model.clusterSpec().type(), model.isExclusive(), from.groups() != to.groups()); + } + + public static boolean canInPlaceResize(int fromCount, NodeResources fromResources, + int toCount, NodeResources toResources, + ClusterSpec.Type type, boolean exclusive, boolean hasTopologyChange) { + if (exclusive) return false; // exclusive resources must match the host + + // Never allow in-place resize when also changing topology or decreasing cluster size + if (hasTopologyChange || toCount < fromCount) return false; + + // Do not allow increasing cluster size and decreasing node resources at the same time for content nodes + if (type.isContent() && toCount > fromCount && !toResources.satisfies(fromResources.justNumbers())) + return false; + + return true; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 92f86325cf7..6a01a2bcd18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; @@ -87,7 +87,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); cluster = updateCompletion(cluster, clusterNodes); - var current = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); + var current = new AllocatableResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); // Autoscale unless an autoscaling is already in progress Autoscaling autoscaling = null; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 3d0c1069584..a67a513550a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -23,7 +23,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableResources; import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; import com.yahoo.vespa.hosted.provision.autoscale.Limits; @@ -182,12 +182,12 @@ public class NodeRepositoryProvisioner implements Provisioner { .not().retired() .not().removable(); boolean firstDeployment = nodes.isEmpty(); - AllocatableClusterResources currentResources = + var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableClusterResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) - : new AllocatableClusterResources(nodes, nodeRepository); - var clusterModel = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, nodeRepository.metricsDb(), nodeRepository.clock()); - return within(Limits.of(requested), currentResources, firstDeployment, clusterModel); + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) + : new AllocatableResources(nodes, nodeRepository); + var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); + return within(Limits.of(requested), model, firstDeployment); } private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { @@ -197,21 +197,19 @@ public class NodeRepositoryProvisioner implements Provisioner { /** Make the minimal adjustments needed to the current resources to stay within the limits */ private ClusterResources within(Limits limits, - AllocatableClusterResources current, - boolean firstDeployment, - ClusterModel clusterModel) { + ClusterModel model, + boolean firstDeployment) { if (limits.min().equals(limits.max())) return limits.min(); // Don't change current deployments that are still legal - if (! firstDeployment && current.advertisedResources().isWithin(limits.min(), limits.max())) - return current.advertisedResources(); + if (! firstDeployment && model.current().advertisedResources().isWithin(limits.min(), limits.max())) + return model.current().advertisedResources(); // Otherwise, find an allocation that preserves the current resources as well as possible return allocationOptimizer.findBestAllocation(Load.one(), - current, - clusterModel, + model, limits) - .orElseThrow(() -> newNoAllocationPossible(current.clusterSpec(), limits)) + .orElseThrow(() -> newNoAllocationPossible(model.current().clusterSpec(), limits)) .advertisedResources(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index cea0608013d..77f37cadc0b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.autoscale.ResourceChange; import java.time.Duration; import java.util.Map; @@ -162,16 +163,11 @@ public interface NodeSpec { @Override public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, ClusterSpec.Type type, boolean hasTopologyChange, int currentClusterSize) { - if (exclusive) return false; // exclusive resources must match the host - // Never allow in-place resize when also changing topology or decreasing cluster size - if (hasTopologyChange || count < currentClusterSize) return false; + return ResourceChange.canInPlaceResize(currentClusterSize, currentNodeResources, count, requestedNodeResources, + type, exclusive, hasTopologyChange) + && + currentSpareHostResources.add(currentNodeResources.justNumbers()).satisfies(requestedNodeResources); - // Do not allow increasing cluster size and decreasing node resources at the same time for content nodes - if (type.isContent() && count > currentClusterSize && !requestedNodeResources.satisfies(currentNodeResources.justNumbers())) - return false; - - // Otherwise, allowed as long as the host can satisfy the new requested resources - return currentSpareHostResources.add(currentNodeResources.justNumbers()).satisfies(requestedNodeResources); } @Override |