1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.autoscale;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.applications.Cluster;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster
* based on observed behavior.
*
* @author bratseth
*/
public class Autoscaler {
/** What cost difference factor is worth a reallocation? */
private static final double costDifferenceWorthReallocation = 0.1;
/** What difference factor for a resource is worth a reallocation? */
private static final double resourceDifferenceWorthReallocation = 0.1;
private final NodeMetricsDb metricsDb;
private final NodeRepository nodeRepository;
private final AllocationOptimizer allocationOptimizer;
public Autoscaler(NodeMetricsDb metricsDb, NodeRepository nodeRepository) {
this.metricsDb = metricsDb;
this.nodeRepository = nodeRepository;
this.allocationOptimizer = new AllocationOptimizer(nodeRepository);
}
/**
* Suggest a scaling of a cluster. This returns a better allocation (if found)
* without taking min and max limits into account.
*
* @param clusterNodes the list of all the active nodes in a cluster
* @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
*/
public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) {
return autoscale(clusterNodes, Limits.empty(), cluster.exclusive())
.map(AllocatableClusterResources::toAdvertisedClusterResources);
}
/**
* Autoscale a cluster by load. This returns a better allocation (if found) inside the min and max limits.
*
* @param clusterNodes the list of all the active nodes in a cluster
* @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
*/
public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) {
if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut
return autoscale(clusterNodes, Limits.of(cluster), cluster.exclusive())
.map(AllocatableClusterResources::toAdvertisedClusterResources);
}
private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits, boolean exclusive) {
if (unstable(clusterNodes)) return Optional.empty();
ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type();
AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository);
Optional<Double> cpuLoad = averageLoad(Resource.cpu, clusterNodes, clusterType);
Optional<Double> memoryLoad = averageLoad(Resource.memory, clusterNodes, clusterType);
Optional<Double> diskLoad = averageLoad(Resource.disk, clusterNodes, clusterType);
if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty();
var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation);
Optional<AllocatableClusterResources> bestAllocation =
allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive);
if (bestAllocation.isEmpty()) return Optional.empty();
if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty();
return bestAllocation;
}
/** Returns true if both total real resources and total cost are similar */
private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) {
return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) &&
similar(a.realResources().vcpu() * a.nodes(),
b.realResources().vcpu() * b.nodes(), resourceDifferenceWorthReallocation) &&
similar(a.realResources().memoryGb() * a.nodes(),
b.realResources().memoryGb() * b.nodes(), resourceDifferenceWorthReallocation) &&
similar(a.realResources().diskGb() * a.nodes(),
b.realResources().diskGb() * b.nodes(), resourceDifferenceWorthReallocation);
}
private boolean similar(double r1, double r2, double threshold) {
return Math.abs(r1 - r2) / (( r1 + r2) / 2) < threshold;
}
/**
* Returns the average load of this resource in the measurement window,
* or empty if we are not in a position to make decisions from these measurements at this time.
*/
private Optional<Double> averageLoad(Resource resource, List<Node> clusterNodes, ClusterSpec.Type clusterType) {
NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(clusterType)),
resource,
clusterNodes.stream().map(Node::hostname).collect(Collectors.toList()));
// Require a total number of measurements scaling with the number of nodes,
// but don't require that we have at least that many from every node
if (window.measurementCount()/clusterNodes.size() < minimumMeasurementsPerNode(clusterType)) return Optional.empty();
if (window.hostnames() != clusterNodes.size()) return Optional.empty();
return Optional.of(window.average());
}
/** The duration of the window we need to consider to make a scaling decision. See also minimumMeasurementsPerNode */
static Duration scalingWindow(ClusterSpec.Type clusterType) {
if (clusterType.isContent()) return Duration.ofHours(12);
return Duration.ofHours(1);
}
/** Measurements are currently taken once a minute. See also scalingWindow */
static int minimumMeasurementsPerNode(ClusterSpec.Type clusterType) {
if (clusterType.isContent()) return 60;
return 20;
}
public static boolean unstable(List<Node> nodes) {
return nodes.stream().anyMatch(node -> node.status().wantToRetire() ||
node.allocation().get().membership().retired() ||
node.allocation().get().isRemovable());
}
}
|
