1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.autoscale;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.applications.Cluster;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster
* based on observed behavior.
*
* @author bratseth
*/
public class Autoscaler {
private static final int minimumMeasurementsPerNode = 60; // 1 hour
/** What cost difference factor is worth a reallocation? */
private static final double costDifferenceWorthReallocation = 0.1;
/** What difference factor for a resource is worth a reallocation? */
private static final double resourceDifferenceWorthReallocation = 0.1;
private final NodeMetricsDb metricsDb;
private final NodeRepository nodeRepository;
private final AllocationOptimizer allocationOptimizer;
public Autoscaler(NodeMetricsDb metricsDb, NodeRepository nodeRepository) {
this.metricsDb = metricsDb;
this.nodeRepository = nodeRepository;
this.allocationOptimizer = new AllocationOptimizer(nodeRepository);
}
/**
* Suggest a scaling of a cluster. This returns a better allocation (if found)
* without taking min and max limits into account.
*
* @param clusterNodes the list of all the active nodes in a cluster
* @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
*/
public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) {
return autoscale(clusterNodes, Limits.empty(), cluster.exclusive())
.map(AllocatableClusterResources::toAdvertisedClusterResources);
}
/**
* Autoscale a cluster by load. This returns a better allocation (if found) inside the min and max limits.
*
* @param clusterNodes the list of all the active nodes in a cluster
* @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
*/
public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) {
if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut
return autoscale(clusterNodes, Limits.of(cluster), cluster.exclusive())
.map(AllocatableClusterResources::toAdvertisedClusterResources);
}
private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits, boolean exclusive) {
if (unstable(clusterNodes)) return Optional.empty();
ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type();
AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository);
Optional<Double> cpuLoad = averageLoad(Resource.cpu, clusterNodes, clusterType);
Optional<Double> memoryLoad = averageLoad(Resource.memory, clusterNodes, clusterType);
Optional<Double> diskLoad = averageLoad(Resource.disk, clusterNodes, clusterType);
if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty();
var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation);
Optional<AllocatableClusterResources> bestAllocation =
allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive);
if (bestAllocation.isEmpty()) return Optional.empty();
if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty();
return bestAllocation;
}
/** Returns true if both total real resources and total cost are similar */
private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) {
return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) &&
similar(a.realResources().vcpu() * a.nodes(),
b.realResources().vcpu() * b.nodes(), resourceDifferenceWorthReallocation) &&
similar(a.realResources().memoryGb() * a.nodes(),
b.realResources().memoryGb() * b.nodes(), resourceDifferenceWorthReallocation) &&
similar(a.realResources().diskGb() * a.nodes(),
b.realResources().diskGb() * b.nodes(), resourceDifferenceWorthReallocation);
}
private boolean similar(double r1, double r2, double threshold) {
return Math.abs(r1 - r2) / (( r1 + r2) / 2) < threshold;
}
/**
* Returns the average load of this resource in the measurement window,
* or empty if we are not in a position to make decisions from these measurements at this time.
*/
private Optional<Double> averageLoad(Resource resource, List<Node> clusterNodes, ClusterSpec.Type clusterType) {
NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(clusterType)),
resource,
clusterNodes.stream().map(Node::hostname).collect(Collectors.toList()));
// Require a total number of measurements scaling with the number of nodes,
// but don't require that we have at least that many from every node
if (window.measurementCount()/clusterNodes.size() < minimumMeasurementsPerNode) return Optional.empty();
if (window.hostnames() != clusterNodes.size()) return Optional.empty();
return Optional.of(window.average());
}
/** The duration of the window we need to consider to make a scaling decision */
static Duration scalingWindow(ClusterSpec.Type clusterType) {
if (clusterType.isContent()) return Duration.ofHours(12);
return Duration.ofHours(1);
}
public static boolean unstable(List<Node> nodes) {
return nodes.stream().anyMatch(node -> node.status().wantToRetire() ||
node.allocation().get().membership().retired() ||
node.allocation().get().isRemovable());
}
}
|
