1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
import ai.vespa.metrics.ConfigServerMetrics;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.flags.IntFlag;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.History;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.logging.Logger;
/**
* Throttles provisioning of new hosts in dynamically provisioned zones.
*
* @author mpolden
*/
public class ProvisioningThrottler {
/** Metric that indicates whether throttling is active where 1 means active and 0 means inactive */
private static final String throttlingActiveMetric = ConfigServerMetrics.THROTTLED_HOST_PROVISIONING.baseName();
private static final Logger LOG = Logger.getLogger(ProvisioningThrottler.class.getName());
private static final Duration WINDOW = Duration.ofDays(1);
private final NodeRepository nodeRepository;
private final Metric metric;
private final IntFlag maxHostsPerHourFlag;
public ProvisioningThrottler(NodeRepository nodeRepository, Metric metric) {
this.nodeRepository = Objects.requireNonNull(nodeRepository);
this.metric = Objects.requireNonNull(metric);
this.maxHostsPerHourFlag = PermanentFlags.MAX_HOSTS_PER_HOUR.bindTo(nodeRepository.flagSource());
}
/** Returns whether provisioning should be throttled at given instant */
public boolean throttle(NodeList allNodes, Agent agent) {
Instant startOfWindow = nodeRepository.clock().instant().minus(WINDOW);
int provisionedRecently = allNodes.matching(host -> host.history().hasEventAfter(History.Event.Type.provisioned,
startOfWindow))
.size();
boolean throttle = throttle(provisionedRecently, maxHostsPerHourFlag.value(), agent);
metric.set(throttlingActiveMetric, throttle ? 1 : 0, null);
return throttle;
}
static boolean throttle(int recentHosts, int maxPerHour, Agent agent) {
double rate = recentHosts / (double) WINDOW.toMillis();
boolean throttle = (rate * Duration.ofHours(1).toMillis()) > maxPerHour;
if (throttle) {
LOG.warning(String.format("Throttling provisioning of new hosts by %s: %d hosts have been provisioned " +
"in the past %s, which exceeds maximum rate of %d hosts/hour", agent,
recentHosts, WINDOW, maxPerHour));
}
return throttle;
}
}
|
