1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Deployer;
import com.yahoo.config.provision.NodeType;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* This maintainer detects changes to nodes that must be expedited, and redeploys affected applications.
*
* The purpose of this is to redeploy affected applications faster than achieved by
* {@link PeriodicApplicationMaintainer}, to reduce the time period where the node repository and the application model
* is out of sync.
*
* Why can't the manual change directly make the application redeployment?
*
* Because we want to queue redeployments to avoid overloading config servers.
*
* @author bratseth
* @author mpolden
*/
public class ExpeditedChangeApplicationMaintainer extends ApplicationMaintainer {
ExpeditedChangeApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, Duration interval) {
super(deployer, metric, nodeRepository, interval);
}
@Override
protected Map<ApplicationId, String> applicationsNeedingMaintenance() {
NodeList allNodes = nodeRepository().nodes().list();
Map<ApplicationId, String> applications = new HashMap<>();
allNodes.nodeType(NodeType.tenant, NodeType.proxy)
.matching(node -> node.allocation().isPresent())
.groupingBy(node -> node.allocation().get().owner())
.forEach((applicationId, nodes) -> {
hasNodesWithChanges(applicationId, nodes)
.ifPresent(reason -> applications.put(applicationId, reason));
});
// A ready proxy node should trigger a redeployment as it will activate the node.
if (!allNodes.state(Node.State.ready, Node.State.reserved).nodeType(NodeType.proxy).isEmpty()) {
applications.merge(ApplicationId.from("hosted-vespa", "routing", "default"),
"nodes being ready",
(oldValue, newValue) -> oldValue + ", " + newValue);
}
return applications;
}
/**
* Deploy in the maintenance thread to avoid scheduling multiple deployments of the same application if it takes
* longer to deploy than the (short) maintenance interval of this
*/
@Override
protected void deploy(ApplicationId application, String reason) {
deployNow(application, reason);
}
/** Returns the reason for doing an expedited deploy. */
private Optional<String> hasNodesWithChanges(ApplicationId applicationId, NodeList nodes) {
Optional<Instant> activationTime = deployer().activationTime(applicationId);
if (activationTime.isEmpty()) return Optional.empty();
List<String> reasons = nodes.stream()
.flatMap(node -> node.history()
.events()
.stream()
.filter(event -> expediteChangeBy(event.agent()))
.filter(event -> activationTime.get().isBefore(event.at()))
.map(event -> event.type() + (event.agent() == Agent.system ? "" : " by " + event.agent())))
.sorted()
.distinct()
.toList();
return reasons.isEmpty() ?
Optional.empty() :
Optional.of("recent node events: [" + String.join(", ", reasons) + "]");
}
@Override
protected boolean canDeployNow(ApplicationId application) {
return activeNodesByApplication().get(application) != null;
}
@Override
protected Map<ApplicationId, NodeList> activeNodesByApplication() {
return nodeRepository().nodes()
.list(Node.State.active)
.not().tester()
.groupingBy(node -> node.allocation().get().owner());
}
/** Returns whether to expedite changes performed by agent */
private boolean expediteChangeBy(Agent agent) {
return switch (agent) {
case operator, HostEncrypter, HostResumeProvisioner, RebuildingOsUpgrader -> true;
default -> false;
};
}
}
|
