1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
package com.yahoo.vespa.hosted.provision.maintenance;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.hosted.provision.Node;
import org.junit.Test;
import java.time.Duration;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
/**
* @author bratseth
*/
public class NodeRebooterTest {
@Test
public void testRebootScheduling() throws InterruptedException {
Duration rebootInterval = Duration.ofMinutes(250);
MaintenanceTester tester = new MaintenanceTester();
tester.createReadyTenantNodes(15);
tester.createReadyHostNodes(15);
// New nodes are rebooted when transitioning from dirty to ready. Advance the time so that additional reboots
// will be performed.
tester.clock.advance(rebootInterval);
NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, rebootInterval, new JobControl(tester.nodeRepository().database()));
maintenanceIntervals(rebooter, tester, 1);
assertEquals("All tenant nodes have reboot scheduled",
15,
withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
assertEquals("No nodes have 2 reboots scheduled",
0,
withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
maintenanceIntervals(rebooter, tester, 11);
assertEquals("Reboot interval is 10x iteration interval, so most nodes are now rebooted twice",
30,
withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(Node.State.ready)).size());
}
private void maintenanceIntervals(NodeRebooter rebooter, MaintenanceTester tester, int iterations) {
for (int i = 0; i < iterations; i++) {
tester.clock.advance(Duration.ofMinutes(25));
for (int j = 0; j < 60; j++) { // multiple runs to remove effects from the probabilistic smoothing in the reboot maintainer
rebooter.maintain();
simulateReboot(tester);
}
}
}
/** Set current reboot generation to the wanted reboot generation whenever it is larger (i.e record a reboot) */
private void simulateReboot(MaintenanceTester tester) {
for (Node node : tester.nodeRepository.getNodes(Node.State.ready, Node.State.active)) {
if (node.status().reboot().wanted() > node.status().reboot().current())
tester.nodeRepository.write(node.withCurrentRebootGeneration(node.status().reboot().wanted(),
tester.clock.instant()));
}
}
/** Returns the subset of the give nodes which have the given current reboot generation */
private List<Node> withCurrentRebootGeneration(long generation, List<Node> nodes) {
return nodes.stream().filter(n -> n.status().reboot().current() == generation).collect(Collectors.toList());
}
}
|
