Smooth reboots

This will smooth future reboots also after a mass reboot has injected
clumping in the node rboot times.

Credits: Frode :-)

2 files changed, 10 insertions, 8 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
index c54c2c543a6..75c1c32478f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
@@ -47,8 +47,8 @@ public class NodeRebooter extends Maintainer {
     
     private boolean shouldReboot(Node node) {
         Optional<History.Event> lastReboot = node.history().event(History.Event.Type.rebooted);
-        if (lastReboot.isPresent())
-            return lastReboot.get().at().plus(rebootInterval).isBefore(clock.instant());
+        if (lastReboot.isPresent() && lastReboot.get().at().plus(rebootInterval).isAfter(clock.instant()))
+            return false;
         else // schedule with a probability such that reboots of nodes are spread roughly over the reboot interval
             return random.nextDouble() < (double)rate().getSeconds() / (double)rebootInterval.getSeconds();
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
index 8d359a892ea..8ec66e2b886 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
@@ -27,7 +27,7 @@ public class NodeRebooterTest {
         
         NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, rebootInterval);
 
-        maintenanceIterations(rebooter, tester, 1);
+        maintenanceIntervals(rebooter, tester, 1);
         assertEquals("All tenant nodes have reboot scheduled",
                      15,
                      withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
@@ -38,17 +38,19 @@ public class NodeRebooterTest {
                      15,
                      withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.host, Node.State.ready)).size());
 
-        maintenanceIterations(rebooter, tester, 11);
-        assertEquals("Reboot interval is 10x iteration interval, so the same number of nodes are now rebooted twice",
+        maintenanceIntervals(rebooter, tester, 11);
+        assertEquals("Reboot interval is 10x iteration interval, so most nodes are now rebooted twice",
                      15,
                      withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
     }
     
-    private void maintenanceIterations(NodeRebooter rebooter, MaintenanceTester tester, int iterations) {
+    private void maintenanceIntervals(NodeRebooter rebooter, MaintenanceTester tester, int iterations) {
         for (int i = 0; i < iterations; i++) {
             tester.clock.advance(Duration.ofMinutes(25));
-            rebooter.maintain();
-            simulateReboot(tester);
+            for (int j = 0; j < 30; j++) { // multiple runs to remove effects from the probabilistic smoothing in the reboot maintainer
+                rebooter.maintain();
+                simulateReboot(tester);
+            }
         }
     }