Start controller maintainers with fixed spacing

2 files changed, 14 insertions, 1 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
index bec3e969887..eab7eed27f8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -45,7 +45,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         this.permittedSystems = ImmutableSet.copyOf(permittedSystems);
 
         service = new ScheduledThreadPoolExecutor(1);
-        service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS);
+        service.scheduleAtFixedRate(this, staggeredDelay(controller, interval), interval.toMillis(), TimeUnit.MILLISECONDS);
         jobControl.started(name());
     }
     
@@ -91,4 +91,12 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         return name();
     }
 
+    private static long staggeredDelay(Controller controller, Duration interval) {
+        int indexInCluster = 1 + controller.curator().cluster().indexOf(controller.hostname());
+        long intervalMillis = Math.max(1, interval.toMillis());
+        long nextCycleStart = ((controller.clock().millis() / intervalMillis) + 1) * intervalMillis;
+        long staggeredStart = nextCycleStart + intervalMillis * indexInCluster / Math.max(1, controller.curator().cluster().size());
+        return staggeredStart - controller.clock().millis();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
index c10ceef6adb..7bbecf88ae7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
@@ -32,6 +32,11 @@ public class MaintainerTest {
         assertEquals(1, executions.get());
     }
 
+    @Test
+    public void staggering() {
+        System.err.println(3 * 2 / 3);
+    }
+
     private Maintainer maintainerIn(SystemName system, AtomicInteger executions) {
         return new Maintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
                               "MockMaintainer", EnumSet.of(system)) {