Revert "Revert "Start controller maintainers with fixed spacing""

3 files changed, 41 insertions, 3 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 33555c08a43..10c47ef937d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -63,14 +63,14 @@ public class ControllerMaintenance extends AbstractComponent {
         outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl);
         versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl);
         upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
-        readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl);
+        readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl);
         clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient);
         clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl);
         deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl);
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
         dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
-        jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl);
+        jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl);
         osUpgraders = osUpgraders(controller, jobControl);
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, organization, apiAuthorityConfig);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
index bec3e969887..bd0c11d20f6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -38,6 +41,9 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     }
 
     public Maintainer(Controller controller, Duration interval, JobControl jobControl, String name, Set<SystemName> permittedSystems) {
+        if (interval.isNegative() || interval.isZero())
+            throw new IllegalArgumentException("Interval must be positive, but was " + interval);
+
         this.controller = controller;
         this.maintenanceInterval = interval;
         this.jobControl = jobControl;
@@ -45,7 +51,8 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         this.permittedSystems = ImmutableSet.copyOf(permittedSystems);
 
         service = new ScheduledThreadPoolExecutor(1);
-        service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS);
+        long delay = staggeredDelay(controller.curator().cluster(), controller.hostname(), controller.clock().instant(), interval);
+        service.scheduleAtFixedRate(this, delay, interval.toMillis(), TimeUnit.MILLISECONDS);
         jobControl.started(name());
     }
     
@@ -91,4 +98,13 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         return name();
     }
 
+    static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) {
+        if ( ! cluster.contains(host))
+            return interval.toMillis();
+
+        long offset = cluster.indexOf(host) * interval.toMillis() / cluster.size();
+        long timeUntilNextRun = Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
+        return timeUntilNextRun + interval.toMillis() / cluster.size();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
index c10ceef6adb..d6fd9d88fd9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
@@ -1,13 +1,17 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.Assert.assertEquals;
@@ -32,6 +36,24 @@ public class MaintainerTest {
         assertEquals(1, executions.get());
     }
 
+    @Test
+    public void staggering() {
+        List<HostName> cluster = Arrays.asList(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3"));
+        Instant now = Instant.ofEpochMilli(1001);
+        Duration interval = Duration.ofMillis(300);
+
+        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
+        now = Instant.ofEpochMilli(1101);
+        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
+        assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval));
+    }
+
     private Maintainer maintainerIn(SystemName system, AtomicInteger executions) {
         return new Maintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
                               "MockMaintainer", EnumSet.of(system)) {