Spread maintainer interval boundaries based on class name hash

3 files changed, 27 insertions, 18 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
index 68af9aa0a49..fdb73e405c0 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
@@ -148,14 +148,16 @@ public abstract class Maintainer implements Runnable {
         return name == null ? this.getClass().getSimpleName() : name;
     }
 
-    /** Returns the initial delay of this calculated from cluster index of given hostname */
-    static Duration staggeredDelay(Duration interval, Instant now, String hostname, List<String> clusterHostnames) {
+    /** Returns the initial delay of this calculated from cluster index of the hostname of this node, and the maintainer name. */
+    Duration staggeredDelay(Duration interval, Instant now, String hostname, List<String> clusterHostnames) {
         Objects.requireNonNull(clusterHostnames);
         if ( ! clusterHostnames.contains(hostname))
             return interval;
 
-        long offset = clusterHostnames.indexOf(hostname) * interval.toMillis() / clusterHostnames.size();
-        return Duration.ofMillis(Math.floorMod(offset - now.toEpochMilli(), interval.toMillis()));
+        long nodeOffset = clusterHostnames.indexOf(hostname) * interval.toMillis() / clusterHostnames.size();
+        long maintainerOffset = getClass().getName().hashCode() % interval.toMillis();
+        long totalOffset = nodeOffset + maintainerOffset;
+        return Duration.ofMillis(Math.floorMod(totalOffset - now.toEpochMilli(), interval.toMillis()));
     }
 
     private static Duration requireInterval(Duration interval) {
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
index cdb5e36a455..01e23c81971 100644
--- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java
@@ -24,21 +24,28 @@ public class MaintainerTest {
         List<String> cluster = List.of("cfg1", "cfg2", "cfg3");
         Duration interval = Duration.ofMillis(300);
         Instant now = Instant.ofEpochMilli(1000);
-        assertEquals(200, Maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
-        assertEquals(0, Maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
-        assertEquals(100, Maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
+        // ∠( ᐛ 」∠)＿
+        class MaintainerWithBestHashE extends TestMaintainer { MaintainerWithBestHashE() { super(null, jobControl, new TestJobMetrics()); } }
+        class MaintainerWithBestHashF extends TestMaintainer { MaintainerWithBestHashF() { super(null, jobControl, new TestJobMetrics()); } }
+        class MaintainerWithBestHashG extends TestMaintainer { MaintainerWithBestHashG() { super(null, jobControl, new TestJobMetrics()); } }
+        Maintainer maintainer = new MaintainerWithBestHashF();
+        assertEquals(200, maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
+        assertEquals(299, new MaintainerWithBestHashE().staggeredDelay(interval, now, "cfg2", cluster).toMillis());
+        assertEquals(0, maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
+        assertEquals(1, new MaintainerWithBestHashG().staggeredDelay(interval, now, "cfg2", cluster).toMillis());
+        assertEquals(100, maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
 
         now = Instant.ofEpochMilli(1001);
-        assertEquals(199, Maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
-        assertEquals(299, Maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
-        assertEquals(99, Maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
+        assertEquals(199, maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
+        assertEquals(299, maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
+        assertEquals(99, maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
 
         now = Instant.ofEpochMilli(1101);
-        assertEquals(99, Maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
-        assertEquals(199, Maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
-        assertEquals(299, Maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
+        assertEquals(99, maintainer.staggeredDelay(interval, now, "cfg1", cluster).toMillis());
+        assertEquals(199, maintainer.staggeredDelay(interval, now, "cfg2", cluster).toMillis());
+        assertEquals(299, maintainer.staggeredDelay(interval, now, "cfg3", cluster).toMillis());
 
-        assertEquals(300, Maintainer.staggeredDelay(interval, now, "cfg0", cluster).toMillis());
+        assertEquals(300, maintainer.staggeredDelay(interval, now, "cfg0", cluster).toMillis());
     }
 
     @Test
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
index d8191b98a51..9e994312738 100644
--- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java
@@ -15,20 +15,20 @@ class TestMaintainer extends Maintainer {
     private double success = 1.0;
     private RuntimeException exceptionToThrow = null;
 
-    public TestMaintainer(String name, JobControl jobControl, JobMetrics jobMetrics) {
+    TestMaintainer(String name, JobControl jobControl, JobMetrics jobMetrics) {
         super(name, Duration.ofDays(1), Clock.systemUTC(), jobControl, jobMetrics, List.of(), false);
     }
 
-    public int totalRuns() {
+    int totalRuns() {
         return totalRuns;
     }
 
-    public TestMaintainer successOnNextRun(double success) {
+    TestMaintainer successOnNextRun(double success) {
         this.success = success;
         return this;
     }
 
-    public TestMaintainer throwOnNextRun(RuntimeException e) {
+    TestMaintainer throwOnNextRun(RuntimeException e) {
         this.exceptionToThrow = e;
         return this;
     }