Hold lock when running maintenance job

2 files changed, 18 insertions, 3 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index e4a52636d22..d4a324517be 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
@@ -59,8 +60,11 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     @Override
     public void run() {
         try {
-            if (jobControl.isActive(name()))
-                maintain();
+            if (jobControl.isActive(name())) {
+                runWithLock();
+            }
+        } catch (UncheckedTimeoutException ignored) {
+            // Another config server is running this job
         } catch (Throwable e) {
             log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e);
         }
@@ -84,6 +88,13 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     @Override
     public final String toString() { return name(); }
 
+    /** Run this while holding the job lock */
+    public void runWithLock() {
+        try (var lock = nodeRepository.database().lockMaintenanceJob(name())) {
+            maintain();
+        }
+    }
+
     /** Called once each time this maintenance job should run */
     protected abstract void maintain();
     
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 87fc2c6323a..8ecdb0cbb1f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -366,8 +366,11 @@ public class CuratorDatabaseClient {
         return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper);
     }
 
-
     // Maintenance jobs
+    public Lock lockMaintenanceJob(String jobName) {
+        return lock(lockRoot.append("maintenanceJobLocks").append(jobName), defaultLockTimeout);
+    }
+
     public Set<String> readInactiveJobs() {
         try {
             return read(inactiveJobsPath(), stringSetSerializer::fromJson).orElseGet(HashSet::new);
@@ -554,4 +557,5 @@ public class CuratorDatabaseClient {
                 .mapToObj(i -> firstProvisionIndex + i)
                 .collect(Collectors.toList());
     }
+
 }