Clean orphans when failing to get the lock

1 files changed, 14 insertions, 1 deletions

diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java
index 28d3b052ce2..f021f4ca955 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java
@@ -286,10 +286,12 @@ class SingletonManager {
             }
             catch (UncheckedTimeoutException e) {
                 logger.log(FINE, e, () -> "Timed out acquiring lock for '" + path + "' within " + tickTimeout);
+                cleanOrphans();
                 return;
             }
             catch (RuntimeException e) {
-                logger.log(INFO, e, () -> "Failed acquiring lock for '" + path + "' within " + tickTimeout);
+                logger.log(WARNING, e, () -> "Failed acquiring lock for '" + path + "' within " + tickTimeout);
+                cleanOrphans();
                 return;
             }
             try {
@@ -304,6 +306,17 @@ class SingletonManager {
             }
         }
 
+        private void cleanOrphans() {
+            List<String> orphans = null;
+            try {
+                for (String orphan : orphans = curator.framework().getZookeeperClient().getZooKeeper().getEphemerals(path.getAbsolute()))
+                    curator.delete(path.append(orphan));
+            }
+            catch (Exception e) {
+                logger.log(WARNING, "Failed cleaning orphans: " + orphans, e);
+            }
+        }
+
         /**
          * Attempt to activate or deactivate if status has changed.
          * If activation fails, we release the lock, so a different container may acquire it.