Add count of failed releases

4 files changed, 18 insertions, 10 deletions

diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
index 60e497683a6..920bba22804 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
@@ -56,11 +56,12 @@ public class Lock implements Mutex {
 
     @Override
     public void close() {
-        ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath);
         try {
             mutex.release();
+            ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath);
         }
         catch (Exception e) {
+            ThreadLockStats.getCurrentThreadLockStats().lockReleaseFailed(lockPath);
             throw new RuntimeException("Exception releasing lock '" + lockPath + "'");
         }
     }
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java
index c092fb6c289..3b06377ccf7 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java
@@ -29,7 +29,8 @@ public class LockAttempt {
     }
 
     public enum LockState {
-        ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true);
+        ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true),
+        RELEASED_WITH_ERROR(true);
 
         private final boolean terminal;
 
@@ -84,6 +85,7 @@ public class LockAttempt {
     void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); }
     void timedOut() { setTerminalState(LockState.TIMED_OUT); }
     void released() { setTerminalState(LockState.RELEASED); }
+    void releasedWithError() { setTerminalState(LockState.RELEASED_WITH_ERROR); }
 
     void lockAcquired() {
         lockState = LockState.ACQUIRED;
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java
index 5309bf755d7..561ea9a7ed2 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java
@@ -18,7 +18,7 @@ public class LockCounters {
     final AtomicInteger locksReleasedCount = new AtomicInteger(0);
 
     final AtomicInteger noLocksErrorCount = new AtomicInteger(0);
-    final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0);
+    final AtomicInteger lockReleaseErrorCount = new AtomicInteger(0);
 
     public int invokeAcquireCount() { return invokeAcquireCount.get(); }
     public int inCriticalRegionCount() { return inCriticalRegionCount.get(); }
@@ -27,7 +27,7 @@ public class LockCounters {
     public int lockAcquiredCount() { return lockAcquiredCount.get(); }
     public int locksReleasedCount() { return locksReleasedCount.get(); }
     public int noLocksErrorCount() { return noLocksErrorCount.get(); }
-    public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); }
+    public int lockReleaseErrorCount() { return lockReleaseErrorCount.get(); }
 
     @Override
     public String toString() {
@@ -39,7 +39,7 @@ public class LockCounters {
                 ", lockAcquiredCount=" + lockAcquiredCount +
                 ", locksReleasedCount=" + locksReleasedCount +
                 ", noLocksErrorCount=" + noLocksErrorCount +
-                ", timeoutOnReentrancyErrorCount=" + timeoutOnReentrancyErrorCount +
+                ", locksReleaseErrorCount=" + lockReleaseErrorCount +
                 '}';
     }
 
@@ -55,11 +55,12 @@ public class LockCounters {
                 lockAcquiredCount.get() == that.lockAcquiredCount.get() &&
                 locksReleasedCount.get() == that.locksReleasedCount.get() &&
                 noLocksErrorCount.get() == that.noLocksErrorCount.get() &&
-                timeoutOnReentrancyErrorCount.get() == that.timeoutOnReentrancyErrorCount.get();
+                lockReleaseErrorCount.get() == that.lockReleaseErrorCount.get();
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount, lockAcquiredCount, locksReleasedCount, noLocksErrorCount, timeoutOnReentrancyErrorCount);
+        return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount,
+                lockAcquiredCount, locksReleasedCount, noLocksErrorCount, lockReleaseErrorCount);
     }
 }
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java
index 077c3e2fd3d..db26523ec37 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java
@@ -95,9 +95,6 @@ public class ThreadLockStats {
     /** Mutable method (see class doc) */
     public void acquireTimedOut(String lockPath) {
         LockCounters lockCounters = getLockCounters(lockPath);
-        if (lockAttempts.size() > 1) {
-            lockCounters.timeoutOnReentrancyErrorCount.incrementAndGet();
-        }
 
         lockCounters.acquireTimedOutCount.incrementAndGet();
         removeLastLockAttempt(lockCounters, LockAttempt::timedOut);
@@ -120,6 +117,13 @@ public class ThreadLockStats {
         removeLastLockAttempt(lockCounters, LockAttempt::released);
     }
 
+    /** Mutable method (see class doc) */
+    public void lockReleaseFailed(String lockPath) {
+        LockCounters lockCounters = getLockCounters(lockPath);
+        lockCounters.lockReleaseErrorCount.incrementAndGet();
+        removeLastLockAttempt(lockCounters, LockAttempt::releasedWithError);
+    }
+
     private LockCounters getLockCounters(String lockPath) {
         return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters());
     }