Count events per zk path and move to separate package

6 files changed, 72 insertions, 97 deletions

diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json
index 614c58f3cf6..f4ad1ab4372 100644
--- a/zkfacade/abi-spec.json
+++ b/zkfacade/abi-spec.json
@@ -109,65 +109,5 @@
       "public void close()"
     ],
     "fields": []
-  },
-  "com.yahoo.vespa.curator.LockInfo$LockState": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.vespa.curator.LockInfo$LockState[] values()",
-      "public static com.yahoo.vespa.curator.LockInfo$LockState valueOf(java.lang.String)",
-      "public boolean isTerminal()"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRING",
-      "public static final enum com.yahoo.vespa.curator.LockInfo$LockState TIMED_OUT",
-      "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRED",
-      "public static final enum com.yahoo.vespa.curator.LockInfo$LockState FAILED_TO_REENTER",
-      "public static final enum com.yahoo.vespa.curator.LockInfo$LockState RELEASED"
-    ]
-  },
-  "com.yahoo.vespa.curator.LockInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public static com.yahoo.vespa.curator.LockInfo invokingAcquire(int, java.time.Duration)",
-      "public int getThreadHoldCountOnAcquire()",
-      "public java.time.Instant getTimeAcquiredWasInvoked()",
-      "public java.time.Duration getAcquireTimeout()",
-      "public com.yahoo.vespa.curator.LockInfo$LockState getLockState()",
-      "public java.util.Optional getTimeLockWasAcquired()",
-      "public java.util.Optional getTimeTerminalStateWasReached()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.vespa.curator.ThreadLockInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public static int invokeAcquireCount()",
-      "public static int inCriticalRegionCount()",
-      "public static int acquireTimedOutCount()",
-      "public static int lockAcquiredCount()",
-      "public static int locksReleasedCount()",
-      "public static int noLocksErrorCount()",
-      "public static int failedToAcquireReentrantLockCount()",
-      "public static int timeoutOnReentrancyErrorCount()",
-      "public static java.util.List getThreadLockInfos()",
-      "public java.lang.String getThreadName()",
-      "public java.lang.String getLockPath()",
-      "public java.util.List getLockInfos()"
-    ],
-    "fields": []
   }
 }
\ No newline at end of file
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
index 299fceb8d7f..da6e3109695 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.curator;
 import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.path.Path;
 import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.curator.stats.ThreadLockInfo;
 import org.apache.curator.framework.recipes.locks.InterProcessLock;
 
 import java.time.Duration;
@@ -39,7 +40,7 @@ public class Lock implements Mutex {
                 threadLockInfo.acquireTimedOut();
 
                 throw new UncheckedTimeoutException("Timed out after waiting " + timeout +
-                        " to acquire lock '" + lockPath + "'");
+                                                    " to acquire lock '" + lockPath + "'");
             }
             threadLockInfo.lockAcquired();
 
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java
new file mode 100644
index 00000000000..9052db5ce5a
--- /dev/null
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.curator.stats;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A collection of counters for events related to lock acquisition and release.
+ *
+ * @author hakon
+ */
+public class LockCounters {
+    final AtomicInteger invokeAcquireCount = new AtomicInteger(0);
+    final AtomicInteger inCriticalRegionCount = new AtomicInteger(0);
+    final AtomicInteger acquireTimedOutCount = new AtomicInteger(0);
+    final AtomicInteger lockAcquiredCount = new AtomicInteger(0);
+    final AtomicInteger locksReleasedCount = new AtomicInteger(0);
+
+    final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0);
+    final AtomicInteger noLocksErrorCount = new AtomicInteger(0);
+    final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0);
+
+    public int invokeAcquireCount() { return invokeAcquireCount.get(); }
+    public int inCriticalRegionCount() { return inCriticalRegionCount.get(); }
+    public int acquireTimedOutCount() { return acquireTimedOutCount.get(); }
+    public int lockAcquiredCount() { return lockAcquiredCount.get(); }
+    public int locksReleasedCount() { return locksReleasedCount.get(); }
+    public int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); }
+    public int noLocksErrorCount() { return noLocksErrorCount.get(); }
+    public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); }
+}
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java
index 870ee12ebda..b192210c3de 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java
@@ -1,5 +1,5 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.curator;
+package com.yahoo.vespa.curator.stats;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -9,6 +9,8 @@ import java.util.Optional;
  * Information about a lock.
  *
  * <p>Should be mutated by a single thread.  Other threads may see an inconsistent state of this instance.</p>
+ *
+ * @author hakon
  */
 public class LockInfo {
 
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java
index 9fbcb550ddd..bba39e6dc49 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java
@@ -1,8 +1,11 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.curator;
+package com.yahoo.vespa.curator.stats;
+
+import com.yahoo.vespa.curator.Lock;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
@@ -16,6 +19,8 @@ import java.util.function.Consumer;
  * {@link Lock}.  Instances of this class contain information tied to a specific thread and lock path.
  *
  * <p>Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.</p>
+ *
+ * @author hakon
  */
 public class ThreadLockInfo {
 
@@ -24,44 +29,35 @@ public class ThreadLockInfo {
     private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10;
     private static final ConcurrentLinkedDeque<LockInfo> completedLockInfos = new ConcurrentLinkedDeque<>();
 
-    private static final AtomicInteger invokeAcquireCount = new AtomicInteger(0);
-    private static final AtomicInteger inCriticalRegionCount = new AtomicInteger(0);
-    private static final AtomicInteger acquireTimedOutCount = new AtomicInteger(0);
-    private static final AtomicInteger lockAcquiredCount = new AtomicInteger(0);
-    private static final AtomicInteger locksReleasedCount = new AtomicInteger(0);
-
-    private static final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0);
-    private static final AtomicInteger noLocksErrorCount = new AtomicInteger(0);
-    private static final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0);
+    private static final ConcurrentHashMap<String, LockCounters> countersByLockPath = new ConcurrentHashMap<>();
 
     private final Thread thread;
     private final String lockPath;
     private final ReentrantLock lock;
+    private final LockCounters lockCountersForPath;
 
     /** The locks are reentrant so there may be more than 1 lock for this thread. */
     private final ConcurrentLinkedQueue<LockInfo> lockInfos = new ConcurrentLinkedQueue<>();
 
-    public static int invokeAcquireCount() { return invokeAcquireCount.get(); }
-    public static int inCriticalRegionCount() { return inCriticalRegionCount.get(); }
-    public static int acquireTimedOutCount() { return acquireTimedOutCount.get(); }
-    public static int lockAcquiredCount() { return lockAcquiredCount.get(); }
-    public static int locksReleasedCount() { return locksReleasedCount.get(); }
-    public static int noLocksErrorCount() { return noLocksErrorCount.get(); }
-    public static int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); }
-    public static int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); }
+    public static Map<String, LockCounters> getLockCountersByPath() { return Map.copyOf(countersByLockPath); }
+
     public static List<ThreadLockInfo> getThreadLockInfos() { return List.copyOf(locks.values()); }
 
     /** Returns the per-thread singleton ThreadLockInfo. */
-    static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) {
+    public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) {
         return locks.computeIfAbsent(
                 Thread.currentThread(),
-                currentThread -> new ThreadLockInfo(currentThread, lockPath, lock));
+                currentThread -> {
+                    LockCounters lockCounters = countersByLockPath.computeIfAbsent(lockPath, ignored -> new LockCounters());
+                    return new ThreadLockInfo(currentThread, lockPath, lock, lockCounters);
+                });
     }
 
-    ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock) {
+    ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock, LockCounters lockCountersForPath) {
         this.thread = currentThread;
         this.lockPath = lockPath;
         this.lock = lock;
+        this.lockCountersForPath = lockCountersForPath;
     }
 
     public String getThreadName() { return thread.getName(); }
@@ -69,35 +65,36 @@ public class ThreadLockInfo {
     public List<LockInfo> getLockInfos() { return List.copyOf(lockInfos); }
 
     /** Mutable method (see class doc) */
-    void invokingAcquire(Duration timeout) {
-        invokeAcquireCount.incrementAndGet();
-        inCriticalRegionCount.incrementAndGet();
+    public void invokingAcquire(Duration timeout) {
+        lockCountersForPath.invokeAcquireCount.incrementAndGet();
+        lockCountersForPath.inCriticalRegionCount.incrementAndGet();
         lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout));
     }
 
     /** Mutable method (see class doc) */
-    void acquireTimedOut() {
+    public void acquireTimedOut() {
         if (lockInfos.size() > 1) {
-            timeoutOnReentrancyErrorCount.incrementAndGet();
+            lockCountersForPath.timeoutOnReentrancyErrorCount.incrementAndGet();
         }
 
-        removeLastLockInfo(acquireTimedOutCount, LockInfo::timedOut);
+        removeLastLockInfo(lockCountersForPath.timeoutOnReentrancyErrorCount, LockInfo::timedOut);
     }
 
     /** Mutable method (see class doc) */
-    void lockAcquired() {
-        lockAcquiredCount.incrementAndGet();
+    public void lockAcquired() {
+        lockCountersForPath.lockAcquiredCount.incrementAndGet();
+
         getLastLockInfo().ifPresent(LockInfo::lockAcquired);
     }
 
     /** Mutable method (see class doc) */
-    void failedToAcquireReentrantLock() {
-        removeLastLockInfo(failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock);
+    public void failedToAcquireReentrantLock() {
+        removeLastLockInfo(lockCountersForPath.failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock);
     }
 
     /** Mutable method (see class doc) */
-    void lockReleased() {
-        removeLastLockInfo(locksReleasedCount, LockInfo::released);
+    public void lockReleased() {
+        removeLastLockInfo(lockCountersForPath.locksReleasedCount, LockInfo::released);
     }
 
     private Optional<LockInfo> getLastLockInfo() {
@@ -106,10 +103,10 @@ public class ThreadLockInfo {
 
     private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer<LockInfo> completeLockInfo) {
         metricToIncrement.incrementAndGet();
-        inCriticalRegionCount.decrementAndGet();
+        lockCountersForPath.inCriticalRegionCount.decrementAndGet();
 
         if (lockInfos.isEmpty()) {
-            noLocksErrorCount.incrementAndGet();
+            lockCountersForPath.noLocksErrorCount.incrementAndGet();
             return;
         }
 
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java
new file mode 100644
index 00000000000..15a81ffea70
--- /dev/null
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.curator.stats;
+
+import com.yahoo.osgi.annotation.ExportPackage;