diff options
| -rw-r--r-- | node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java | 75 | ||||
| -rw-r--r-- | zkfacade/abi-spec.json | 60 | ||||
| -rw-r--r-- | zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java | 3 | ||||
| -rw-r--r-- | zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java | 30 | ||||
| -rw-r--r-- | zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java (renamed from zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java) | 4 | ||||
| -rw-r--r-- | zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java (renamed from zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java) | 67 | ||||
| -rw-r--r-- | zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java | 5 |
7 files changed, 116 insertions, 128 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 817063592b1..bc4401ee03a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -5,14 +5,22 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.vespa.curator.LockInfo; -import com.yahoo.vespa.curator.ThreadLockInfo; +import com.yahoo.vespa.curator.stats.LockCounters; +import com.yahoo.vespa.curator.stats.LockInfo; +import com.yahoo.vespa.curator.stats.ThreadLockInfo; import java.io.IOException; import java.io.OutputStream; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.TreeMap; +/** + * Returns information related to ZooKeeper locks. + * + * @author hakon + */ public class LocksResponse extends HttpResponse { private final Slime slime; @@ -23,38 +31,43 @@ public class LocksResponse extends HttpResponse { this.slime = new Slime(); Cursor root = slime.setObject(); - root.setLong("in-critical-region", ThreadLockInfo.inCriticalRegionCount()); - root.setLong("invoke-acquire", ThreadLockInfo.invokeAcquireCount()); - root.setLong("acquire-timed-out", ThreadLockInfo.acquireTimedOutCount()); - root.setLong("lock-acquired", ThreadLockInfo.lockAcquiredCount()); - root.setLong("locks-released", ThreadLockInfo.locksReleasedCount()); - root.setLong("acquire-reentrant-lock-errors", ThreadLockInfo.failedToAcquireReentrantLockCount()); - root.setLong("no-locks-errors", ThreadLockInfo.noLocksErrorCount()); - root.setLong("timeout-on-reentrancy-errors", ThreadLockInfo.timeoutOnReentrancyErrorCount()); + Map<String, LockCounters> lockCountersByPath = new TreeMap<>(ThreadLockInfo.getLockCountersByPath()); + + Cursor lockPathsCursor = root.setArray("lock-paths"); + lockCountersByPath.forEach((lockPath, lockCounters) -> { + Cursor lockPathCursor = lockPathsCursor.addObject(); + lockPathCursor.setString("path", lockPath); + lockPathCursor.setLong("in-critical-region", lockCounters.inCriticalRegionCount()); + lockPathCursor.setLong("invoke-acquire", lockCounters.invokeAcquireCount()); + lockPathCursor.setLong("acquire-timed-out", lockCounters.acquireTimedOutCount()); + lockPathCursor.setLong("lock-acquired", lockCounters.lockAcquiredCount()); + lockPathCursor.setLong("locks-released", lockCounters.locksReleasedCount()); + lockPathCursor.setLong("acquire-reentrant-lock-errors", lockCounters.failedToAcquireReentrantLockCount()); + lockPathCursor.setLong("no-locks-errors", lockCounters.noLocksErrorCount()); + lockPathCursor.setLong("timeout-on-reentrancy-errors", lockCounters.timeoutOnReentrancyErrorCount()); + }); List<ThreadLockInfo> threadLockInfos = ThreadLockInfo.getThreadLockInfos(); - if (!threadLockInfos.isEmpty()) { - Cursor threadsCursor = root.setArray("threads"); - threadLockInfos.forEach(threadLockInfo -> { - Cursor threadLockInfoCursor = threadsCursor.addObject(); - threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); - threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); + Cursor threadsCursor = root.setArray("threads"); + threadLockInfos.forEach(threadLockInfo -> { + Cursor threadLockInfoCursor = threadsCursor.addObject(); + threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); + threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); - List<LockInfo> lockInfos = threadLockInfo.getLockInfos(); - if (!lockInfos.isEmpty()) { - Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); - lockInfos.forEach(lockInfo -> { - Cursor lockInfoCursor = lockInfosCursor.addObject(); - lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); - lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); - lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); - lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); - lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); - lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); - }); - } - }); - } + List<LockInfo> lockInfos = threadLockInfo.getLockInfos(); + if (!lockInfos.isEmpty()) { + Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); + lockInfos.forEach(lockInfo -> { + Cursor lockInfoCursor = lockInfosCursor.addObject(); + lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); + lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); + lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); + lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); + lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); + lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + }); + } + }); } @Override diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index 614c58f3cf6..f4ad1ab4372 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -109,65 +109,5 @@ "public void close()" ], "fields": [] - }, - "com.yahoo.vespa.curator.LockInfo$LockState": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.vespa.curator.LockInfo$LockState[] values()", - "public static com.yahoo.vespa.curator.LockInfo$LockState valueOf(java.lang.String)", - "public boolean isTerminal()" - ], - "fields": [ - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRING", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState TIMED_OUT", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRED", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState FAILED_TO_REENTER", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState RELEASED" - ] - }, - "com.yahoo.vespa.curator.LockInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.vespa.curator.LockInfo invokingAcquire(int, java.time.Duration)", - "public int getThreadHoldCountOnAcquire()", - "public java.time.Instant getTimeAcquiredWasInvoked()", - "public java.time.Duration getAcquireTimeout()", - "public com.yahoo.vespa.curator.LockInfo$LockState getLockState()", - "public java.util.Optional getTimeLockWasAcquired()", - "public java.util.Optional getTimeTerminalStateWasReached()" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.ThreadLockInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static int invokeAcquireCount()", - "public static int inCriticalRegionCount()", - "public static int acquireTimedOutCount()", - "public static int lockAcquiredCount()", - "public static int locksReleasedCount()", - "public static int noLocksErrorCount()", - "public static int failedToAcquireReentrantLockCount()", - "public static int timeoutOnReentrancyErrorCount()", - "public static java.util.List getThreadLockInfos()", - "public java.lang.String getThreadName()", - "public java.lang.String getLockPath()", - "public java.util.List getLockInfos()" - ], - "fields": [] } }
\ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 299fceb8d7f..da6e3109695 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.curator; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.curator.stats.ThreadLockInfo; import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; @@ -39,7 +40,7 @@ public class Lock implements Mutex { threadLockInfo.acquireTimedOut(); throw new UncheckedTimeoutException("Timed out after waiting " + timeout + - " to acquire lock '" + lockPath + "'"); + " to acquire lock '" + lockPath + "'"); } threadLockInfo.lockAcquired(); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java new file mode 100644 index 00000000000..9052db5ce5a --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A collection of counters for events related to lock acquisition and release. + * + * @author hakon + */ +public class LockCounters { + final AtomicInteger invokeAcquireCount = new AtomicInteger(0); + final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); + final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); + final AtomicInteger lockAcquiredCount = new AtomicInteger(0); + final AtomicInteger locksReleasedCount = new AtomicInteger(0); + + final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); + final AtomicInteger noLocksErrorCount = new AtomicInteger(0); + final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); + + public int invokeAcquireCount() { return invokeAcquireCount.get(); } + public int inCriticalRegionCount() { return inCriticalRegionCount.get(); } + public int acquireTimedOutCount() { return acquireTimedOutCount.get(); } + public int lockAcquiredCount() { return lockAcquiredCount.get(); } + public int locksReleasedCount() { return locksReleasedCount.get(); } + public int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } + public int noLocksErrorCount() { return noLocksErrorCount.get(); } + public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index 870ee12ebda..b192210c3de 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator; +package com.yahoo.vespa.curator.stats; import java.time.Duration; import java.time.Instant; @@ -9,6 +9,8 @@ import java.util.Optional; * Information about a lock. * * <p>Should be mutated by a single thread. Other threads may see an inconsistent state of this instance.</p> + * + * @author hakon */ public class LockInfo { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 9fbcb550ddd..bba39e6dc49 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -1,8 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator; +package com.yahoo.vespa.curator.stats; + +import com.yahoo.vespa.curator.Lock; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; @@ -16,6 +19,8 @@ import java.util.function.Consumer; * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. * * <p>Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.</p> + * + * @author hakon */ public class ThreadLockInfo { @@ -24,44 +29,35 @@ public class ThreadLockInfo { private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10; private static final ConcurrentLinkedDeque<LockInfo> completedLockInfos = new ConcurrentLinkedDeque<>(); - private static final AtomicInteger invokeAcquireCount = new AtomicInteger(0); - private static final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); - private static final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); - private static final AtomicInteger lockAcquiredCount = new AtomicInteger(0); - private static final AtomicInteger locksReleasedCount = new AtomicInteger(0); - - private static final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); - private static final AtomicInteger noLocksErrorCount = new AtomicInteger(0); - private static final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); + private static final ConcurrentHashMap<String, LockCounters> countersByLockPath = new ConcurrentHashMap<>(); private final Thread thread; private final String lockPath; private final ReentrantLock lock; + private final LockCounters lockCountersForPath; /** The locks are reentrant so there may be more than 1 lock for this thread. */ private final ConcurrentLinkedQueue<LockInfo> lockInfos = new ConcurrentLinkedQueue<>(); - public static int invokeAcquireCount() { return invokeAcquireCount.get(); } - public static int inCriticalRegionCount() { return inCriticalRegionCount.get(); } - public static int acquireTimedOutCount() { return acquireTimedOutCount.get(); } - public static int lockAcquiredCount() { return lockAcquiredCount.get(); } - public static int locksReleasedCount() { return locksReleasedCount.get(); } - public static int noLocksErrorCount() { return noLocksErrorCount.get(); } - public static int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } - public static int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } + public static Map<String, LockCounters> getLockCountersByPath() { return Map.copyOf(countersByLockPath); } + public static List<ThreadLockInfo> getThreadLockInfos() { return List.copyOf(locks.values()); } /** Returns the per-thread singleton ThreadLockInfo. */ - static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { + public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { return locks.computeIfAbsent( Thread.currentThread(), - currentThread -> new ThreadLockInfo(currentThread, lockPath, lock)); + currentThread -> { + LockCounters lockCounters = countersByLockPath.computeIfAbsent(lockPath, ignored -> new LockCounters()); + return new ThreadLockInfo(currentThread, lockPath, lock, lockCounters); + }); } - ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock) { + ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock, LockCounters lockCountersForPath) { this.thread = currentThread; this.lockPath = lockPath; this.lock = lock; + this.lockCountersForPath = lockCountersForPath; } public String getThreadName() { return thread.getName(); } @@ -69,35 +65,36 @@ public class ThreadLockInfo { public List<LockInfo> getLockInfos() { return List.copyOf(lockInfos); } /** Mutable method (see class doc) */ - void invokingAcquire(Duration timeout) { - invokeAcquireCount.incrementAndGet(); - inCriticalRegionCount.incrementAndGet(); + public void invokingAcquire(Duration timeout) { + lockCountersForPath.invokeAcquireCount.incrementAndGet(); + lockCountersForPath.inCriticalRegionCount.incrementAndGet(); lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout)); } /** Mutable method (see class doc) */ - void acquireTimedOut() { + public void acquireTimedOut() { if (lockInfos.size() > 1) { - timeoutOnReentrancyErrorCount.incrementAndGet(); + lockCountersForPath.timeoutOnReentrancyErrorCount.incrementAndGet(); } - removeLastLockInfo(acquireTimedOutCount, LockInfo::timedOut); + removeLastLockInfo(lockCountersForPath.timeoutOnReentrancyErrorCount, LockInfo::timedOut); } /** Mutable method (see class doc) */ - void lockAcquired() { - lockAcquiredCount.incrementAndGet(); + public void lockAcquired() { + lockCountersForPath.lockAcquiredCount.incrementAndGet(); + getLastLockInfo().ifPresent(LockInfo::lockAcquired); } /** Mutable method (see class doc) */ - void failedToAcquireReentrantLock() { - removeLastLockInfo(failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); + public void failedToAcquireReentrantLock() { + removeLastLockInfo(lockCountersForPath.failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); } /** Mutable method (see class doc) */ - void lockReleased() { - removeLastLockInfo(locksReleasedCount, LockInfo::released); + public void lockReleased() { + removeLastLockInfo(lockCountersForPath.locksReleasedCount, LockInfo::released); } private Optional<LockInfo> getLastLockInfo() { @@ -106,10 +103,10 @@ public class ThreadLockInfo { private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer<LockInfo> completeLockInfo) { metricToIncrement.incrementAndGet(); - inCriticalRegionCount.decrementAndGet(); + lockCountersForPath.inCriticalRegionCount.decrementAndGet(); if (lockInfos.isEmpty()) { - noLocksErrorCount.incrementAndGet(); + lockCountersForPath.noLocksErrorCount.incrementAndGet(); return; } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java new file mode 100644 index 00000000000..15a81ffea70 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.curator.stats; + +import com.yahoo.osgi.annotation.ExportPackage; |