Add thread name template as dimension for locked load for curator locks

2 files changed, 31 insertions, 1 deletions

diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java
index 0d5320c387a..58780f3906e 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java
@@ -2,6 +2,9 @@
 package com.yahoo.vespa.curator.stats;
 
 import java.time.Duration;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
 
 import static java.lang.Math.round;
 
@@ -22,18 +25,20 @@ public class LatencyMetrics {
     private final Duration maxActiveLatency;
     private final double startHz;
     private final double endHz;
+    private final Map<String, Double> loadByThread;
     private final double load;
     private final int maxLoad;
     private final int currentLoad;
 
     public LatencyMetrics(Duration latency, Duration maxLatency, Duration maxActiveLatency,
-                          double startHz, double endHz,
+                          double startHz, double endHz, Map<String, Double> loadByThread,
                           double load, int maxLoad, int currentLoad) {
         this.latency = latency;
         this.maxLatency = maxLatency;
         this.maxActiveLatency = maxActiveLatency;
         this.startHz = startHz;
         this.endHz = endHz;
+        this.loadByThread = new TreeMap<>(loadByThread);
         this.load = load;
         this.maxLoad = maxLoad;
         this.currentLoad = currentLoad;
@@ -54,6 +59,13 @@ public class LatencyMetrics {
     /** Returns the average number of intervals that ended in the period per second. */
     public double endHz() { return roundTo3DecimalPlaces(endHz); }
 
+    /** Returns the average load of the implied time periond, per thread, with 3 decimal places precision. */
+    public Map<String, Double> loadByThread() {
+        Map<String, Double> result = new TreeMap<>();
+        loadByThread.forEach((name, load) -> result.put(name, roundTo3DecimalPlaces(load)));
+        return Collections.unmodifiableMap(result);
+    }
+
     /** The average load of the implied time period, with 3 decimal places precision. */
     public double load() { return roundTo3DecimalPlaces(load); }
 
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyStats.java
index 367d5ab2b9f..999b3b4c6d2 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyStats.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyStats.java
@@ -3,7 +3,9 @@ package com.yahoo.vespa.curator.stats;
 
 import java.time.Duration;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Optional;
 import java.util.function.LongSupplier;
 import java.util.logging.Level;
@@ -36,6 +38,7 @@ public class LatencyStats {
     private long startOfPeriodNanos;
     private long endOfPeriodNanos;
     private double cumulativeLoadNanos;
+    private final Map<String, Long> cumulativeLoadNanosByThread = new HashMap<>();
     private Duration cumulativeLatency;
     private Duration maxLatency;
     private int numIntervalsStarted;
@@ -92,6 +95,8 @@ public class LatencyStats {
 
     private static class ActiveIntervalInfo {
         private final long startNanos;
+        // Poor man's attempt at collapsing thread names into their pool names, as that is the relevant (task) level here.
+        private final String threadNameTemplate = Thread.currentThread().getName().replaceAll("\\d+", "*");
         public ActiveIntervalInfo(long startOfIntervalNanos) { this.startNanos = startOfIntervalNanos; }
         public long startOfIntervalNanos() { return startNanos; }
     }
@@ -109,6 +114,11 @@ public class LatencyStats {
     private void pushEndOfPeriodToNow() {
         long currentNanos = nanoTimeSupplier.getAsLong();
         cumulativeLoadNanos += activeIntervals.size() * (currentNanos - endOfPeriodNanos);
+        for (ActiveIntervalInfo activeInterval : activeIntervals) {
+            cumulativeLoadNanosByThread.merge(activeInterval.threadNameTemplate,
+                                              currentNanos - endOfPeriodNanos,
+                                              Long::sum);
+        }
         endOfPeriodNanos = currentNanos;
     }
 
@@ -146,15 +156,22 @@ public class LatencyStats {
                 .orElse(maxLatency);
 
         final double startHz, endHz, load;
+        final Map<String, Double> loadByThread = new HashMap<>();
         long periodNanos = endOfPeriodNanos - startOfPeriodNanos;
         if (periodNanos > 0) {
             double periodSeconds = periodNanos / 1_000_000_000.0;
             startHz = numIntervalsStarted / periodSeconds;
             endHz = numIntervalsEnded / periodSeconds;
             load = cumulativeLoadNanos / periodNanos;
+            cumulativeLoadNanosByThread.forEach((name, threadLoad) -> {
+                if (threadLoad > 0) loadByThread.put(name, threadLoad / (double) periodNanos);
+            });
         } else {
             startHz = endHz = 0.0;
             load = activeIntervals.size();
+            for (ActiveIntervalInfo activeInterval : activeIntervals) {
+                loadByThread.put(activeInterval.threadNameTemplate, 1.0);
+            }
         }
 
         return new LatencyMetrics(latency,
@@ -162,6 +179,7 @@ public class LatencyStats {
                                   maxActiveLatency,
                                   startHz,
                                   endHz,
+                                  loadByThread,
                                   load,
                                   maxLoad,
                                   activeIntervals.size());