Add thread pool name as dimension to metric values

4 files changed, 49 insertions, 24 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java
index 117fd995ff4..3789d5cbedc 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java
@@ -32,6 +32,7 @@ public class ContainerThreadPool extends AbstractComponent implements AutoClosea
     }
 
     public ContainerThreadPool(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) {
+        ThreadPoolMetric threadPoolMetric = new ThreadPoolMetric(metric, threadpoolConfig.name());
         int maxNumThreads = computeMaximumThreadPoolSize(threadpoolConfig.maxthreads());
         int coreNumThreads = computeCoreThreadPoolSize(threadpoolConfig.corePoolSize(), maxNumThreads);
         WorkerCompletionTimingThreadPoolExecutor executor =
@@ -39,12 +40,12 @@ public class ContainerThreadPool extends AbstractComponent implements AutoClosea
                         (int)threadpoolConfig.keepAliveTime() * 1000, TimeUnit.MILLISECONDS,
                         createQ(threadpoolConfig.queueSize(), maxNumThreads),
                         ThreadFactoryFactory.getThreadFactory(threadpoolConfig.name()),
-                        metric);
+                        threadPoolMetric);
         // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also
         // get the dreaded thread locals initialized even if they will never run.
         // That counters what we we want to achieve with the Q that will prefer thread locality.
         executor.prestartAllCoreThreads();
-        threadpool = new ExecutorServiceWrapper(executor, metric, processTerminator,
+        threadpool = new ExecutorServiceWrapper(executor, threadPoolMetric, processTerminator,
                 threadpoolConfig.maxThreadExecutionTimeSeconds() * 1000L);
     }
 
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
index f7b0a22120a..1144d1ebbf6 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
@@ -3,7 +3,6 @@ package com.yahoo.container.handler.threadpool;
 
 import com.google.common.util.concurrent.ForwardingExecutorService;
 import com.yahoo.container.protect.ProcessTerminator;
-import com.yahoo.jdisc.Metric;
 
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
@@ -22,7 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 class ExecutorServiceWrapper extends ForwardingExecutorService {
 
     private final WorkerCompletionTimingThreadPoolExecutor wrapped;
-    private final Metric metric;
+    private final ThreadPoolMetric metric;
     private final ProcessTerminator processTerminator;
     private final long maxThreadExecutionTimeMillis;
     private final Thread metricReporter;
@@ -30,16 +29,15 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
 
     ExecutorServiceWrapper(
             WorkerCompletionTimingThreadPoolExecutor wrapped,
-            Metric metric, ProcessTerminator processTerminator,
+            ThreadPoolMetric metric, ProcessTerminator processTerminator,
             long maxThreadExecutionTimeMillis) {
         this.wrapped = wrapped;
         this.metric = metric;
         this.processTerminator = processTerminator;
         this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis;
 
-        metric.set(MetricNames.THREAD_POOL_SIZE, wrapped.getPoolSize(), null);
-        metric.set(MetricNames.ACTIVE_THREADS, wrapped.getActiveCount(), null);
-        metric.add(MetricNames.REJECTED_REQUEST, 0, null);
+        metric.reportThreadPoolSize(wrapped.getPoolSize());
+        metric.reportActiveThreads(wrapped.getActiveCount());
         metricReporter = new Thread(this::reportMetrics);
         metricReporter.setDaemon(true);
         metricReporter.start();
@@ -48,8 +46,8 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
     private final void reportMetrics() {
         try {
             while (!closed.get()) {
-                metric.set(MetricNames.THREAD_POOL_SIZE, wrapped.getPoolSize(), null);
-                metric.set(MetricNames.ACTIVE_THREADS, wrapped.getActiveCount(), null);
+                metric.reportThreadPoolSize(wrapped.getPoolSize());
+                metric.reportActiveThreads(wrapped.getActiveCount());
                 Thread.sleep(100);
             }
         } catch (InterruptedException e) { }
@@ -72,7 +70,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
         try {
             super.execute(command);
         } catch (RejectedExecutionException e) {
-            metric.add(MetricNames.REJECTED_REQUEST, 1, null);
+            metric.reportRejectRequest();
             long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadAssignmentTimeMillis;
             if (timeSinceLastReturnedThreadMillis > maxThreadExecutionTimeMillis)
                 processTerminator.logAndDie("No worker threads have been available for " +
@@ -84,11 +82,5 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
     @Override
     protected ExecutorService delegate() { return wrapped; }
 
-    private static final class MetricNames {
-        private static final String REJECTED_REQUEST = "serverRejectedRequests";
-        private static final String THREAD_POOL_SIZE = "serverThreadPoolSize";
-        private static final String ACTIVE_THREADS   = "serverActiveThreads";
-    }
-
 }
 
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
new file mode 100644
index 00000000000..d9ab020bcb8
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
@@ -0,0 +1,35 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.handler.threadpool;
+
+import com.yahoo.jdisc.Metric;
+
+import java.util.Map;
+
+/**
+ * @author bjorncs
+ */
+class ThreadPoolMetric {
+
+    private static final String THREAD_POOL_NAME_DIMENSION = "threadpool";
+
+    private final Metric metric;
+    private final Metric.Context defaultContext;
+    private final String threadPoolName;
+
+    ThreadPoolMetric(Metric metric, String threadPoolName) {
+        this.metric = metric;
+        this.threadPoolName = threadPoolName;
+        this.defaultContext = metric.createContext(Map.of(THREAD_POOL_NAME_DIMENSION, threadPoolName));
+    }
+
+    void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); }
+    void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); }
+    void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); }
+    void reportUnhandledException(Throwable t) {
+        Metric.Context ctx = metric.createContext(Map.of(
+                THREAD_POOL_NAME_DIMENSION, threadPoolName,
+                "exception", t.getClass().getSimpleName()));
+        metric.set("jdisc.thread_pool.unhandled_exceptions", 1L, ctx);
+    }
+
+}
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java
index 9742e7ecfc3..56f8319c110 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java
@@ -1,9 +1,6 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.handler.threadpool;
 
-import com.yahoo.jdisc.Metric;
-
-import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -20,12 +17,12 @@ import java.util.concurrent.atomic.AtomicLong;
  */
 class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor {
 
-    private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions";
+
 
     volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis();
     private final AtomicLong startedCount = new AtomicLong(0);
     private final AtomicLong completedCount = new AtomicLong(0);
-    private final Metric metric;
+    private final ThreadPoolMetric metric;
 
     WorkerCompletionTimingThreadPoolExecutor(
             int corePoolSize,
@@ -34,7 +31,7 @@ class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor {
             TimeUnit unit,
             BlockingQueue<Runnable> workQueue,
             ThreadFactory threadFactory,
-            Metric metric) {
+            ThreadPoolMetric metric) {
         super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
         this.metric = metric;
     }
@@ -51,7 +48,7 @@ class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor {
         super.afterExecute(r, t);
         completedCount.incrementAndGet();
         if (t != null) {
-            metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, metric.createContext(Map.of("exception", t.getClass().getSimpleName())));
+            metric.reportUnhandledException(t);
         }
     }