Introduce new metrics for container threadpool

Add the new metrics to improve observability of container threadpools.
These will be included in the updated container tuning docs.

3 files changed, 23 insertions, 9 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
index 99f49f10526..6dd6b2d122e 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
@@ -43,9 +43,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
         this.queueCapacity = threadPoolIsOnlyQ
                 ? wrapped.getMaximumPoolSize()
                 : maxQueueCapacity;
-
-        metric.reportThreadPoolSize(wrapped.getPoolSize());
-        metric.reportActiveThreads(wrapped.getActiveCount());
         reportMetrics();
         metricReporter = new Thread(this::reportMetricsRegularly);
         metricReporter.setName(name + "-threadpool-metric-reporter");
@@ -55,6 +52,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
     private void reportMetrics() {
         int activeThreads = wrapped.getActiveCount();
         metric.reportThreadPoolSize(wrapped.getPoolSize());
+        metric.reportMaxAllowedThreadPoolSize(wrapped.getMaximumPoolSize());
         metric.reportActiveThreads(activeThreads);
         int queueSize = threadPoolIsOnlyQ ? activeThreads : wrapped.getQueue().size();
         metric.reportWorkQueueSize(queueSize);
@@ -81,7 +79,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
 
     @Override
     public void shutdown() {
-        super.shutdown();
         synchronized (closed) {
             closed.set(true);
             closed.notify();
@@ -89,6 +86,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
         try {
             metricReporter.join();
         } catch (InterruptedException e) {}
+        super.shutdown();
     }
 
     /**
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
index cc70abac59b..1c7a1cc4ebe 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
@@ -22,9 +22,23 @@ class ThreadPoolMetric {
         this.defaultContext = metric.createContext(Map.of(THREAD_POOL_NAME_DIMENSION, threadPoolName));
     }
 
-    void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); }
-    void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); }
-    void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); }
+    void reportRejectRequest() {
+        metric.add("serverRejectedRequests", 1L, defaultContext);
+        metric.add("jdisc.thread_pool.rejected_tasks", 1L, defaultContext);
+    }
+
+    void reportThreadPoolSize(long size) {
+        metric.set("serverThreadPoolSize", size, defaultContext);
+        metric.set("jdisc.thread_pool.size", size, defaultContext);
+    }
+
+    void reportMaxAllowedThreadPoolSize(long size) { metric.set("jdisc.thread_pool.max_allowed_size", size, defaultContext); }
+
+    void reportActiveThreads(long threads) {
+        metric.set("serverActiveThreads", threads, defaultContext);
+        metric.set("jdisc.thread_pool.active_threads", threads, defaultContext);
+    }
+
     void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); }
     void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); }
     void reportUnhandledException(Throwable t) {
diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
index 536f7f599f2..c3f91000296 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
@@ -79,8 +79,9 @@ public class ContainerThreadPoolImplTest {
         ThreadPoolExecutor executor = createPool(metrics, 3, 1200);
         assertEquals(3, executor.getMaximumPoolSize());
         assertEquals(1200, executor.getQueue().remainingCapacity());
-        assertEquals(4, metrics.innvocations().size());
+        assertEquals(7, metrics.innvocations().size());
         assertEquals(3L, metrics.innvocations().get("serverThreadPoolSize").val);
+        assertEquals(3L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val);
         assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val);
         assertEquals(1200L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val);
         assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val);
@@ -91,8 +92,9 @@ public class ContainerThreadPoolImplTest {
         ThreadPoolExecutor executor = createPool(metrics, 0, 0);
         assertEquals(CPUS*4, executor.getMaximumPoolSize());
         assertEquals(0, executor.getQueue().remainingCapacity());
-        assertEquals(4, metrics.innvocations().size());
+        assertEquals(7, metrics.innvocations().size());
         assertEquals(64L, metrics.innvocations().get("serverThreadPoolSize").val);
+        assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val);
         assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val);
         assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val);
         assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val);