Merge pull request #22163 from vespa-engine/bjorncs/container-threadpool

Introduce new metrics for container threadpool

4 files changed, 34 insertions, 15 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 34586df424f..3a8cf23a49e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -154,6 +154,17 @@ public class VespaMetricSet {
             addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes);
             addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffixes);
             addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.rejected_tasks", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.size", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.max_allowed_size", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.active_threads", suffixes);
+
+            addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", suffixes);
+            addMetric(metrics, "jdisc.http.jetty.threadpool.thread.min", suffixes);
+            addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", suffixes);
+            addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", suffixes);
+            addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", suffixes);
+            addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", suffixes);
         }
 
         metrics.add(new Metric("httpapi_latency.max"));
@@ -223,12 +234,6 @@ public class VespaMetricSet {
 
         metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate"));
 
-        addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", List.of("last"));
-        addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", List.of("last"));
-        addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", List.of("sum", "count", "min", "max"));
-        addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", List.of("sum", "count", "min", "max"));
-        addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", List.of("sum", "count", "min", "max"));
-
         addMetric(metrics, "jdisc.http.filtering.request.handled", List.of("rate"));
         addMetric(metrics, "jdisc.http.filtering.request.unhandled", List.of("rate"));
         addMetric(metrics, "jdisc.http.filtering.response.handled", List.of("rate"));
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
index 99f49f10526..6dd6b2d122e 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java
@@ -43,9 +43,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
         this.queueCapacity = threadPoolIsOnlyQ
                 ? wrapped.getMaximumPoolSize()
                 : maxQueueCapacity;
-
-        metric.reportThreadPoolSize(wrapped.getPoolSize());
-        metric.reportActiveThreads(wrapped.getActiveCount());
         reportMetrics();
         metricReporter = new Thread(this::reportMetricsRegularly);
         metricReporter.setName(name + "-threadpool-metric-reporter");
@@ -55,6 +52,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
     private void reportMetrics() {
         int activeThreads = wrapped.getActiveCount();
         metric.reportThreadPoolSize(wrapped.getPoolSize());
+        metric.reportMaxAllowedThreadPoolSize(wrapped.getMaximumPoolSize());
         metric.reportActiveThreads(activeThreads);
         int queueSize = threadPoolIsOnlyQ ? activeThreads : wrapped.getQueue().size();
         metric.reportWorkQueueSize(queueSize);
@@ -81,7 +79,6 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
 
     @Override
     public void shutdown() {
-        super.shutdown();
         synchronized (closed) {
             closed.set(true);
             closed.notify();
@@ -89,6 +86,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService {
         try {
             metricReporter.join();
         } catch (InterruptedException e) {}
+        super.shutdown();
     }
 
     /**
diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
index cc70abac59b..1c7a1cc4ebe 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java
@@ -22,9 +22,23 @@ class ThreadPoolMetric {
         this.defaultContext = metric.createContext(Map.of(THREAD_POOL_NAME_DIMENSION, threadPoolName));
     }
 
-    void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); }
-    void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); }
-    void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); }
+    void reportRejectRequest() {
+        metric.add("serverRejectedRequests", 1L, defaultContext);
+        metric.add("jdisc.thread_pool.rejected_tasks", 1L, defaultContext);
+    }
+
+    void reportThreadPoolSize(long size) {
+        metric.set("serverThreadPoolSize", size, defaultContext);
+        metric.set("jdisc.thread_pool.size", size, defaultContext);
+    }
+
+    void reportMaxAllowedThreadPoolSize(long size) { metric.set("jdisc.thread_pool.max_allowed_size", size, defaultContext); }
+
+    void reportActiveThreads(long threads) {
+        metric.set("serverActiveThreads", threads, defaultContext);
+        metric.set("jdisc.thread_pool.active_threads", threads, defaultContext);
+    }
+
     void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); }
     void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); }
     void reportUnhandledException(Throwable t) {
diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
index 536f7f599f2..c3f91000296 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java
@@ -79,8 +79,9 @@ public class ContainerThreadPoolImplTest {
         ThreadPoolExecutor executor = createPool(metrics, 3, 1200);
         assertEquals(3, executor.getMaximumPoolSize());
         assertEquals(1200, executor.getQueue().remainingCapacity());
-        assertEquals(4, metrics.innvocations().size());
+        assertEquals(7, metrics.innvocations().size());
         assertEquals(3L, metrics.innvocations().get("serverThreadPoolSize").val);
+        assertEquals(3L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val);
         assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val);
         assertEquals(1200L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val);
         assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val);
@@ -91,8 +92,9 @@ public class ContainerThreadPoolImplTest {
         ThreadPoolExecutor executor = createPool(metrics, 0, 0);
         assertEquals(CPUS*4, executor.getMaximumPoolSize());
         assertEquals(0, executor.getQueue().remainingCapacity());
-        assertEquals(4, metrics.innvocations().size());
+        assertEquals(7, metrics.innvocations().size());
         assertEquals(64L, metrics.innvocations().get("serverThreadPoolSize").val);
+        assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val);
         assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val);
         assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val);
         assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val);