diff options
4 files changed, 23 insertions, 6 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 120e323e652..78e97719af0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; @@ -129,7 +130,12 @@ public class VespaMetricSet { metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); - metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate")); + { + List<String> suffices = List.of("sum", "count", "last", "min", "max"); + addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffices); + } metrics.add(new Metric("httpapi_latency.max")); metrics.add(new Metric("httpapi_latency.sum")); @@ -687,4 +693,10 @@ public class VespaMetricSet { return metrics; } + private static void addMetric(Set<Metric> metrics, String metricName, List<String> aggregateSuffices) { + for (String suffix : aggregateSuffices) { + metrics.add(new Metric(metricName + "." + suffix)); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java index 4d8c245a25a..46b3a86798b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java @@ -44,8 +44,9 @@ public class DefaultContainerThreadpool extends AbstractComponent implements Aut // get the dreaded thread locals initialized even if they will never run. // That counters what we we want to achieve with the Q that will prefer thread locality. executor.prestartAllCoreThreads(); - threadpool = new ExecutorServiceWrapper(executor, threadPoolMetric, processTerminator, - config.maxThreadExecutionTimeSeconds() * 1000L); + threadpool = new ExecutorServiceWrapper( + executor, threadPoolMetric, processTerminator, config.maxThreadExecutionTimeSeconds() * 1000L, + config.name(), config.queueSize()); } @Override public Executor executor() { return threadpool; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 3b2b5697e5c..771c1da82b6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -24,17 +24,19 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { private final ThreadPoolMetric metric; private final ProcessTerminator processTerminator; private final long maxThreadExecutionTimeMillis; + private final int queueCapacity; private final Thread metricReporter; private final AtomicBoolean closed = new AtomicBoolean(false); ExecutorServiceWrapper( WorkerCompletionTimingThreadPoolExecutor wrapped, ThreadPoolMetric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis, String name) { + long maxThreadExecutionTimeMillis, String name, int queueCapacity) { this.wrapped = wrapped; this.metric = metric; this.processTerminator = processTerminator; this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; + this.queueCapacity = queueCapacity; metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); @@ -44,13 +46,13 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { metricReporter.start(); } - int queuedTasks() { return wrapped.getQueue().size(); } - private void reportMetrics() { try { while (!closed.get()) { metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); + metric.reportWorkQueueSize(wrapped.getQueue().size()); + metric.reportWorkQueueCapacity(queueCapacity); Thread.sleep(100); } } catch (InterruptedException e) { } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index d9ab020bcb8..18ccf3ba8e5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -25,6 +25,8 @@ class ThreadPoolMetric { void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); } void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); } void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); } + void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); } + void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); } void reportUnhandledException(Throwable t) { Metric.Context ctx = metric.createContext(Map.of( THREAD_POOL_NAME_DIMENSION, threadPoolName, |