Merge pull request #8892 from vespa-engine/lesters/unbounded-docproc-threads

Set number of potential docproc threads to unbounded

3 files changed, 19 insertions, 9 deletions

diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java
index 1d3e85057ec..23ab73c7f1d 100644
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java
+++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java
@@ -20,8 +20,8 @@ public class DocprocThreadPoolExecutor extends ThreadPoolExecutor {
 
     public DocprocThreadPoolExecutor(int maxNumThreads, BlockingQueue<Runnable> queue, DocprocThreadManager threadMgr) {
         super((maxNumThreads > 0) ? maxNumThreads : Runtime.getRuntime().availableProcessors(),
-              (maxNumThreads > 0) ? maxNumThreads : Runtime.getRuntime().availableProcessors(),
-              5, TimeUnit.MINUTES,
+              (maxNumThreads > 0) ? maxNumThreads : 8192,
+              1, TimeUnit.SECONDS,
               queue,
               new DaemonThreadFactory("docproc-"));
         this.threadManager = threadMgr;
diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
index 1b84d0f9ffa..07efd419c97 100644
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
+++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
@@ -90,11 +90,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
                                      DocumentProcessingHandlerParameters params) {
         this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry,
              new DocprocThreadPoolExecutor(params.getMaxNumThreads(),
-                                           (params.getMaxQueueTimeMs() > 0)
-                                               ? new ThroughputLimitQueue<>(params.getMaxQueueTimeMs())
-                                               : (params.getMaxQueueTimeMs() < 0)
-                                                   ? new LinkedBlockingQueue<>()
-                                                   : new PriorityBlockingQueue<>(), //Probably no need to bound this queue, see bug #4254537
+                                           chooseQueueType(params),
                                            new DocprocThreadManager(params.getMaxConcurrentFactor(),
                                                                     params.getDocumentExpansionFactor(),
                                                                     params.getContainerCoreMemoryMb(),
@@ -106,6 +102,19 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
              params.getContainerDocConfig());
     }
 
+    private static BlockingQueue<Runnable> chooseQueueType(DocumentProcessingHandlerParameters params) {
+        if (params.getMaxQueueTimeMs() > 0) {
+            return new ThroughputLimitQueue<>(params.getMaxQueueTimeMs());
+        }
+        if (params.getMaxQueueTimeMs() == 0) {
+            return new PriorityBlockingQueue<>(); // Probably no need to bound this queue, see bug #4254537
+        }
+        if (params.getMaxNumThreads() > 0) {
+            return new LinkedBlockingQueue<>();
+        }
+        return new SynchronousQueue<>();
+    }
+
     @Inject
     public DocumentProcessingHandler(ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry,
                                      ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry,
diff --git a/docproc/src/main/resources/configdefinitions/docproc.def b/docproc/src/main/resources/configdefinitions/docproc.def
index 3b4c4fc5b16..645fbcce1b7 100644
--- a/docproc/src/main/resources/configdefinitions/docproc.def
+++ b/docproc/src/main/resources/configdefinitions/docproc.def
@@ -4,8 +4,9 @@ namespace=config.docproc
 # Queue size (in milliseconds) for this node
 # Positive size gives a ThroughPutLimitQueue. ### Experimental.
 # 0 Gives a PriorityQueue
-# Negative values gives a ordinary LinkedBlockingQueue. This is fastest and most efficient.
+# Negative values gives a SynchronousQueue if numthreads <= 0 (which increases thread pool size as required),
+# otherwise a LinkedBlockingQueue
 maxqueuetimems int default=-1
 
-#The number of threads in the DocprocHandler worker thread pool
+# The number of threads in the DocprocHandler worker thread pool
 numthreads int default=-1