Allow applications to override threadpool configuration in services.xml

9 files changed, 197 insertions, 23 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
index a43941f20d0..feaa6eb5940 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
@@ -33,7 +33,8 @@ public class ContainerDocumentApi {
         var handler = newVespaClientHandler(
                 "com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, options);
         cluster.addComponent(handler);
-        var executor = new Threadpool("feedapi-handler", cluster, options);
+        var executor = new Threadpool(
+                "feedapi-handler", cluster, options.feedApiThreadpoolOptions, options.feedThreadPoolSizeFactor);
         handler.inject(executor);
         handler.addComponent(executor);
     }
@@ -43,7 +44,8 @@ public class ContainerDocumentApi {
         var handler = newVespaClientHandler(
                 "com.yahoo.document.restapi.resource.RestApi", "/document/v1/*", options);
         cluster.addComponent(handler);
-        var executor = new Threadpool("restapi-handler", cluster, options);
+        var executor = new Threadpool(
+                "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor);
         handler.inject(executor);
         handler.addComponent(executor);
     }
@@ -71,10 +73,17 @@ public class ContainerDocumentApi {
 
     public static final class Options {
         private final Collection<String> bindings;
+        private final ContainerThreadpool.UserOptions restApiThreadpoolOptions;
+        private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions;
         private final double feedThreadPoolSizeFactor;
 
-        public Options(Collection<String> bindings, double feedThreadPoolSizeFactor) {
+        public Options(Collection<String> bindings,
+                       ContainerThreadpool.UserOptions restApiThreadpoolOptions,
+                       ContainerThreadpool.UserOptions feedApiThreadpoolOptions,
+                       double feedThreadPoolSizeFactor) {
             this.bindings = Collections.unmodifiableCollection(bindings);
+            this.restApiThreadpoolOptions = restApiThreadpoolOptions;
+            this.feedApiThreadpoolOptions = feedApiThreadpoolOptions;
             this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor;
         }
     }
@@ -82,32 +91,39 @@ public class ContainerDocumentApi {
     private static class Threadpool extends ContainerThreadpool {
 
         private final ContainerCluster<?> cluster;
-        private final Options options;
+        private final double feedThreadPoolSizeFactor;
 
-        Threadpool(String name, ContainerCluster<?> cluster, Options options) {
-            super(name);
+        Threadpool(String name,
+                   ContainerCluster<?> cluster,
+                   ContainerThreadpool.UserOptions threadpoolOptions,
+                   double feedThreadPoolSizeFactor ) {
+            super(name, threadpoolOptions);
             this.cluster = cluster;
-            this.options = options;
+            this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor;
         }
 
         @Override
         public void getConfig(ContainerThreadpoolConfig.Builder builder) {
             super.getConfig(builder);
-            builder.maxThreads(maxPoolSize(cluster, options));
-            builder.minThreads(minPoolSize(cluster, options));
+
+            // User options overrides below configuration
+            if (hasUserOptions()) return;
+
+            builder.maxThreads(maxPoolSize());
+            builder.minThreads(minPoolSize());
             builder.queueSize(500);
         }
 
-        private static int maxPoolSize(ContainerCluster<?> cluster, Options options) {
+        private int maxPoolSize() {
             double vcpu = vcpu(cluster);
             if (vcpu == 0) return FALLBACK_MAX_POOL_SIZE;
-            return Math.max(2, (int)Math.ceil(vcpu * options.feedThreadPoolSizeFactor));
+            return Math.max(2, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor));
         }
 
-        private static int minPoolSize(ContainerCluster<?> cluster, Options options) {
+        private int minPoolSize() {
             double vcpu = vcpu(cluster);
             if (vcpu == 0) return FALLBACK_CORE_POOL_SIZE;
-            return Math.max(1, (int)Math.ceil(vcpu * options.feedThreadPoolSizeFactor * 0.5));
+            return Math.max(1, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor * 0.5));
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java
index 6e4514c31b4..c4d252ccbfe 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java
@@ -5,9 +5,12 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.container.handler.threadpool.ContainerThreadPool;
 import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig;
 import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
+import org.w3c.dom.Element;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -18,17 +21,32 @@ import java.util.stream.Collectors;
 public class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer {
 
     private final String name;
+    private final UserOptions userOptions;
 
-    public ContainerThreadpool(String name) {
+    public ContainerThreadpool(String name) { this(name, null); }
+
+    public ContainerThreadpool(String name, UserOptions userOptions) {
         super(new ComponentModel(
                 BundleInstantiationSpecification.getFromStrings(
                         "threadpool@" + name,
                         ContainerThreadPool.class.getName(),
                         null)));
         this.name = name;
+        this.userOptions = userOptions;
+    }
+
+    @Override
+    public void getConfig(ContainerThreadpoolConfig.Builder builder) {
+        builder.name(this.name);
+        if (userOptions != null) {
+            builder.maxThreads(userOptions.maxThreads);
+            builder.minThreads(userOptions.minThreads);
+            builder.queueSize(userOptions.queueSize);
+        }
     }
 
-    @Override public void getConfig(ContainerThreadpoolConfig.Builder builder) { builder.name(this.name); }
+    protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); }
+    protected boolean hasUserOptions() { return userOptions().isPresent(); }
 
     protected static double vcpu(ContainerCluster<?> cluster) {
         List<Double> vcpus = cluster.getContainers().stream()
@@ -40,4 +58,29 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr
         if (vcpus.size() != 1 || vcpus.get(0) == 0) return 0;
         return vcpus.get(0);
     }
+
+    public static class UserOptions {
+        private final int maxThreads;
+        private final int minThreads;
+        private final int queueSize;
+
+        private UserOptions(int maxThreads, int minThreads, int queueSize) {
+            this.maxThreads = maxThreads;
+            this.minThreads = minThreads;
+            this.queueSize = queueSize;
+        }
+
+        public static Optional<UserOptions> fromXml(Element xml) {
+            Element element = XML.getChild(xml, "threadpool");
+            if (element == null) return Optional.empty();
+            return Optional.of(new UserOptions(
+                    intOption(element, "max-threads"),
+                    intOption(element, "min-threads"),
+                    intOption(element, "queue-size")));
+        }
+
+        private static int intOption(Element element, String name) {
+            return Integer.parseInt(XML.getChild(element, name).getTextContent());
+        }
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 150e103e304..638c02caf55 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -54,6 +54,7 @@ import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.ContainerModel;
 import com.yahoo.vespa.model.container.ContainerModelEvaluation;
+import com.yahoo.vespa.model.container.ContainerThreadpool;
 import com.yahoo.vespa.model.container.IdentityProvider;
 import com.yahoo.vespa.model.container.SecretStore;
 import com.yahoo.vespa.model.container.component.BindingPattern;
@@ -786,7 +787,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                                                      "com.yahoo.search.searchchain.ExecutionFactory"));
 
         cluster.addComponent(
-                new SearchHandler(cluster, serverBindings(searchElement, SearchHandler.DEFAULT_BINDING), deployState));
+                new SearchHandler(
+                        cluster,
+                        serverBindings(searchElement, SearchHandler.DEFAULT_BINDING),
+                        ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null),
+                        deployState));
     }
 
     private void addGUIHandler(ApplicationContainerCluster cluster) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
index 34de21de404..99ae6184f5c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.model.clients.ContainerDocumentApi;
+import com.yahoo.vespa.model.container.ContainerThreadpool;
 import org.w3c.dom.Element;
 
 import java.util.ArrayList;
@@ -20,7 +21,17 @@ public class DocumentApiOptionsBuilder {
 
 
     public static ContainerDocumentApi.Options build(DeployState deployState, Element spec) {
-        return new ContainerDocumentApi.Options(getBindings(spec), deployState.getProperties().feedCoreThreadPoolSizeFactor());
+        return new ContainerDocumentApi.Options(
+                getBindings(spec),
+                threadpoolOptions(spec, "rest-api"),
+                threadpoolOptions(spec, "http-client-api"),
+                deployState.getProperties().feedCoreThreadPoolSizeFactor());
+    }
+
+    private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) {
+        Element element = XML.getChild(spec, elementName);
+        if (element == null) return null;
+        return ContainerThreadpool.UserOptions.fromXml(element).orElse(null);
     }
 
     private static List<String> getBindings(Element spec) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java
index 81681f6bc52..81ab2cc1503 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java
@@ -24,11 +24,14 @@ class SearchHandler extends ProcessingHandler<SearchChains> {
 
     private final ApplicationContainerCluster cluster;
 
-    SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, DeployState deployState) {
+    SearchHandler(ApplicationContainerCluster cluster,
+                  List<BindingPattern> bindings,
+                  ContainerThreadpool.UserOptions threadpoolOptions,
+                  DeployState deployState) {
         super(cluster.getSearchChains(), HANDLER_CLASS);
         this.cluster = cluster;
         bindings.forEach(this::addServerBindings);
-        Threadpool threadpool = new Threadpool(cluster, deployState);
+        Threadpool threadpool = new Threadpool(cluster, threadpoolOptions, deployState);
         inject(threadpool);
         addComponent(threadpool);
     }
@@ -37,8 +40,8 @@ class SearchHandler extends ProcessingHandler<SearchChains> {
         private final ApplicationContainerCluster cluster;
         private final DeployState deployState;
 
-        Threadpool(ApplicationContainerCluster cluster, DeployState deployState) {
-            super("search-handler");
+        Threadpool(ApplicationContainerCluster cluster, UserOptions options, DeployState deployState) {
+            super("search-handler", options);
             this.cluster = cluster;
             this.deployState = deployState;
         }
@@ -50,6 +53,9 @@ class SearchHandler extends ProcessingHandler<SearchChains> {
             builder.maxThreadExecutionTimeSeconds(190);
             builder.keepAliveTime(5.0);
 
+            // User options overrides below configuration
+            if (hasUserOptions()) return;
+
             double threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor();
             double vcpu = vcpu(cluster);
             if (threadPoolSizeFactor <= 0 || vcpu == 0) {
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 3c8b60fb84b..98ea696ceef 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -105,6 +105,12 @@ SslProvider = element ssl-provider {
     BundleSpec
 }
 
+Threadpool = element threadpool {
+    element max-threads { xsd:nonNegativeInteger } &
+    element min-threads { xsd:nonNegativeInteger } &
+    element queue-size { xsd:nonNegativeInteger }
+}
+
 # REST-API:
 
 RestApi = element rest-api {
@@ -142,7 +148,8 @@ SearchInContainer = element search {
     SearchChain* &
     Provider* &
     Renderer* &
-    GenericConfig*
+    GenericConfig* &
+    Threadpool?
 }
 
 SearchChain = element chain {
@@ -207,10 +214,18 @@ DocumentApi = element document-api {
    element retrydelay { xsd:double { minInclusive = "0.0" } }? &
    element timeout { xsd:double { minInclusive = "0.0" } }? &
    element tracelevel { xsd:positiveInteger }? &
-   element mbusport { xsd:positiveInteger }?
+   element mbusport { xsd:positiveInteger }? &
+   DocumentRestApi? &
+   HttpClientApi?
 }
 
+DocumentRestApi = element rest-api {
+    Threadpool?
+}
 
+HttpClientApi = element http-client-api {
+    Threadpool?
+}
 
 # NODES:
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
index 93117821c5a..def5da3a9c2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
@@ -121,6 +121,43 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa
         assertEquals(8, config.minThreads());
     }
 
+    @Test
+    public void threadpools_configuration_can_be_overridden() {
+        Element elem = DomBuilderTest.parse(
+                "<container id='cluster1' version='1.0'>",
+                "  <document-api>",
+                "    <rest-api>",
+                "      <threadpool>",
+                "        <max-threads>20</max-threads>",
+                "        <min-threads>10</min-threads>",
+                "        <queue-size>0</queue-size>",
+                "      </threadpool>",
+                "    </rest-api>",
+                "    <http-client-api>",
+                "      <threadpool>",
+                "        <max-threads>50</max-threads>",
+                "        <min-threads>25</min-threads>",
+                "        <queue-size>1000</queue-size>",
+                "      </threadpool>",
+                "    </http-client-api>",
+                "  </document-api>",
+                nodesXml,
+                "</container>");
+        createModel(root, elem);
+
+        ContainerThreadpoolConfig restApiThreadpoolConfig = root.getConfig(
+                ContainerThreadpoolConfig.class, "cluster1/component/com.yahoo.document.restapi.resource.RestApi/threadpool@restapi-handler");
+        assertEquals(20, restApiThreadpoolConfig.maxThreads());
+        assertEquals(10, restApiThreadpoolConfig.minThreads());
+        assertEquals(0, restApiThreadpoolConfig.queueSize());
+
+        ContainerThreadpoolConfig feedThreadpoolConfig = root.getConfig(
+                ContainerThreadpoolConfig.class, "cluster1/component/com.yahoo.vespa.http.server.FeedHandler/threadpool@feedapi-handler");
+        assertEquals(50, feedThreadpoolConfig.maxThreads());
+        assertEquals(25, feedThreadpoolConfig.minThreads());
+        assertEquals(1000, feedThreadpoolConfig.queueSize());
+    }
+
     private static class HostProvisionerWithCustomRealResource implements HostProvisioner {
 
         @Override
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
index 180f21551e2..4c1fda44038 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
@@ -247,6 +247,27 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
         assertEquals(0, config.queueSize());
     }
 
+    @Test
+    public void threadpool_configuration_can_be_overridden() {
+        Element clusterElem = DomBuilderTest.parse(
+                "<container id='default' version='1.0'>",
+                "  <search>",
+                "    <threadpool>",
+                "      <max-threads>100</max-threads>",
+                "      <min-threads>80</min-threads>",
+                "      <queue-size>10</queue-size>",
+                "    </threadpool>",
+                "  </search>",
+                nodesXml,
+                "</container>");
+        createModel(root, clusterElem);
+        ContainerThreadpoolConfig config = root.getConfig(
+                ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASS + "/threadpool@search-handler");
+        assertEquals(100, config.maxThreads());
+        assertEquals(80, config.minThreads());
+        assertEquals(10, config.queueSize());
+    }
+
     private VespaModel getVespaModelWithMusic(String hosts, String services) {
         return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create();
     }
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index e7ea2683e3f..683e2dc0b0d 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -158,6 +158,20 @@
       <timeout>5.55</timeout>
       <route>default</route>
       <maxpendingdocs>100</maxpendingdocs>
+      <rest-api>
+        <threadpool>
+          <max-threads>50</max-threads>
+          <min-threads>10</min-threads>
+          <queue-size>1000</queue-size>
+        </threadpool>
+      </rest-api>
+      <http-client-api>
+        <threadpool>
+          <max-threads>50</max-threads>
+          <min-threads>10</min-threads>
+          <queue-size>1000</queue-size>
+        </threadpool>
+      </http-client-api>
     </document-api>
 
     <search>
@@ -184,6 +198,12 @@
       </chain>
 
       <chain id="achain" searchers="asearcher anothersearcher" inherits="wonkaparentchain" excludes="notneededsearcher"/>
+
+      <threadpool>
+        <max-threads>500</max-threads>
+        <min-threads>500</min-threads>
+        <queue-size>0</queue-size>
+      </threadpool>
     </search>
 
     <processing>