summaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorBjørn Christian Seime <bjorncs@verizonmedia.com>2020-08-27 13:32:21 +0200
committerGitHub <noreply@github.com>2020-08-27 13:32:21 +0200
commit2c54ef0b9ac013d0cb1e7d5394312fd575cce9f2 (patch)
treeec7afb86e3d603e0c5d3a1f0239c873a79e15d23 /config-model
parenta183d1b84ed8752dcf8f9486f7e9a36982db99b1 (diff)
parent1929d3fa248548d031dffe5c4cfccfd9cfac2bda (diff)
Merge pull request #13990 from vespa-engine/balder/container-thread-pool-by-bjorncs
Revert "Revert "Bjorncs/container thread pool""
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java84
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java61
6 files changed, 154 insertions, 25 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
index 159a87be27d..0624028732f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
@@ -5,35 +5,79 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ThreadPoolExecutorComponent;
import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.component.SystemBindingPattern;
import com.yahoo.vespa.model.container.component.UserBindingPattern;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
/**
* @author Einar M R Rosenvinge
+ * @author bjorncs
*/
public class ContainerDocumentApi {
- private static final String vespaClientBundleSpecification = "vespaclient-container-plugin";
+ private static final int FALLBACK_MAX_POOL_SIZE = 0; // Use fallback based on actual logical core count on host
+ private static final int FALLBACK_CORE_POOL_SIZE = 0; // Use fallback based on actual logical core count on host
+
+ private final ContainerCluster<?> cluster;
private final Options options;
+ private final Handler<AbstractConfigProducer<?>> feedHandler;
+ private final Handler<AbstractConfigProducer<?>> restApiHandler;
- public ContainerDocumentApi(ContainerCluster cluster, Options options) {
+ public ContainerDocumentApi(ContainerCluster<?> cluster, Options options) {
+ this.cluster = cluster;
this.options = options;
- setupHandlers(cluster);
+ this.restApiHandler = addRestApiHandler(cluster, options);
+ this.feedHandler = addFeedHandler(cluster, options);
}
- private void setupHandlers(ContainerCluster cluster) {
- cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "/document/v1/*"));
- cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"));
+
+ public void addNodesDependentThreadpoolConfiguration() {
+ if (cluster.getContainers().isEmpty()) throw new IllegalStateException("Cluster is empty");
+ ThreadPoolExecutorComponent feedHandlerExecutor = newExecutorComponent("feedapi-handler", cluster, options);
+ feedHandler.inject(feedHandlerExecutor);
+ feedHandler.addComponent(feedHandlerExecutor);
+ ThreadPoolExecutorComponent restApiHandlerExecutor = newExecutorComponent("restapi-handler", cluster, options);
+ restApiHandler.inject(restApiHandlerExecutor);
+ restApiHandler.addComponent(restApiHandlerExecutor);
}
- private Handler<AbstractConfigProducer<?>> newVespaClientHandler(String componentId, String bindingSuffix) {
- Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel(
- BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), ""));
+ private static Handler<AbstractConfigProducer<?>> addFeedHandler(ContainerCluster<?> cluster, Options options) {
+ String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi";
+ var handler = newVespaClientHandler(
+ "com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, options);
+ cluster.addComponent(handler);
+ return handler;
+ }
+
+
+ private static Handler<AbstractConfigProducer<?>> addRestApiHandler(ContainerCluster<?> cluster, Options options) {
+ var handler = newVespaClientHandler(
+ "com.yahoo.document.restapi.resource.RestApi", "/document/v1/*", options);
+ cluster.addComponent(handler);
+ return handler;
+ }
+ private static ThreadPoolExecutorComponent newExecutorComponent(String name, ContainerCluster<?> cluster, Options options) {
+ int maxPoolSize = maxPoolSize(cluster);
+ return new ThreadPoolExecutorComponent.Builder(name)
+ .maxPoolSize(maxPoolSize)
+ .corePoolSize(corePoolSize(maxPoolSize, options))
+ .queueSize(500)
+ .build();
+ }
+
+ private static Handler<AbstractConfigProducer<?>> newVespaClientHandler(
+ String componentId,
+ String bindingSuffix,
+ Options options) {
+ Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel(
+ BundleInstantiationSpecification.getFromStrings(componentId, null, "vespaclient-container-plugin"), ""));
if (options.bindings.isEmpty()) {
handler.addServerBindings(
SystemBindingPattern.fromHttpPath(bindingSuffix),
@@ -45,17 +89,33 @@ public class ContainerDocumentApi {
UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash),
UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash + '/'));
}
-
}
return handler;
}
+ private static int maxPoolSize(ContainerCluster<?> cluster) {
+ List<Double> vcpus = cluster.getContainers().stream()
+ .filter(c -> c.getHostResource() != null && c.getHostResource().realResources() != null)
+ .map(c -> c.getHostResource().realResources().vcpu())
+ .distinct()
+ .collect(Collectors.toList());
+ // We can only use host resource for calculation if all container nodes in the cluster are homogeneous (in terms of vcpu)
+ if (vcpus.size() != 1 || vcpus.get(0) == 0) return FALLBACK_MAX_POOL_SIZE;
+ return (int)Math.ceil(vcpus.get(0));
+ }
+
+ private static int corePoolSize(int maxPoolSize, Options options) {
+ if (maxPoolSize == FALLBACK_MAX_POOL_SIZE) return FALLBACK_CORE_POOL_SIZE;
+ return (int) Math.ceil(options.feedCoreThreadPoolSizeFactor * maxPoolSize);
+ }
+
public static final class Options {
private final Collection<String> bindings;
+ private final double feedCoreThreadPoolSizeFactor;
-
- public Options(Collection<String> bindings) {
+ public Options(Collection<String> bindings, double feedCoreThreadPoolSizeFactor) {
this.bindings = Collections.unmodifiableCollection(bindings);
+ this.feedCoreThreadPoolSizeFactor = feedCoreThreadPoolSizeFactor;
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 8bb456ab7e7..6d8f3056cef 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -17,6 +17,7 @@ import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.container.core.ApplicationMetadataConfig;
import com.yahoo.container.core.document.ContainerDocumentConfig;
+import com.yahoo.container.handler.ThreadPoolProvider;
import com.yahoo.container.di.config.PlatformBundlesConfig;
import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.container.jdisc.JdiscBindingsConfig;
@@ -175,7 +176,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
addComponent(new StatisticsComponent());
addSimpleComponent(AccessLog.class);
- addComponent(new ThreadPoolExecutorComponent.Builder("default-pool").build());
+ addSimpleComponent(ThreadPoolProvider.class);
addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class);
addSimpleComponent(SecurityFilterInvoker.class);
addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java
index 2926cb3ee6c..aac73ce7636 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java
@@ -2,15 +2,15 @@
package com.yahoo.vespa.model.container;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
-import com.yahoo.container.handler.ThreadPoolProvider;
import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.container.handler.threadpool.ContainerThreadPool;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.model.container.component.SimpleComponent;
import java.time.Duration;
/**
- * Component definition for a {@link java.util.concurrent.Executor} using {@link ThreadPoolProvider}.
+ * Component definition for a {@link java.util.concurrent.Executor} using {@link ContainerThreadPool}.
*
* @author bjorncs
*/
@@ -26,8 +26,8 @@ public class ThreadPoolExecutorComponent extends SimpleComponent implements Thre
private ThreadPoolExecutorComponent(Builder builder) {
super(new ComponentModel(
BundleInstantiationSpecification.getFromStrings(
- "threadpool-provider@" + builder.name,
- ThreadPoolProvider.class.getName(),
+ "threadpool@" + builder.name,
+ ContainerThreadPool.class.getName(),
null)));
this.name = builder.name;
this.maxPoolSize = builder.maxPoolSize;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 51583588201..e0130e9814c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -182,7 +182,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
addProcessing(deployState, spec, cluster);
addSearch(deployState, spec, cluster);
addDocproc(deployState, spec, cluster);
- addDocumentApi(spec, cluster); // NOTE: Must be done after addSearch
+ addDocumentApi(deployState, spec, cluster); // NOTE: Must be done after addSearch
cluster.addDefaultHandlersExceptStatus();
addStatusHandlers(cluster, context.getDeployState().isHosted());
@@ -197,6 +197,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
addClientProviders(deployState, spec, cluster);
addServerProviders(deployState, spec, cluster);
+ addHandlerSpecificThreadpools(cluster);
addAthensCopperArgos(cluster, context); // Must be added after nodes.
}
@@ -211,6 +212,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
}
}
+ private void addHandlerSpecificThreadpools(ContainerCluster<?> cluster) {
+ ContainerDocumentApi documentApi = cluster.getDocumentApi();
+ if (documentApi != null) {
+ documentApi.addNodesDependentThreadpoolConfiguration();
+ }
+ }
+
private void addAthensCopperArgos(ApplicationContainerCluster cluster, ConfigModelContext context) {
if ( ! context.getDeployState().isHosted()) return;
app.getDeployment().map(DeploymentSpec::fromXml)
@@ -398,8 +406,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
cluster.addServlet(new ServletBuilder().build(deployState, cluster, servletElem));
}
- private void addDocumentApi(Element spec, ApplicationContainerCluster cluster) {
- ContainerDocumentApi containerDocumentApi = buildDocumentApi(cluster, spec);
+ private void addDocumentApi(DeployState deployState, Element spec, ApplicationContainerCluster cluster) {
+ ContainerDocumentApi containerDocumentApi = buildDocumentApi(deployState, cluster, spec);
if (containerDocumentApi == null) return;
cluster.setDocumentApi(containerDocumentApi);
@@ -828,11 +836,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
return result.toArray(BindingPattern[]::new);
}
- private ContainerDocumentApi buildDocumentApi(ApplicationContainerCluster cluster, Element spec) {
+ private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec) {
Element documentApiElement = XML.getChild(spec, "document-api");
if (documentApiElement == null) return null;
- ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement);
+ ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(deployState, documentApiElement);
return new ContainerDocumentApi(cluster, documentApiOptions);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
index 61464799812..34de21de404 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.xml;
+import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.text.XML;
import com.yahoo.vespa.model.clients.ContainerDocumentApi;
import org.w3c.dom.Element;
@@ -18,8 +19,8 @@ public class DocumentApiOptionsBuilder {
private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName());
- public static ContainerDocumentApi.Options build(Element spec) {
- return new ContainerDocumentApi.Options(getBindings(spec));
+ public static ContainerDocumentApi.Options build(DeployState deployState, Element spec) {
+ return new ContainerDocumentApi.Options(getBindings(spec), deployState.getProperties().feedCoreThreadPoolSizeFactor());
}
private static List<String> getBindings(Element spec) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
index 73a68429b6d..37e5dc21346 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java
@@ -1,7 +1,19 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.xml;
+import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
+import com.yahoo.config.model.provision.Host;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.model.test.MockRoot;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.net.HostName;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.component.SystemBindingPattern;
@@ -11,22 +23,27 @@ import org.w3c.dom.Element;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
/**
* @author Einar M R Rosenvinge
+ * @author bjorncs
*/
public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBase {
private Map<String, Handler<?>> getHandlers(String clusterName) {
- ContainerCluster cluster = (ContainerCluster) root.getChildren().get(clusterName);
+ ContainerCluster<?> cluster = (ContainerCluster<?>) root.getChildren().get(clusterName);
Map<String, Handler<?>> handlerMap = new HashMap<>();
Collection<Handler<?>> handlers = cluster.getHandlers();
for (Handler<?> handler : handlers) {
@@ -83,4 +100,46 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa
is(true));
assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2));
}
+
+ @Test
+ public void feeding_api_have_separate_threadpools() {
+ Element elem = DomBuilderTest.parse(
+ "<container id='cluster1' version='1.0'>",
+ " <document-api />",
+ nodesXml,
+ "</container>");
+ root = new MockRoot("root", new MockApplicationPackage.Builder().build(), new HostProvisionerWithCustomRealResource());
+ createModel(root, elem);
+ Map<String, Handler<?>> handlers = getHandlers("cluster1");
+ Handler<?> feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler");
+ Set<String> injectedComponentIds = feedApiHandler.getInjectedComponentIds();
+ assertThat(injectedComponentIds, hasItem("threadpool@feedapi-handler"));
+
+ ThreadpoolConfig config = root.getConfig(ThreadpoolConfig.class, "cluster1/component/com.yahoo.vespa.http.server.FeedHandler/threadpool@feedapi-handler");
+ assertEquals(4, config.maxthreads());
+ assertEquals(4, config.corePoolSize());
+ }
+
+ private static class HostProvisionerWithCustomRealResource implements HostProvisioner {
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ Host host = new Host(HostName.getLocalhost());
+ ClusterMembership membership = ClusterMembership.from(
+ ClusterSpec
+ .specification(
+ ClusterSpec.Type.container,
+ ClusterSpec.Id.from("id"))
+ .vespaVersion("")
+ .group(ClusterSpec.Group.from(0))
+ .build(),
+ 0);
+ return new HostSpec(
+ host.hostname(), new NodeResources(4, 0, 0, 0), NodeResources.unspecified(), NodeResources.unspecified(),
+ membership, Optional.empty(), Optional.empty(), Optional.empty());
+ }
+
+ @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(); }
+ }
+
}