diff options
author | Jon Bratseth <bratseth@oath.com> | 2019-02-13 18:31:20 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-02-13 18:31:20 +0100 |
commit | d33923da995c09bc4716e9e678dc7615ed534051 (patch) | |
tree | 95cdb1bd796dcddfabae571e989a197315564674 | |
parent | acb90e7599dccdcd0d5b0969adba4431a5cc82c9 (diff) | |
parent | 1cdea7e5d75ae804c856f188026d6f75eb64d591 (diff) |
Merge pull request #8494 from vespa-engine/balder/automatic-thread-config-if-flavor-is-known
Use number of cores to make a good assumption on number of threads to…
7 files changed, 84 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 0af11fdb941..b7035594c71 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -2,7 +2,12 @@ package com.yahoo.config.model.provision; import com.yahoo.config.model.api.HostProvisioner; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.net.HostName; import java.util.ArrayList; @@ -25,6 +30,10 @@ public class SingleNodeProvisioner implements HostProvisioner { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } + public SingleNodeProvisioner(Flavor flavor) { + host = new Host(HostName.getLocalhost()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases(),flavor); + } @Override public HostSpec allocateHost(String alias) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index baf673570e4..13d4064f55f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; @@ -57,6 +58,10 @@ public class MockRoot extends AbstractConfigProducerRoot { public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) { this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build()); } + public MockRoot(String rootConfigId, ApplicationPackage applicationPackage, HostProvisioner provisioner) { + this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage) + .modelHostProvisioner(provisioner).build()); + } public MockRoot(String rootConfigId, DeployState deployState) { super(rootConfigId); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 4257e47218e..c75421c9636 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.provision.Flavor; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.content.core.StorStatusConfig; @@ -71,8 +73,4 @@ public abstract class ContentNode extends AbstractService public int getHealthPort() { return getRelativePort(2); } - - public String getRootDirectory() { - return rootDirectory; - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index 537eca5d7b8..9ea92048f3b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -19,12 +20,13 @@ import org.w3c.dom.Element; * Class to provide config related to a specific storage node. */ @RestartConfigs({StorFilestorConfig.class, StorBucketmoverConfig.class}) -public class StorageNode extends ContentNode implements StorServerConfig.Producer { +public class StorageNode extends ContentNode implements StorServerConfig.Producer, StorFilestorConfig.Producer { static final String rootFolder = Defaults.getDefaults().underVespaHome("var/db/vespa/search/"); private final Double capacity; private final boolean retired; + private final StorageCluster cluster; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageNode> { @Override @@ -41,6 +43,7 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce distributionKey); this.retired = retired; this.capacity = capacity; + this.cluster = cluster; } @Override @@ -81,4 +84,13 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce } } + @Override + public void getConfig(StorFilestorConfig.Builder builder) { + if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { + Flavor nodeFlavor = getHostResource().getFlavor().get(); + builder.num_threads(Math.max(4, (int)nodeFlavor.getMinCpuCores())); + } + cluster.getConfig(builder); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 4f7440e590b..27f5af52b6c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -1,6 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; import com.yahoo.vespa.config.content.core.StorVisitorConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; @@ -20,8 +24,16 @@ import static org.junit.Assert.fail; public class StorageClusterTest { + StorageCluster parse(String xml, Flavor flavor) throws Exception { + MockRoot root = new MockRoot("", new MockApplicationPackage.Builder().build(), + new SingleNodeProvisioner(flavor)); + return parse(xml, root); + } StorageCluster parse(String xml) throws Exception { MockRoot root = new MockRoot(); + return parse(xml, root); + } + StorageCluster parse(String xml, MockRoot root) throws Exception { root.getDeployState().getDocumentModel().getDocumentManager().add( new NewDocumentType(new NewDocumentType.Name("music")) ); @@ -95,8 +107,8 @@ public class StorageClusterTest { @Test public void testPersistenceThreads() throws Exception { - StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); - parse( + + StorageCluster stc = parse( "<cluster id=\"bees\">\n" + " <documents/>" + " <tuning>\n" + @@ -109,19 +121,31 @@ public class StorageClusterTest { " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + - "</cluster>" - ).getConfig(builder); + "</cluster>", + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) + ); - StorFilestorConfig config = new StorFilestorConfig(builder); + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + stc.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(4, config.num_threads()); - assertEquals(false, config.enable_multibit_split_optimalization()); + assertEquals(4, config.num_threads()); + assertEquals(false, config.enable_multibit_split_optimalization()); + } + { + assertEquals(1, stc.getChildren().size()); + StorageNode sn = stc.getChildren().values().iterator().next(); + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + sn.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(4, config.num_threads()); + } } @Test public void testNoPersistenceThreads() throws Exception { - StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); - parse( + StorageCluster stc = parse( "<cluster id=\"bees\">\n" + " <documents/>" + " <tuning>\n" + @@ -129,12 +153,24 @@ public class StorageClusterTest { " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + - "</cluster>" - ).getConfig(builder); - - StorFilestorConfig config = new StorFilestorConfig(builder); + "</cluster>", + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) + ); - assertEquals(6, config.num_threads()); + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + stc.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(6, config.num_threads()); + } + { + assertEquals(1, stc.getChildren().size()); + StorageNode sn = stc.getChildren().values().iterator().next(); + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + sn.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(9, config.num_threads()); + } } @Test diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 849d92696ff..852e9a4af01 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -458,6 +458,7 @@ "public void <init>(java.lang.String, java.util.Optional)", "public void <init>(java.lang.String, com.yahoo.config.provision.ClusterMembership, com.yahoo.config.provision.Flavor, java.util.Optional)", "public void <init>(java.lang.String, java.util.List)", + "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.Flavor)", "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.ClusterMembership)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index f438b8ee76f..f0e8774759d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -40,6 +40,9 @@ public class HostSpec implements Comparable<HostSpec> { public HostSpec(String hostname, List<String> aliases) { this(hostname, aliases, Optional.empty(), Optional.empty()); } + public HostSpec(String hostname, List<String> aliases, Flavor flavor) { + this(hostname, aliases, Optional.of(flavor), Optional.empty()); + } public HostSpec(String hostname, List<String> aliases, ClusterMembership membership) { this(hostname, aliases, Optional.empty(), Optional.of(membership)); |