diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-07-03 17:23:47 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-07-03 17:23:47 +0200 |
commit | 9b7a9a8b1dbc828e632cf9f812db271c4a132320 (patch) | |
tree | b077d3d6a8623d1aef8bb6a8632a484a8f387aa1 | |
parent | df3d9cfed3dea0011fecdd6a7c7f5161a0e91f0e (diff) | |
parent | 275dfef0f2c9b9f2acfbefeb759c34d76c973e43 (diff) |
Merge pull request #13799 from vespa-engine/balder/only-use-encode-help-on-distributor
Only the distributor need help encoding. Proton has enough workers.
4 files changed, 70 insertions, 28 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index c55548cfd2b..4b07b78a713 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -50,6 +51,13 @@ public class Distributor extends ContentNode { } @Override + public void getConfig(StorCommunicationmanagerConfig.Builder builder) { + super.getConfig(builder); + // Single distributor needs help to encode the messages. + builder.mbus.dispatch_on_encode(true); + } + + @Override public String getStartupCommand() { return "exec sbin/vespa-distributord -c $VESPA_CONFIG_ID"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index f41188eccde..d40abec5bb1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; @@ -92,4 +93,10 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce cluster.getConfig(builder); } + @Override + public void getConfig(StorCommunicationmanagerConfig.Builder builder) { + super.getConfig(builder); + builder.mbus.dispatch_on_encode(false); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 365dc74274d..8ea846cc790 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -14,8 +14,8 @@ import org.junit.Test; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Test for content DistributorCluster. @@ -48,7 +48,7 @@ public class DistributorTest { getConfig(builder); StorServerConfig config = new StorServerConfig(builder); - assertEquals(true, config.is_distributor()); + assertTrue(config.is_distributor()); assertEquals("foofighters", config.cluster_name()); } @@ -62,7 +62,7 @@ public class DistributorTest { " </group>" + "</cluster>").getConfig(builder); StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); - assertEquals(false, conf.enable_revert()); + assertFalse(conf.enable_revert()); } @Test @@ -85,7 +85,7 @@ public class DistributorTest { assertEquals(26214400, conf.splitsize()); assertEquals(13107200, conf.joinsize()); assertEquals(8, conf.minsplitcount()); - assertEquals(false, conf.inlinebucketsplitting()); + assertFalse(conf.inlinebucketsplitting()); } @Test @@ -110,7 +110,7 @@ public class DistributorTest { assertEquals(33544432, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(8, conf.minsplitcount()); - assertEquals(true, conf.inlinebucketsplitting()); + assertTrue(conf.inlinebucketsplitting()); cluster = parseCluster("<cluster id=\"storage\">\n" + " <redundancy>2</redundancy>" + @@ -137,7 +137,7 @@ public class DistributorTest { assertEquals(33544432, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(1, conf.minsplitcount()); - assertEquals(true, conf.inlinebucketsplitting()); + assertTrue(conf.inlinebucketsplitting()); } @Test @@ -269,9 +269,29 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); + assertTrue(config.mbus().dispatch_on_encode()); assertEquals(14066, config.rpcport()); } + @Test + public void testCommunicationManagerDefaults() { + StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); + DistributorCluster cluster = + parse("<cluster id=\"storage\">" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>"); + + cluster.getChildren().get("0").getConfig(builder); + StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); + assertTrue(config.mbus().dispatch_on_encode()); + assertFalse(config.mbus().dispatch_on_decode()); + assertEquals(4, config.mbus().num_threads()); + assertEquals(StorCommunicationmanagerConfig.Mbus.Optimize_for.LATENCY, config.mbus().optimize_for()); + } + private StorDistributormanagerConfig clusterXmlToConfig(String xml) { StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); parse(xml).getConfig(builder); @@ -288,14 +308,14 @@ public class DistributorTest { public void bucket_activation_disabled_if_no_documents_in_indexed_mode() { StorDistributormanagerConfig config = clusterXmlToConfig( generateXmlForDocTypes(DocType.storeOnly("music"))); - assertThat(config.disable_bucket_activation(), is(true)); + assertTrue(config.disable_bucket_activation()); } @Test public void bucket_activation_enabled_with_single_indexed_document() { StorDistributormanagerConfig config = clusterXmlToConfig( generateXmlForDocTypes(DocType.index("music"))); - assertThat(config.disable_bucket_activation(), is(false)); + assertFalse(config.disable_bucket_activation()); } @Test @@ -303,7 +323,7 @@ public class DistributorTest { StorDistributormanagerConfig config = clusterXmlToConfig( generateXmlForDocTypes(DocType.index("music"), DocType.index("movies"))); - assertThat(config.disable_bucket_activation(), is(false)); + assertFalse(config.disable_bucket_activation()); } @Test @@ -312,14 +332,14 @@ public class DistributorTest { generateXmlForDocTypes(DocType.storeOnly("music"), DocType.streaming("bunnies"), DocType.index("movies"))); - assertThat(config.disable_bucket_activation(), is(false)); + assertFalse(config.disable_bucket_activation()); } @Test public void bucket_activation_disabled_for_single_streaming_type() { StorDistributormanagerConfig config = clusterXmlToConfig( generateXmlForDocTypes(DocType.streaming("music"))); - assertThat(config.disable_bucket_activation(), is(true)); + assertTrue(config.disable_bucket_activation()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 96fe7fc5775..1ac33c4a7e5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; import com.yahoo.vespa.config.content.core.StorVisitorConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; @@ -36,14 +35,6 @@ public class StorageClusterTest { return parse(xml, root); } - StorageCluster parse(String xml, Flavor flavor, ModelContext.Properties properties) { - MockRoot root = new MockRoot("", new DeployState.Builder() - .applicationPackage(new MockApplicationPackage.Builder().build()) - .modelHostProvisioner(new SingleNodeProvisioner(flavor)) - .properties(properties).build()); - return parse(xml, root); - } - StorageCluster parse(String xml) { MockRoot root = new MockRoot(); return parse(xml, root); @@ -63,18 +54,34 @@ public class StorageClusterTest { @Test public void testBasics() { - StorServerConfig.Builder builder = new StorServerConfig.Builder(); - parse("<content id=\"foofighters\"><documents/>\n" + + StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + - "</content>\n"). - getConfig(builder); + "</content>\n"); + assertEquals(1, storage.getChildren().size()); + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + storage.getConfig(builder); StorServerConfig config = new StorServerConfig(builder); - assertEquals(false, config.is_distributor()); + assertFalse(config.is_distributor()); assertEquals("foofighters", config.cluster_name()); } + @Test + public void testCommunicationManagerDefaults() { + StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>\n"); + StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); + storage.getChildren().get("0").getConfig(builder); + StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); + assertFalse(config.mbus().dispatch_on_encode()); + assertFalse(config.mbus().dispatch_on_decode()); + assertEquals(4, config.mbus().num_threads()); + assertEquals(StorCommunicationmanagerConfig.Mbus.Optimize_for.LATENCY, config.mbus().optimize_for()); + } @Test public void testMerges() { @@ -97,7 +104,7 @@ public class StorageClusterTest { } @Test - public void testVisitors() throws Exception { + public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); parse( "<cluster id=\"bees\">\n" + |