diff options
author | Tor Brede Vekterli <vekterli@oath.com> | 2017-12-13 16:16:02 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-12-13 16:16:02 +0100 |
commit | 5b8b37a42585f25eafd293ad27f68f03ec1a31f1 (patch) | |
tree | 2841b59fa5c9f9bd24302ab906cbb90b50f42c04 | |
parent | d831c170de2293c131e55b1c851abb3713c9c0f7 (diff) | |
parent | 2d5b13aa37bfeddfbc0b4fe46c2ef438c711b444 (diff) |
Merge pull request #4439 from vespa-engine/geirst/produce-bucketspaces-config-for-entire-content-cluster
Geirst/produce bucketspaces config for entire content cluster
7 files changed, 120 insertions, 117 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 81aca977400..0405f96cd89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.document.select.DocumentSelector; @@ -16,15 +14,13 @@ import org.w3c.dom.Element; import java.util.logging.Logger; - /** * Generates distributor-specific configuration. */ public class DistributorCluster extends AbstractConfigProducer<Distributor> implements StorDistributormanagerConfig.Producer, StorServerConfig.Producer, - MetricsmanagerConfig.Producer, - BucketspacesConfig.Producer { + MetricsmanagerConfig.Producer { public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString()); @@ -151,20 +147,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.is_distributor(true); } - private static final String DEFAULT_BUCKET_SPACE = "default"; - private static final String GLOBAL_BUCKET_SPACE = "global"; - - @Override - public void getConfig(BucketspacesConfig.Builder builder) { - for (NewDocumentType docType : parent.getDocumentDefinitions().values()) { - BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); - docTypeBuilder.name(docType.getName()); - String bucketSpace = (parent.isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); - docTypeBuilder.bucketspace(bucketSpace); - builder.documenttype(docTypeBuilder); - } - } - public String getClusterName() { return parent.getName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7889b857fff..7654fbc217b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -7,12 +7,11 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; @@ -59,7 +58,8 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri StorDistributormanagerConfig.Producer, FleetcontrollerConfig.Producer, MetricsmanagerConfig.Producer, - MessagetyperouteselectorpolicyConfig.Producer { + MessagetyperouteselectorpolicyConfig.Producer, + BucketspacesConfig.Producer { // TODO: Make private private String documentSelection; @@ -694,4 +694,18 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } } + + private static final String DEFAULT_BUCKET_SPACE = "default"; + private static final String GLOBAL_BUCKET_SPACE = "global"; + + @Override + public void getConfig(BucketspacesConfig.Builder builder) { + for (NewDocumentType docType : getDocumentDefinitions().values()) { + BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); + docTypeBuilder.name(docType.getName()); + String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + docTypeBuilder.bucketspace(bucketSpace); + builder.documenttype(docTypeBuilder); + } + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 5f18b28d6ce..41bba055a50 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; import org.junit.Test; @@ -15,6 +17,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; /** * Unit tests for content search cluster. @@ -36,8 +39,8 @@ public class ContentSearchClusterTest { private static ContentCluster createClusterWithGlobalType() throws Exception { return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( - new ContentClusterBuilder.DocType("global", true), - new ContentClusterBuilder.DocType("regular"))).getXml(), + DocType.indexGlobal("global"), + DocType.index("regular"))).getXml(), createSearchDefinitions("global", "regular")); } @@ -108,10 +111,31 @@ public class ContentSearchClusterTest { .content("field ref_to_c type reference<c> { indexing: attribute }").build()); searchDefinitions.add(new SearchDefinitionBuilder().name("c").build()); return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( - new ContentClusterBuilder.DocType("a"), - new ContentClusterBuilder.DocType("b", true), - new ContentClusterBuilder.DocType("c", true))).getXml(), + DocType.index("a"), + DocType.indexGlobal("b"), + DocType.indexGlobal("c"))).getXml(), searchDefinitions); } + private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) { + BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder(); + cluster.getConfig(builder); + return new BucketspacesConfig(builder); + } + + private static void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) { + assertEquals(expName, docType.name()); + assertEquals(expBucketSpace, docType.bucketspace()); + } + + @Test + public void require_that_bucket_spaces_config_is_produced_for_content_cluster() throws Exception { + BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType()); + assertEquals(2, config.documenttype().size()); + assertDocumentType("global", "global", config.documenttype(0)); + assertDocumentType("regular", "default", config.documenttype(1)); + // Safeguard against flipping the switch + assertFalse(config.enable_multiple_bucket_spaces()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index d4e804d3f95..48b7ccdad6b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -1,23 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import org.junit.Test; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.hamcrest.Matchers.*; + /** * Test for content DistributorCluster. */ @@ -303,100 +302,48 @@ public class DistributorTest { return new StorDistributormanagerConfig(builder); } - private static class DocDef { - public final String type; - public final String mode; - public final boolean global; - - private DocDef(String type, String mode, boolean global) { - this.type = type; - this.mode = mode; - this.global = global; - } - - public static DocDef storeOnly(String type) { - return new DocDef(type, "store-only", false); - } - - public static DocDef index(String type) { - return new DocDef(type, "index", false); - } - - public static DocDef indexGlobal(String type) { - return new DocDef(type, "index", true); - } - - public static DocDef streaming(String type) { - return new DocDef(type, "streaming", false); - } - } - - private String generateXmlForDocDefs(DocDef... defs) { + private String generateXmlForDocTypes(DocType... docTypes) { return "<content id='storage'>\n" + - " <documents>\n" + - Arrays.stream(defs) - .map(def -> String.format(" <document type='%s' mode='%s' global='%s'/>", - def.type, def.mode, (def.global ? "true" : "false"))) - .collect(Collectors.joining("\n")) + - "\n </documents>\n" + - "</content>"; + DocType.listToXml(docTypes) + + "\n</content>"; } @Test public void bucket_activation_disabled_if_no_documents_in_indexed_mode() { StorDistributormanagerConfig config = clusterXmlToConfig( - generateXmlForDocDefs(DocDef.storeOnly("music"))); + generateXmlForDocTypes(DocType.storeOnly("music"))); assertThat(config.disable_bucket_activation(), is(true)); } @Test public void bucket_activation_enabled_with_single_indexed_document() { StorDistributormanagerConfig config = clusterXmlToConfig( - generateXmlForDocDefs(DocDef.index("music"))); + generateXmlForDocTypes(DocType.index("music"))); assertThat(config.disable_bucket_activation(), is(false)); } @Test public void bucket_activation_enabled_with_multiple_indexed_documents() { StorDistributormanagerConfig config = clusterXmlToConfig( - generateXmlForDocDefs(DocDef.index("music"), - DocDef.index("movies"))); + generateXmlForDocTypes(DocType.index("music"), + DocType.index("movies"))); assertThat(config.disable_bucket_activation(), is(false)); } @Test public void bucket_activation_enabled_if_at_least_one_document_indexed() { StorDistributormanagerConfig config = clusterXmlToConfig( - generateXmlForDocDefs(DocDef.storeOnly("music"), - DocDef.streaming("bunnies"), - DocDef.index("movies"))); + generateXmlForDocTypes(DocType.storeOnly("music"), + DocType.streaming("bunnies"), + DocType.index("movies"))); assertThat(config.disable_bucket_activation(), is(false)); } @Test public void bucket_activation_disabled_for_single_streaming_type() { StorDistributormanagerConfig config = clusterXmlToConfig( - generateXmlForDocDefs(DocDef.streaming("music"))); + generateXmlForDocTypes(DocType.streaming("music"))); assertThat(config.disable_bucket_activation(), is(true)); } - private BucketspacesConfig clusterXmlToBucketspacesConfig(String xml) { - BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder(); - parse(xml).getConfig(builder); - return new BucketspacesConfig(builder); - } - - private void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) { - assertEquals(expName, docType.name()); - assertEquals(expBucketSpace, docType.bucketspace()); - } - - @Test - public void bucket_spaces_config_is_produced_for_distributor_cluster() { - BucketspacesConfig config = clusterXmlToBucketspacesConfig( - generateXmlForDocDefs(DocDef.index("music"), DocDef.indexGlobal("movies"))); - assertEquals(2, config.documenttype().size()); - assertDocumentType("movies", "global", config.documenttype(0)); - assertDocumentType("music", "default", config.documenttype(1)); - } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java index 592e90efd22..95c57bb544c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -18,29 +18,10 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; */ public class ContentClusterBuilder { - public static class DocType { - private final String name; - private final boolean global; - - public DocType(String name, boolean global) { - this.name = name; - this.global = global; - } - - public DocType(String name) { - this(name, false); - } - - public String toXml() { - return (global ? "<document mode='index' type='" + name + "' global='true'/>" : - "<document mode='index' type='" + name + "'/>"); - } - } - private String name = "mycluster"; private int redundancy = 1; private int searchableCopies = 1; - private List<DocType> docTypes = Arrays.asList(new DocType("test")); + private List<DocType> docTypes = Arrays.asList(DocType.index("test")); private String groupXml = getSimpleGroupXml(); private Optional<String> dispatchXml = Optional.empty(); private Optional<Double> protonDiskLimit = Optional.empty(); @@ -66,7 +47,7 @@ public class ContentClusterBuilder { public ContentClusterBuilder docTypes(String ... docTypes) { this.docTypes = Arrays.asList(docTypes).stream(). - map(type -> new DocType(type)). + map(type -> DocType.index(type)). collect(Collectors.toList()); return this; } @@ -103,9 +84,7 @@ public class ContentClusterBuilder { public String getXml() { String xml = joinLines("<content version='1.0' id='" + name + "'>", " <redundancy>" + redundancy + "</redundancy>", - " <documents>", - docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")), - " </documents>", + DocType.listToXml(docTypes), " <engine>", " <proton>", " <searchable-copies>" + searchableCopies + "</searchable-copies>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java new file mode 100644 index 00000000000..3a5f679509b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.model.content.utils; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Definition of a document type used for testing. + * + * @author geirst + */ +public class DocType { + private final String type; + private final String mode; + private final boolean global; + + private DocType(String type, String mode, boolean global) { + this.type = type; + this.mode = mode; + this.global = global; + } + + public String toXml() { + return (global ? "<document mode='" + mode + "' type='" + type + "' global='true'/>" : + "<document mode='" + mode + "' type='" + type + "'/>"); + } + + public static DocType storeOnly(String type) { + return new DocType(type, "store-only", false); + } + + public static DocType index(String type) { + return new DocType(type, "index", false); + } + + public static DocType indexGlobal(String type) { + return new DocType(type, "index", true); + } + + public static DocType streaming(String type) { + return new DocType(type, "streaming", false); + } + + public static String listToXml(DocType... docTypes) { + return listToXml(Arrays.asList(docTypes)); + } + + public static String listToXml(List<DocType> docTypes) { + return "<documents>\n" + + docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")) + "\n" + + "</documents>"; + } + +} diff --git a/storage/src/vespa/storage/config/bucketspaces.def b/storage/src/vespa/storage/config/bucketspaces.def index 3ed1abba0b4..4db107ec1ee 100644 --- a/storage/src/vespa/storage/config/bucketspaces.def +++ b/storage/src/vespa/storage/config/bucketspaces.def @@ -9,3 +9,6 @@ documenttype[].name string ## The bucket space this document type belongs to. documenttype[].bucketspace string + +## Switch to enable multiple bucket spaces in content layer and content nodes. +enable_multiple_bucket_spaces bool default=false |