diff options
author | Tor Brede Vekterli <vekterli@oath.com> | 2018-12-05 16:00:30 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@oath.com> | 2018-12-05 16:04:48 +0100 |
commit | d29450ded189ee78657b0ae8778ae879df7a3b5f (patch) | |
tree | dc39c8ab36f2b3df5733e443dad0d7f10bcf27d3 /config-model | |
parent | 200663867930d081a49644c42f6090a926a327c3 (diff) |
Add and use cross-cluster bucket space config
Adds a new config `AllClustersBucketSpacesConfig` which includes all
document type to bucket space mappings across all configured content
clusters. Inject this config into `RestApi` to ensure all changes
to the mapping is observed. This also removes the remaining per-request
config fetching during Document V1 visit ops.
Diffstat (limited to 'config-model')
5 files changed, 101 insertions, 2 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index af9eba92a98..7be4b27860c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -19,6 +19,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -212,6 +213,14 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override + public void getConfig(AllClustersBucketSpacesConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + for (ContentCluster cluster : model.getContentClusters().values()) { + builder.cluster(cluster.getName(), cluster.clusterBucketSpaceConfigBuilder()); + } + } + + @Override public void getConfig(ModelConfig.Builder builder) { builder.vespaVersion(vespaVersion.toFullString()); for (HostResource modelHost : getHostSystem().getHosts()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index 2a30a89dd8a..cb30a64e1fc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -12,6 +12,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; /** @@ -29,6 +30,7 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, ZookeepersConfig.Producer, LoadTypeConfig.Producer, ClusterListConfig.Producer, + AllClustersBucketSpacesConfig.Producer, ModelConfig.Producer, ApplicationIdConfig.Producer { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 27da41c2bfa..23c12465121 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.documentmodel.NewDocumentType; @@ -736,13 +737,26 @@ public class ContentCluster extends AbstractConfigProducer implements private static final String DEFAULT_BUCKET_SPACE = "default"; private static final String GLOBAL_BUCKET_SPACE = "global"; + private String bucketSpaceOfDocumentType(NewDocumentType docType) { + return (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + } + + public AllClustersBucketSpacesConfig.Cluster.Builder clusterBucketSpaceConfigBuilder() { + AllClustersBucketSpacesConfig.Cluster.Builder builder = new AllClustersBucketSpacesConfig.Cluster.Builder(); + for (NewDocumentType docType : getDocumentDefinitions().values()) { + AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder typeBuilder = new AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder(); + typeBuilder.bucketSpace(bucketSpaceOfDocumentType(docType)); + builder.documentType(docType.getName(), typeBuilder); + } + return builder; + } + @Override public void getConfig(BucketspacesConfig.Builder builder) { for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); - docTypeBuilder.bucketspace(bucketSpace); + docTypeBuilder.bucketspace(bucketSpaceOfDocumentType(docType)); builder.documenttype(docTypeBuilder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 53456c627a4..1d925eb212f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -29,6 +30,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -823,6 +825,63 @@ public class ContentClusterTest extends ContentBaseTest { new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } + private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, + List<String> defaultSpaceTypes, List<String> globalSpaceTypes) { + AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); + assertNotNull(cluster); + assertEquals(defaultSpaceTypes.size() + globalSpaceTypes.size(), cluster.documentType().size()); + assertClusterHasTypesInBucketSpace(cluster, "default", defaultSpaceTypes); + assertClusterHasTypesInBucketSpace(cluster, "global", globalSpaceTypes); + } + + private void assertClusterHasTypesInBucketSpace(AllClustersBucketSpacesConfig.Cluster cluster, + String bucketSpace, List<String> expectedTypes) { + for (String type : expectedTypes) { + assertNotNull(cluster.documentType(type)); + assertEquals(bucketSpace, cluster.documentType(type).bucketSpace()); + } + } + + @Test + public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + String xml = + "<services>" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\"/>" + + "</admin>" + + "<content version=\"1.0\" id=\"foocluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"bunnies\" mode=\"index\"/>" + + " <document type=\"hares\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "<content version=\"1.0\" id=\"barcluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "</services>"; + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + + AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); + model.getConfig(builder, "client"); + AllClustersBucketSpacesConfig config = builder.build(); + + assertEquals(2, config.cluster().size()); + + assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); + assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); + } + private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 83b4cfebca5..f49cc58224a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -19,6 +20,7 @@ import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClus import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -168,6 +170,19 @@ public class ContentSearchClusterTest { } @Test + public void bucket_space_config_builder_returns_correct_mappings() throws Exception { + ContentCluster cluster = createClusterWithGlobalType(); + BucketspacesConfig expected = getBucketspacesConfig(cluster); + AllClustersBucketSpacesConfig.Cluster actual = cluster.clusterBucketSpaceConfigBuilder().build(); + assertEquals(2, expected.documenttype().size()); + assertEquals(expected.documenttype().size(), actual.documentType().size()); + assertNotNull(actual.documentType("global")); + assertEquals("global", actual.documentType().get("global").bucketSpace()); + assertNotNull(actual.documentType("regular")); + assertEquals("default", actual.documentType().get("regular").bucketSpace()); + } + + @Test public void cluster_with_global_document_types_sets_cluster_controller_global_docs_config_option() throws Exception { ContentCluster cluster = createClusterWithGlobalType(); assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); |