diff options
10 files changed, 126 insertions, 19 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index af9eba92a98..aede68a1dd1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -19,6 +19,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -212,6 +213,14 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override + public void getConfig(AllClustersBucketSpacesConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + for (ContentCluster cluster : model.getContentClusters().values()) { + builder.cluster(cluster.getName(), cluster.clusterBucketSpaceConfigBuilder()); + } + } + + @Override public void getConfig(ModelConfig.Builder builder) { builder.vespaVersion(vespaVersion.toFullString()); for (HostResource modelHost : getHostSystem().getHosts()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index 2a30a89dd8a..9dcd94bf455 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -12,6 +12,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; /** @@ -29,6 +30,7 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, ZookeepersConfig.Producer, LoadTypeConfig.Producer, ClusterListConfig.Producer, + AllClustersBucketSpacesConfig.Producer, ModelConfig.Producer, ApplicationIdConfig.Producer { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 27da41c2bfa..1ec56fe529a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.documentmodel.NewDocumentType; @@ -736,13 +737,26 @@ public class ContentCluster extends AbstractConfigProducer implements private static final String DEFAULT_BUCKET_SPACE = "default"; private static final String GLOBAL_BUCKET_SPACE = "global"; + private String bucketSpaceOfDocumentType(NewDocumentType docType) { + return (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + } + + public AllClustersBucketSpacesConfig.Cluster.Builder clusterBucketSpaceConfigBuilder() { + AllClustersBucketSpacesConfig.Cluster.Builder builder = new AllClustersBucketSpacesConfig.Cluster.Builder(); + for (NewDocumentType docType : getDocumentDefinitions().values()) { + AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder typeBuilder = new AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder(); + typeBuilder.bucketSpace(bucketSpaceOfDocumentType(docType)); + builder.documentType(docType.getName(), typeBuilder); + } + return builder; + } + @Override public void getConfig(BucketspacesConfig.Builder builder) { for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); - docTypeBuilder.bucketspace(bucketSpace); + docTypeBuilder.bucketspace(bucketSpaceOfDocumentType(docType)); builder.documenttype(docTypeBuilder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 53456c627a4..a003e0122a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.metrics.MetricsmanagerConfig; @@ -29,6 +30,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -823,6 +825,63 @@ public class ContentClusterTest extends ContentBaseTest { new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } + private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, + List<String> defaultSpaceTypes, List<String> globalSpaceTypes) { + AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); + assertNotNull(cluster); + assertEquals(defaultSpaceTypes.size() + globalSpaceTypes.size(), cluster.documentType().size()); + assertClusterHasTypesInBucketSpace(cluster, "default", defaultSpaceTypes); + assertClusterHasTypesInBucketSpace(cluster, "global", globalSpaceTypes); + } + + private void assertClusterHasTypesInBucketSpace(AllClustersBucketSpacesConfig.Cluster cluster, + String bucketSpace, List<String> expectedTypes) { + for (String type : expectedTypes) { + assertNotNull(cluster.documentType(type)); + assertEquals(bucketSpace, cluster.documentType(type).bucketSpace()); + } + } + + @Test + public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + String xml = + "<services>" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\"/>" + + "</admin>" + + "<content version=\"1.0\" id=\"foocluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"bunnies\" mode=\"index\"/>" + + " <document type=\"hares\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "<content version=\"1.0\" id=\"barcluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "</services>"; + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + + AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); + model.getConfig(builder, "client"); + AllClustersBucketSpacesConfig config = builder.build(); + + assertEquals(2, config.cluster().size()); + + assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); + assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); + } + private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 83b4cfebca5..98fa179b219 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -19,6 +20,7 @@ import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClus import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -168,6 +170,19 @@ public class ContentSearchClusterTest { } @Test + public void bucket_space_config_builder_returns_correct_mappings() throws Exception { + ContentCluster cluster = createClusterWithGlobalType(); + BucketspacesConfig expected = getBucketspacesConfig(cluster); + AllClustersBucketSpacesConfig.Cluster actual = cluster.clusterBucketSpaceConfigBuilder().build(); + assertEquals(2, expected.documenttype().size()); + assertEquals(expected.documenttype().size(), actual.documentType().size()); + assertNotNull(actual.documentType("global")); + assertEquals("global", actual.documentType().get("global").bucketSpace()); + assertNotNull(actual.documentType("regular")); + assertEquals("default", actual.documentType().get("regular").bucketSpace()); + } + + @Test public void cluster_with_global_document_types_sets_cluster_controller_global_docs_config_option() throws Exception { ContentCluster cluster = createClusterWithGlobalType(); assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 0a7d4ef4381..120c403eb74 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -72,5 +72,7 @@ vespa_generate_config(configdefinitions zookeepers.def) install_config_definition(zookeepers.def cloud.config.zookeepers.def) vespa_generate_config(configdefinitions bucketspaces.def) install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) +vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) +install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.all-clusters-bucket-spaces.def) vespa_generate_config(configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) diff --git a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def new file mode 100644 index 00000000000..7676bc2a03f --- /dev/null +++ b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## This config contains the document types handled by all content clusters +## and the bucket spaces they belong to. + +## The bucket space a document type in a particular cluster belongs to. +cluster{}.documentType{}.bucketSpace string + diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 65e7191e164..2b12c7cd78c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -50,7 +50,7 @@ public class OperationHandlerImpl implements OperationHandler { } public interface BucketSpaceResolver { - Optional<String> clusterBucketSpaceFromDocumentType(String clusterConfigId, String docType); + Optional<String> clusterBucketSpaceFromDocumentType(String clusterId, String docType); } public static class BucketSpaceRoute { @@ -91,20 +91,6 @@ public class OperationHandlerImpl implements OperationHandler { private final ConcurrentResourcePool<SyncSession> syncSessions; - private static ClusterEnumerator defaultClusterEnumerator() { - return () -> new ClusterList("client").getStorageClusters(); - } - - private static BucketSpaceResolver defaultBucketResolver() { - return (clusterConfigId, docType) -> Optional.ofNullable(BucketSpaceEnumerator - .fromConfig(clusterConfigId).getDoctypeToSpaceMapping() - .get(docType)); - } - - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, MetricReceiver metricReceiver) { - this(documentAccess, clusterEnumerator, defaultBucketResolver(), metricReceiver); - } - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, BucketSpaceResolver bucketSpaceResolver, MetricReceiver metricReceiver) { this.documentAccess = documentAccess; @@ -335,7 +321,7 @@ public class OperationHandlerImpl implements OperationHandler { String targetBucketSpace; if (!restUri.isRootOnly()) { String docType = restUri.getDocumentType(); - Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getConfigId(), docType); + Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getName(), docType); if (!resolvedSpace.isPresent()) { throw new RestApiException(Response.createErrorResponse(400, String.format( "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index f088db31c23..60f9101cdc8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -30,6 +30,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -75,6 +76,7 @@ public class RestApi extends LoggingRequestHandler { @Inject public RestApi(LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, + AllClustersBucketSpacesConfig bucketSpacesConfig, ClusterListConfig clusterListConfig, MetricReceiver metricReceiver) { super(parentCtx); @@ -83,6 +85,7 @@ public class RestApi extends LoggingRequestHandler { this.operationHandler = new OperationHandlerImpl( new MessageBusDocumentAccess(params), fixedClusterEnumeratorFromConfig(clusterListConfig), + fixedBucketSpaceResolverFromConfig(bucketSpacesConfig), metricReceiver); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); // 40% of the threads can be blocked before we deny requests. @@ -120,6 +123,14 @@ public class RestApi extends LoggingRequestHandler { return () -> clusters; } + private static OperationHandlerImpl.BucketSpaceResolver fixedBucketSpaceResolverFromConfig( + AllClustersBucketSpacesConfig bucketSpacesConfig) { + return (clusterId, docType) -> + Optional.ofNullable(bucketSpacesConfig.cluster(clusterId)) + .map(cluster -> cluster.documentType(docType)) + .map(type -> type.bucketSpace()); + } + private static Optional<String> requestProperty(String parameter, HttpRequest request) { final String property = request.getProperty(parameter); if (property != null && ! property.isEmpty()) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 4c04070cec4..30f039c31fb 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -132,7 +132,7 @@ public class OperationHandlerImplTest { }); when(documentAccess.createSyncSession(any(SyncParameters.class))).thenReturn(mockSyncSession); OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (configId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); + OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (clusterId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); return new OperationHandlerImpl(documentAccess, clusterEnumerator, bucketSpaceResolver, MetricReceiver.nullImplementation); } } |