diff options
10 files changed, 19 insertions, 126 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 7be4b27860c..af9eba92a98 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -19,7 +19,6 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -213,14 +212,6 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override - public void getConfig(AllClustersBucketSpacesConfig.Builder builder) { - VespaModel model = (VespaModel) getRoot(); - for (ContentCluster cluster : model.getContentClusters().values()) { - builder.cluster(cluster.getName(), cluster.clusterBucketSpaceConfigBuilder()); - } - } - - @Override public void getConfig(ModelConfig.Builder builder) { builder.vespaVersion(vespaVersion.toFullString()); for (HostResource modelHost : getHostSystem().getHosts()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index cb30a64e1fc..2a30a89dd8a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -12,7 +12,6 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; /** @@ -30,7 +29,6 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, ZookeepersConfig.Producer, LoadTypeConfig.Producer, ClusterListConfig.Producer, - AllClustersBucketSpacesConfig.Producer, ModelConfig.Producer, ApplicationIdConfig.Producer { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 23c12465121..27da41c2bfa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.documentmodel.NewDocumentType; @@ -737,26 +736,13 @@ public class ContentCluster extends AbstractConfigProducer implements private static final String DEFAULT_BUCKET_SPACE = "default"; private static final String GLOBAL_BUCKET_SPACE = "global"; - private String bucketSpaceOfDocumentType(NewDocumentType docType) { - return (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); - } - - public AllClustersBucketSpacesConfig.Cluster.Builder clusterBucketSpaceConfigBuilder() { - AllClustersBucketSpacesConfig.Cluster.Builder builder = new AllClustersBucketSpacesConfig.Cluster.Builder(); - for (NewDocumentType docType : getDocumentDefinitions().values()) { - AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder typeBuilder = new AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder(); - typeBuilder.bucketSpace(bucketSpaceOfDocumentType(docType)); - builder.documentType(docType.getName(), typeBuilder); - } - return builder; - } - @Override public void getConfig(BucketspacesConfig.Builder builder) { for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - docTypeBuilder.bucketspace(bucketSpaceOfDocumentType(docType)); + String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + docTypeBuilder.bucketspace(bucketSpace); builder.documenttype(docTypeBuilder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 1d925eb212f..53456c627a4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -30,7 +29,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -825,63 +823,6 @@ public class ContentClusterTest extends ContentBaseTest { new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } - private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, - List<String> defaultSpaceTypes, List<String> globalSpaceTypes) { - AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); - assertNotNull(cluster); - assertEquals(defaultSpaceTypes.size() + globalSpaceTypes.size(), cluster.documentType().size()); - assertClusterHasTypesInBucketSpace(cluster, "default", defaultSpaceTypes); - assertClusterHasTypesInBucketSpace(cluster, "global", globalSpaceTypes); - } - - private void assertClusterHasTypesInBucketSpace(AllClustersBucketSpacesConfig.Cluster cluster, - String bucketSpace, List<String> expectedTypes) { - for (String type : expectedTypes) { - assertNotNull(cluster.documentType(type)); - assertEquals(bucketSpace, cluster.documentType(type).bucketSpace()); - } - } - - @Test - public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { - String xml = - "<services>" + - "<admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - "</admin>" + - "<content version=\"1.0\" id=\"foocluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"bunnies\" mode=\"index\"/>" + - " <document type=\"hares\" mode=\"index\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "<content version=\"1.0\" id=\"barcluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); - VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); - - AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); - model.getConfig(builder, "client"); - AllClustersBucketSpacesConfig config = builder.build(); - - assertEquals(2, config.cluster().size()); - - assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); - assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); - } - private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index f49cc58224a..83b4cfebca5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.FleetcontrollerConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -20,7 +19,6 @@ import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClus import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -170,19 +168,6 @@ public class ContentSearchClusterTest { } @Test - public void bucket_space_config_builder_returns_correct_mappings() throws Exception { - ContentCluster cluster = createClusterWithGlobalType(); - BucketspacesConfig expected = getBucketspacesConfig(cluster); - AllClustersBucketSpacesConfig.Cluster actual = cluster.clusterBucketSpaceConfigBuilder().build(); - assertEquals(2, expected.documenttype().size()); - assertEquals(expected.documenttype().size(), actual.documentType().size()); - assertNotNull(actual.documentType("global")); - assertEquals("global", actual.documentType().get("global").bucketSpace()); - assertNotNull(actual.documentType("regular")); - assertEquals("default", actual.documentType().get("regular").bucketSpace()); - } - - @Test public void cluster_with_global_document_types_sets_cluster_controller_global_docs_config_option() throws Exception { ContentCluster cluster = createClusterWithGlobalType(); assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 25f5963ec70..0a7d4ef4381 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -72,7 +72,5 @@ vespa_generate_config(configdefinitions zookeepers.def) install_config_definition(zookeepers.def cloud.config.zookeepers.def) vespa_generate_config(configdefinitions bucketspaces.def) install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) -vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) -install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.core.all-clusters-bucket-spaces.def) vespa_generate_config(configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) diff --git a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def deleted file mode 100644 index e73dec65261..00000000000 --- a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## This config contains the document types handled by all content clusters -## and the bucket spaces they belong to. - -## The bucket space a document type in a particular cluster belongs to. -cluster{}.documentType{}.bucketSpace string - diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 2b12c7cd78c..65e7191e164 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -50,7 +50,7 @@ public class OperationHandlerImpl implements OperationHandler { } public interface BucketSpaceResolver { - Optional<String> clusterBucketSpaceFromDocumentType(String clusterId, String docType); + Optional<String> clusterBucketSpaceFromDocumentType(String clusterConfigId, String docType); } public static class BucketSpaceRoute { @@ -91,6 +91,20 @@ public class OperationHandlerImpl implements OperationHandler { private final ConcurrentResourcePool<SyncSession> syncSessions; + private static ClusterEnumerator defaultClusterEnumerator() { + return () -> new ClusterList("client").getStorageClusters(); + } + + private static BucketSpaceResolver defaultBucketResolver() { + return (clusterConfigId, docType) -> Optional.ofNullable(BucketSpaceEnumerator + .fromConfig(clusterConfigId).getDoctypeToSpaceMapping() + .get(docType)); + } + + public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, MetricReceiver metricReceiver) { + this(documentAccess, clusterEnumerator, defaultBucketResolver(), metricReceiver); + } + public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, BucketSpaceResolver bucketSpaceResolver, MetricReceiver metricReceiver) { this.documentAccess = documentAccess; @@ -321,7 +335,7 @@ public class OperationHandlerImpl implements OperationHandler { String targetBucketSpace; if (!restUri.isRootOnly()) { String docType = restUri.getDocumentType(); - Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getName(), docType); + Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getConfigId(), docType); if (!resolvedSpace.isPresent()) { throw new RestApiException(Response.createErrorResponse(400, String.format( "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index baf7dcb0f68..f088db31c23 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -30,7 +30,6 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -76,7 +75,6 @@ public class RestApi extends LoggingRequestHandler { @Inject public RestApi(LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, - AllClustersBucketSpacesConfig bucketSpacesConfig, ClusterListConfig clusterListConfig, MetricReceiver metricReceiver) { super(parentCtx); @@ -85,7 +83,6 @@ public class RestApi extends LoggingRequestHandler { this.operationHandler = new OperationHandlerImpl( new MessageBusDocumentAccess(params), fixedClusterEnumeratorFromConfig(clusterListConfig), - fixedBucketSpaceResolverFromConfig(bucketSpacesConfig), metricReceiver); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); // 40% of the threads can be blocked before we deny requests. @@ -123,14 +120,6 @@ public class RestApi extends LoggingRequestHandler { return () -> clusters; } - private static OperationHandlerImpl.BucketSpaceResolver fixedBucketSpaceResolverFromConfig( - AllClustersBucketSpacesConfig bucketSpacesConfig) { - return (clusterId, docType) -> - Optional.ofNullable(bucketSpacesConfig.cluster(clusterId)) - .map(cluster -> cluster.documentType(docType)) - .map(type -> type.bucketSpace()); - } - private static Optional<String> requestProperty(String parameter, HttpRequest request) { final String property = request.getProperty(parameter); if (property != null && ! property.isEmpty()) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 30f039c31fb..4c04070cec4 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -132,7 +132,7 @@ public class OperationHandlerImplTest { }); when(documentAccess.createSyncSession(any(SyncParameters.class))).thenReturn(mockSyncSession); OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (clusterId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); + OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (configId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); return new OperationHandlerImpl(documentAccess, clusterEnumerator, bucketSpaceResolver, MetricReceiver.nullImplementation); } } |