From 9f9693d524562a356b7ead63c953d8c7995c09b0 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 6 Dec 2018 17:16:29 +0100 Subject: Revert "Add and use cross cluster bucket space config" --- .../model/ApplicationConfigProducerRoot.java | 9 ---- .../yahoo/config/model/CommonConfigsProducer.java | 2 - .../model/content/cluster/ContentCluster.java | 18 +------ .../vespa/model/content/ContentClusterTest.java | 59 ---------------------- .../model/content/ContentSearchClusterTest.java | 15 ------ configdefinitions/src/vespa/CMakeLists.txt | 2 - .../src/vespa/all-clusters-bucket-spaces.def | 9 ---- .../document/restapi/OperationHandlerImpl.java | 18 ++++++- .../yahoo/document/restapi/resource/RestApi.java | 11 ---- .../document/restapi/OperationHandlerImplTest.java | 2 +- 10 files changed, 19 insertions(+), 126 deletions(-) delete mode 100644 configdefinitions/src/vespa/all-clusters-bucket-spaces.def diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 7be4b27860c..af9eba92a98 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -19,7 +19,6 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -212,14 +211,6 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer defaultSpaceTypes, List globalSpaceTypes) { - AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); - assertNotNull(cluster); - assertEquals(defaultSpaceTypes.size() + globalSpaceTypes.size(), cluster.documentType().size()); - assertClusterHasTypesInBucketSpace(cluster, "default", defaultSpaceTypes); - assertClusterHasTypesInBucketSpace(cluster, "global", globalSpaceTypes); - } - - private void assertClusterHasTypesInBucketSpace(AllClustersBucketSpacesConfig.Cluster cluster, - String bucketSpace, List expectedTypes) { - for (String type : expectedTypes) { - assertNotNull(cluster.documentType(type)); - assertEquals(bucketSpace, cluster.documentType(type).bucketSpace()); - } - } - - @Test - public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { - String xml = - "" + - "" + - " " + - "" + - "" + - " 1" + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - "" + - "" + - " 1" + - " " + - " " + - " " + - " " + - " " + - " " + - "" + - ""; - List sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); - VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); - - AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); - model.getConfig(builder, "client"); - AllClustersBucketSpacesConfig config = builder.build(); - - assertEquals(2, config.cluster().size()); - - assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); - assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); - } - private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index f49cc58224a..83b4cfebca5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.FleetcontrollerConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -20,7 +19,6 @@ import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClus import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -169,19 +167,6 @@ public class ContentSearchClusterTest { assertDocumentType("regular", "default", config.documenttype(1)); } - @Test - public void bucket_space_config_builder_returns_correct_mappings() throws Exception { - ContentCluster cluster = createClusterWithGlobalType(); - BucketspacesConfig expected = getBucketspacesConfig(cluster); - AllClustersBucketSpacesConfig.Cluster actual = cluster.clusterBucketSpaceConfigBuilder().build(); - assertEquals(2, expected.documenttype().size()); - assertEquals(expected.documenttype().size(), actual.documentType().size()); - assertNotNull(actual.documentType("global")); - assertEquals("global", actual.documentType().get("global").bucketSpace()); - assertNotNull(actual.documentType("regular")); - assertEquals("default", actual.documentType().get("regular").bucketSpace()); - } - @Test public void cluster_with_global_document_types_sets_cluster_controller_global_docs_config_option() throws Exception { ContentCluster cluster = createClusterWithGlobalType(); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 25f5963ec70..0a7d4ef4381 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -72,7 +72,5 @@ vespa_generate_config(configdefinitions zookeepers.def) install_config_definition(zookeepers.def cloud.config.zookeepers.def) vespa_generate_config(configdefinitions bucketspaces.def) install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) -vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) -install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.core.all-clusters-bucket-spaces.def) vespa_generate_config(configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) diff --git a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def deleted file mode 100644 index e73dec65261..00000000000 --- a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## This config contains the document types handled by all content clusters -## and the bucket spaces they belong to. - -## The bucket space a document type in a particular cluster belongs to. -cluster{}.documentType{}.bucketSpace string - diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 2b12c7cd78c..65e7191e164 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -50,7 +50,7 @@ public class OperationHandlerImpl implements OperationHandler { } public interface BucketSpaceResolver { - Optional clusterBucketSpaceFromDocumentType(String clusterId, String docType); + Optional clusterBucketSpaceFromDocumentType(String clusterConfigId, String docType); } public static class BucketSpaceRoute { @@ -91,6 +91,20 @@ public class OperationHandlerImpl implements OperationHandler { private final ConcurrentResourcePool syncSessions; + private static ClusterEnumerator defaultClusterEnumerator() { + return () -> new ClusterList("client").getStorageClusters(); + } + + private static BucketSpaceResolver defaultBucketResolver() { + return (clusterConfigId, docType) -> Optional.ofNullable(BucketSpaceEnumerator + .fromConfig(clusterConfigId).getDoctypeToSpaceMapping() + .get(docType)); + } + + public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, MetricReceiver metricReceiver) { + this(documentAccess, clusterEnumerator, defaultBucketResolver(), metricReceiver); + } + public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, BucketSpaceResolver bucketSpaceResolver, MetricReceiver metricReceiver) { this.documentAccess = documentAccess; @@ -321,7 +335,7 @@ public class OperationHandlerImpl implements OperationHandler { String targetBucketSpace; if (!restUri.isRootOnly()) { String docType = restUri.getDocumentType(); - Optional resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getName(), docType); + Optional resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getConfigId(), docType); if (!resolvedSpace.isPresent()) { throw new RestApiException(Response.createErrorResponse(400, String.format( "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index baf7dcb0f68..f088db31c23 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -30,7 +30,6 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespa.config.content.core.AllClustersBucketSpacesConfig; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -76,7 +75,6 @@ public class RestApi extends LoggingRequestHandler { @Inject public RestApi(LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, - AllClustersBucketSpacesConfig bucketSpacesConfig, ClusterListConfig clusterListConfig, MetricReceiver metricReceiver) { super(parentCtx); @@ -85,7 +83,6 @@ public class RestApi extends LoggingRequestHandler { this.operationHandler = new OperationHandlerImpl( new MessageBusDocumentAccess(params), fixedClusterEnumeratorFromConfig(clusterListConfig), - fixedBucketSpaceResolverFromConfig(bucketSpacesConfig), metricReceiver); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); // 40% of the threads can be blocked before we deny requests. @@ -123,14 +120,6 @@ public class RestApi extends LoggingRequestHandler { return () -> clusters; } - private static OperationHandlerImpl.BucketSpaceResolver fixedBucketSpaceResolverFromConfig( - AllClustersBucketSpacesConfig bucketSpacesConfig) { - return (clusterId, docType) -> - Optional.ofNullable(bucketSpacesConfig.cluster(clusterId)) - .map(cluster -> cluster.documentType(docType)) - .map(type -> type.bucketSpace()); - } - private static Optional requestProperty(String parameter, HttpRequest request) { final String property = request.getProperty(parameter); if (property != null && ! property.isEmpty()) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 30f039c31fb..4c04070cec4 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -132,7 +132,7 @@ public class OperationHandlerImplTest { }); when(documentAccess.createSyncSession(any(SyncParameters.class))).thenReturn(mockSyncSession); OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (clusterId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); + OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (configId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); return new OperationHandlerImpl(documentAccess, clusterEnumerator, bucketSpaceResolver, MetricReceiver.nullImplementation); } } -- cgit v1.2.3