Merge pull request #4439 from vespa-engine/geirst/produce-bucketspaces-config-for-entire-content-cluster

Geirst/produce bucketspaces config for entire content cluster

7 files changed, 120 insertions, 117 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
index 81aca977400..0405f96cd89 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -1,8 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.content;
 
-import com.yahoo.documentmodel.NewDocumentType;
-import com.yahoo.vespa.config.content.core.BucketspacesConfig;
 import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
 import com.yahoo.vespa.config.content.core.StorServerConfig;
 import com.yahoo.document.select.DocumentSelector;
@@ -16,15 +14,13 @@ import org.w3c.dom.Element;
 
 import java.util.logging.Logger;
 
-
 /**
  * Generates distributor-specific configuration.
  */
 public class DistributorCluster extends AbstractConfigProducer<Distributor> implements
         StorDistributormanagerConfig.Producer,
         StorServerConfig.Producer,
-        MetricsmanagerConfig.Producer,
-        BucketspacesConfig.Producer {
+        MetricsmanagerConfig.Producer {
 
     public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString());
 
@@ -151,20 +147,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
         builder.is_distributor(true);
     }
 
-    private static final String DEFAULT_BUCKET_SPACE = "default";
-    private static final String GLOBAL_BUCKET_SPACE = "global";
-
-    @Override
-    public void getConfig(BucketspacesConfig.Builder builder) {
-        for (NewDocumentType docType : parent.getDocumentDefinitions().values()) {
-            BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder();
-            docTypeBuilder.name(docType.getName());
-            String bucketSpace = (parent.isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
-            docTypeBuilder.bucketspace(bucketSpace);
-            builder.documenttype(docTypeBuilder);
-        }
-    }
-
     public String getClusterName() {
         return parent.getName();
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 7889b857fff..7654fbc217b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -7,12 +7,11 @@ import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.RegionName;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig;
 import com.yahoo.vespa.config.content.FleetcontrollerConfig;
 import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.vespa.config.content.core.BucketspacesConfig;
 import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
 import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
@@ -59,7 +58,8 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
         StorDistributormanagerConfig.Producer,
         FleetcontrollerConfig.Producer,
         MetricsmanagerConfig.Producer,
-        MessagetyperouteselectorpolicyConfig.Producer {
+        MessagetyperouteselectorpolicyConfig.Producer,
+        BucketspacesConfig.Producer {
 
     // TODO: Make private
     private String documentSelection;
@@ -694,4 +694,18 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
         }
 
     }
+
+    private static final String DEFAULT_BUCKET_SPACE = "default";
+    private static final String GLOBAL_BUCKET_SPACE = "global";
+
+    @Override
+    public void getConfig(BucketspacesConfig.Builder builder) {
+        for (NewDocumentType docType : getDocumentDefinitions().values()) {
+            BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder();
+            docTypeBuilder.name(docType.getName());
+            String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
+            docTypeBuilder.bucketspace(bucketSpace);
+            builder.documenttype(docTypeBuilder);
+        }
+    }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index 5f18b28d6ce..41bba055a50 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -1,9 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.content;
 
+import com.yahoo.vespa.config.content.core.BucketspacesConfig;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
+import com.yahoo.vespa.model.content.utils.DocType;
 import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
 import org.junit.Test;
 
@@ -15,6 +17,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
 import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster;
 import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions;
 import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
 
 /**
  * Unit tests for content search cluster.
@@ -36,8 +39,8 @@ public class ContentSearchClusterTest {
 
     private static ContentCluster createClusterWithGlobalType() throws Exception {
         return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
-                new ContentClusterBuilder.DocType("global", true),
-                new ContentClusterBuilder.DocType("regular"))).getXml(),
+                DocType.indexGlobal("global"),
+                DocType.index("regular"))).getXml(),
                 createSearchDefinitions("global", "regular"));
     }
 
@@ -108,10 +111,31 @@ public class ContentSearchClusterTest {
                 .content("field ref_to_c type reference<c> { indexing: attribute }").build());
         searchDefinitions.add(new SearchDefinitionBuilder().name("c").build());
         return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
-                new ContentClusterBuilder.DocType("a"),
-                new ContentClusterBuilder.DocType("b", true),
-                new ContentClusterBuilder.DocType("c", true))).getXml(),
+                DocType.index("a"),
+                DocType.indexGlobal("b"),
+                DocType.indexGlobal("c"))).getXml(),
                 searchDefinitions);
     }
 
+    private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) {
+        BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder();
+        cluster.getConfig(builder);
+        return new BucketspacesConfig(builder);
+    }
+
+    private static void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) {
+        assertEquals(expName, docType.name());
+        assertEquals(expBucketSpace, docType.bucketspace());
+    }
+
+    @Test
+    public void require_that_bucket_spaces_config_is_produced_for_content_cluster() throws Exception {
+        BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType());
+        assertEquals(2, config.documenttype().size());
+        assertDocumentType("global", "global", config.documenttype(0));
+        assertDocumentType("regular", "default", config.documenttype(1));
+        // Safeguard against flipping the switch
+        assertFalse(config.enable_multiple_bucket_spaces());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
index d4e804d3f95..48b7ccdad6b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
@@ -1,23 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.content;
 
-import com.yahoo.vespa.config.content.core.BucketspacesConfig;
 import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig;
 import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
 import com.yahoo.vespa.config.content.core.StorServerConfig;
 import com.yahoo.config.model.test.MockRoot;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
+import com.yahoo.vespa.model.content.utils.DocType;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import org.junit.Test;
 
-import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.hamcrest.Matchers.*;
+
 /**
  * Test for content DistributorCluster.
  */
@@ -303,100 +302,48 @@ public class DistributorTest {
         return new StorDistributormanagerConfig(builder);
     }
 
-    private static class DocDef {
-        public final String type;
-        public final String mode;
-        public final boolean global;
-
-        private DocDef(String type, String mode, boolean global) {
-            this.type = type;
-            this.mode = mode;
-            this.global = global;
-        }
-
-        public static DocDef storeOnly(String type) {
-            return new DocDef(type, "store-only", false);
-        }
-
-        public static DocDef index(String type) {
-            return new DocDef(type, "index", false);
-        }
-
-        public static DocDef indexGlobal(String type) {
-            return new DocDef(type, "index", true);
-        }
-
-        public static DocDef streaming(String type) {
-            return new DocDef(type, "streaming", false);
-        }
-    }
-
-    private String generateXmlForDocDefs(DocDef... defs) {
+    private String generateXmlForDocTypes(DocType... docTypes) {
         return "<content id='storage'>\n" +
-               "  <documents>\n" +
-               Arrays.stream(defs)
-                  .map(def -> String.format("    <document type='%s' mode='%s' global='%s'/>",
-                          def.type, def.mode, (def.global ? "true" : "false")))
-                  .collect(Collectors.joining("\n")) +
-               "\n  </documents>\n" +
-               "</content>";
+                DocType.listToXml(docTypes) +
+               "\n</content>";
     }
 
     @Test
     public void bucket_activation_disabled_if_no_documents_in_indexed_mode() {
         StorDistributormanagerConfig config = clusterXmlToConfig(
-                generateXmlForDocDefs(DocDef.storeOnly("music")));
+                generateXmlForDocTypes(DocType.storeOnly("music")));
         assertThat(config.disable_bucket_activation(), is(true));
     }
 
     @Test
     public void bucket_activation_enabled_with_single_indexed_document() {
         StorDistributormanagerConfig config = clusterXmlToConfig(
-                generateXmlForDocDefs(DocDef.index("music")));
+                generateXmlForDocTypes(DocType.index("music")));
         assertThat(config.disable_bucket_activation(), is(false));
     }
 
     @Test
     public void bucket_activation_enabled_with_multiple_indexed_documents() {
         StorDistributormanagerConfig config = clusterXmlToConfig(
-                generateXmlForDocDefs(DocDef.index("music"),
-                                      DocDef.index("movies")));
+                generateXmlForDocTypes(DocType.index("music"),
+                                       DocType.index("movies")));
         assertThat(config.disable_bucket_activation(), is(false));
     }
 
     @Test
     public void bucket_activation_enabled_if_at_least_one_document_indexed() {
         StorDistributormanagerConfig config = clusterXmlToConfig(
-                generateXmlForDocDefs(DocDef.storeOnly("music"),
-                                      DocDef.streaming("bunnies"),
-                                      DocDef.index("movies")));
+                generateXmlForDocTypes(DocType.storeOnly("music"),
+                                       DocType.streaming("bunnies"),
+                                       DocType.index("movies")));
         assertThat(config.disable_bucket_activation(), is(false));
     }
 
     @Test
     public void bucket_activation_disabled_for_single_streaming_type() {
         StorDistributormanagerConfig config = clusterXmlToConfig(
-                generateXmlForDocDefs(DocDef.streaming("music")));
+                generateXmlForDocTypes(DocType.streaming("music")));
         assertThat(config.disable_bucket_activation(), is(true));
     }
 
-    private BucketspacesConfig clusterXmlToBucketspacesConfig(String xml) {
-        BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder();
-        parse(xml).getConfig(builder);
-        return new BucketspacesConfig(builder);
-    }
-
-    private void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) {
-        assertEquals(expName, docType.name());
-        assertEquals(expBucketSpace, docType.bucketspace());
-    }
-
-    @Test
-    public void bucket_spaces_config_is_produced_for_distributor_cluster() {
-        BucketspacesConfig config = clusterXmlToBucketspacesConfig(
-                generateXmlForDocDefs(DocDef.index("music"), DocDef.indexGlobal("movies")));
-        assertEquals(2, config.documenttype().size());
-        assertDocumentType("movies", "global", config.documenttype(0));
-        assertDocumentType("music", "default", config.documenttype(1));
-    }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
index 592e90efd22..95c57bb544c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
@@ -18,29 +18,10 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
  */
 public class ContentClusterBuilder {
 
-    public static class DocType {
-        private final String name;
-        private final boolean global;
-
-        public DocType(String name, boolean global) {
-            this.name = name;
-            this.global = global;
-        }
-
-        public DocType(String name) {
-            this(name, false);
-        }
-
-        public String toXml() {
-            return (global ? "<document mode='index' type='" + name + "' global='true'/>" :
-                    "<document mode='index' type='" + name + "'/>");
-        }
-    }
-
     private String name = "mycluster";
     private int redundancy = 1;
     private int searchableCopies = 1;
-    private List<DocType> docTypes = Arrays.asList(new DocType("test"));
+    private List<DocType> docTypes = Arrays.asList(DocType.index("test"));
     private String groupXml = getSimpleGroupXml();
     private Optional<String> dispatchXml = Optional.empty();
     private Optional<Double> protonDiskLimit = Optional.empty();
@@ -66,7 +47,7 @@ public class ContentClusterBuilder {
 
     public ContentClusterBuilder docTypes(String ... docTypes) {
         this.docTypes = Arrays.asList(docTypes).stream().
-                map(type -> new DocType(type)).
+                map(type -> DocType.index(type)).
                 collect(Collectors.toList());
         return this;
     }
@@ -103,9 +84,7 @@ public class ContentClusterBuilder {
     public String getXml() {
         String xml = joinLines("<content version='1.0' id='" + name + "'>",
                "  <redundancy>" + redundancy + "</redundancy>",
-               "  <documents>",
-                docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")),
-               "  </documents>",
+               DocType.listToXml(docTypes),
                "  <engine>",
                "    <proton>",
                "      <searchable-copies>" + searchableCopies + "</searchable-copies>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java
new file mode 100644
index 00000000000..3a5f679509b
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java
@@ -0,0 +1,54 @@
+package com.yahoo.vespa.model.content.utils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Definition of a document type used for testing.
+ *
+ * @author geirst
+ */
+public class DocType {
+    private final String type;
+    private final String mode;
+    private final boolean global;
+
+    private DocType(String type, String mode, boolean global) {
+        this.type = type;
+        this.mode = mode;
+        this.global = global;
+    }
+
+    public String toXml() {
+        return (global ? "<document mode='" + mode + "' type='" + type + "' global='true'/>" :
+                "<document mode='" + mode + "' type='" + type + "'/>");
+    }
+
+    public static DocType storeOnly(String type) {
+        return new DocType(type, "store-only", false);
+    }
+
+    public static DocType index(String type) {
+        return new DocType(type, "index", false);
+    }
+
+    public static DocType indexGlobal(String type) {
+        return new DocType(type, "index", true);
+    }
+
+    public static DocType streaming(String type) {
+        return new DocType(type, "streaming", false);
+    }
+
+    public static String listToXml(DocType... docTypes) {
+        return listToXml(Arrays.asList(docTypes));
+    }
+
+    public static String listToXml(List<DocType> docTypes) {
+        return "<documents>\n" +
+                docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")) + "\n" +
+                "</documents>";
+    }
+
+}
diff --git a/storage/src/vespa/storage/config/bucketspaces.def b/storage/src/vespa/storage/config/bucketspaces.def
index 3ed1abba0b4..4db107ec1ee 100644
--- a/storage/src/vespa/storage/config/bucketspaces.def
+++ b/storage/src/vespa/storage/config/bucketspaces.def
@@ -9,3 +9,6 @@ documenttype[].name string
 
 ## The bucket space this document type belongs to.
 documenttype[].bucketspace string
+
+## Switch to enable multiple bucket spaces in content layer and content nodes.
+enable_multiple_bucket_spaces bool default=false