Tag globally distributed documents as such in proton config.

5 files changed, 85 insertions, 18 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 7e24285c6fb..5f511a7f2aa 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -34,7 +34,8 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
     private Redundancy redundancy;
 
     private final String clusterName;
-    Map<String, NewDocumentType> documentDefinitions;
+    private final Map<String, NewDocumentType> documentDefinitions;
+    private final Set<NewDocumentType> globallyDistributedDocuments;
 
     /** The search nodes of this if it does not have an indexed cluster */
     List<SearchNode> nonIndexed = new ArrayList<>();
@@ -54,9 +55,12 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
     public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> {
 
         private final Map<String, NewDocumentType> documentDefinitions;
+        private final Set<NewDocumentType> globallyDistributedDocuments;
 
-        public Builder(Map<String, NewDocumentType> documentDefinitions) {
+        public Builder(Map<String, NewDocumentType> documentDefinitions,
+                       Set<NewDocumentType> globallyDistributedDocuments) {
             this.documentDefinitions = documentDefinitions;
+            this.globallyDistributedDocuments = globallyDistributedDocuments;
         }
 
         @Override
@@ -65,7 +69,8 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
             String clusterName = ContentCluster.getClusterName(clusterElem);
             Boolean flushOnShutdown = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown");
 
-            ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, flushOnShutdown != null ? flushOnShutdown : false);
+            ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, globallyDistributedDocuments,
+                    (flushOnShutdown != null ? flushOnShutdown : false));
 
             ModelElement tuning = clusterElem.getChildByPath("engine.proton.tuning");
             if (tuning != null) {
@@ -144,11 +149,14 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
 
     private ContentSearchCluster(AbstractConfigProducer parent,
                                  String clusterName,
-                                 Map<String, NewDocumentType> documentDefinitions, boolean flushOnShutdown)
+                                 Map<String, NewDocumentType> documentDefinitions,
+                                 Set<NewDocumentType> globallyDistributedDocuments,
+                                 boolean flushOnShutdown)
     {
         super(parent, "search");
         this.clusterName = clusterName;
         this.documentDefinitions = documentDefinitions;
+        this.globallyDistributedDocuments = globallyDistributedDocuments;
         this.flushOnShutdown = flushOnShutdown;
     }
 
@@ -263,9 +271,11 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0;
         for (NewDocumentType type : documentDefinitions.values()) {
             ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
-            ddbB.inputdoctypename(type.getFullName().getName())
+            String docTypeName = type.getFullName().getName();
+            ddbB.inputdoctypename(docTypeName)
                 .configid(getConfigId())
-                .visibilitydelay(visibilityDelay);
+                .visibilitydelay(visibilityDelay)
+                .global(isGloballyDistributed(docTypeName));
             if (hasIndexedCluster()) {
                 getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
             }
@@ -294,6 +304,10 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         }
     }
 
+    private boolean isGloballyDistributed(String docTypeName) {
+        return globallyDistributedDocuments.stream().anyMatch(type -> type.getFullName().getName().equals(docTypeName));
+    }
+
     @Override
     public void getConfig(DispatchConfig.Builder builder) {
         if (hasIndexedCluster()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 8d04540e4a6..8ce599590e8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -112,7 +112,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
                                                   globallyDistributedDocuments, routingSelection, redundancy,
                                                   ancestor.getRoot().getDeployState().getProperties().zone());
             c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml());
-            c.search = new ContentSearchCluster.Builder(documentDefinitions).build(c, contentElement.getXml());
+            c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments).build(c, contentElement.getXml());
             c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c);
             c.storageNodes = new StorageCluster.Builder().build(c, w3cContentElement);
             c.distributorNodes = new DistributorCluster.Builder(c).build(c, w3cContentElement);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
index 8e15ca27a61..81d3837e077 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
@@ -42,7 +42,7 @@ public class IndexedSearchClusterChangeValidatorTest {
 
         public static VespaModel newOneDocModel(String sdContent) {
             return new ApplicationPackageBuilder().
-                    addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1"))).
+                    addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
                     addSearchDefinition(new SearchDefinitionBuilder().
                             name("d1").content(sdContent).build()).buildCreator().create();
         }
@@ -53,7 +53,7 @@ public class IndexedSearchClusterChangeValidatorTest {
 
         public static VespaModel newTwoDocModel(String d1Content, String d2Content) {
             return new ApplicationPackageBuilder().
-                    addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1", "d2"))).
+                    addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")).
                     addSearchDefinition(new SearchDefinitionBuilder().
                             name("d1").content(d1Content).build()).
                     addSearchDefinition(new SearchDefinitionBuilder().
@@ -67,8 +67,8 @@ public class IndexedSearchClusterChangeValidatorTest {
 
         public static VespaModel newTwoClusterModel(String d1Content, String d2Content) {
             return new ApplicationPackageBuilder().
-                    addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1"))).
-                    addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList("d2"))).
+                    addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
+                    addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")).
                     addSearchDefinition(new SearchDefinitionBuilder().
                             name("d1").content(d1Content).build()).
                     addSearchDefinition(new SearchDefinitionBuilder().
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index bdac69f2a9f..32688d1ec43 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -4,12 +4,11 @@ package com.yahoo.vespa.model.content;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
-import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
-import junit.framework.Assert;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster;
 import static junit.framework.TestCase.assertEquals;
@@ -29,8 +28,20 @@ public class ContentSearchClusterTest {
 
     private static ContentCluster createClusterWithTwoDocumentType() throws Exception {
         List<String> docTypes = Arrays.asList("foo", "bar");
-        return createCluster(new ContentClusterBuilder().docTypes(docTypes).getXml(),
-                ApplicationPackageUtils.generateSearchDefinitions(docTypes));
+        return createCluster(new ContentClusterBuilder().docTypes("foo", "bar").getXml(),
+                generateSearchDefinitions(docTypes));
+    }
+
+    private static ContentCluster createClusterWithGlobalType() throws Exception {
+        return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(new ContentClusterBuilder.DocType("global", true),
+                new ContentClusterBuilder.DocType("regular"))).getXml(),
+                generateSearchDefinitions(Arrays.asList("global", "regular")));
+    }
+
+    private static List<String> generateSearchDefinitions(List<String> docTypes) {
+        return docTypes.stream().
+                map(type -> new com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder().name(type).build()).
+                collect(Collectors.toList());
     }
 
     private static ProtonConfig getProtonConfig(ContentCluster cluster) {
@@ -69,4 +80,17 @@ public class ContentSearchClusterTest {
                 new ContentClusterBuilder().protonMemoryLimit(0.77).getXml());
     }
 
+    @Test
+    public void requireThatGloballyDistributedDocumentTypeIsTaggedAsSuch() throws Exception {
+        ProtonConfig cfg = getProtonConfig(createClusterWithGlobalType());
+        assertEquals(2, cfg.documentdb().size());
+        assertDocumentDb("global", true, cfg.documentdb(0));
+        assertDocumentDb("regular", false, cfg.documentdb(1));
+    }
+
+    private static void assertDocumentDb(String expName, boolean expGlobal, ProtonConfig.Documentdb db) {
+        assertEquals(expName, db.inputdoctypename());
+        assertEquals(expGlobal, db.global());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
index 313690d4555..004d92baf11 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
@@ -6,6 +6,7 @@ import com.yahoo.text.XML;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import org.w3c.dom.Document;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -18,10 +19,29 @@ import java.util.stream.Collectors;
  */
 public class ContentClusterBuilder {
 
+    public static class DocType {
+        private final String name;
+        private final boolean global;
+
+        public DocType(String name, boolean global) {
+            this.name = name;
+            this.global = global;
+        }
+
+        public DocType(String name) {
+            this(name, false);
+        }
+
+        public String toXml() {
+            return (global ? "<document mode='index' type='" + name + "' global='true'/>" :
+                    "<document mode='index' type='" + name + "'/>");
+        }
+    }
+
     private String name = "mycluster";
     private int redundancy = 1;
     private int searchableCopies = 1;
-    private List<String> docTypes = Arrays.asList("test");
+    private List<DocType> docTypes = Arrays.asList(new DocType("test"));
     private String groupXml = getSimpleGroupXml();
     private Optional<String> dispatchXml = Optional.empty();
     private Optional<Double> protonDiskLimit = Optional.empty();
@@ -45,7 +65,16 @@ public class ContentClusterBuilder {
         return this;
     }
 
-    public ContentClusterBuilder docTypes(List<String> docTypes) {
+    public ContentClusterBuilder docTypes(String ... docTypes) {
+        List<DocType> result = new ArrayList<>();
+        for (String type : docTypes) {
+            result.add(new DocType(type));
+        }
+        this.docTypes = result;
+        return this;
+    }
+
+    public ContentClusterBuilder docTypes(List<DocType> docTypes) {
         this.docTypes = docTypes;
         return this;
     }
@@ -78,7 +107,7 @@ public class ContentClusterBuilder {
         String xml = "<content version='1.0' id='" + name + "'>\n" +
                "  <redundancy>" + redundancy + "</redundancy>\n" +
                "  <documents>\n" +
-                docTypes.stream().map(type -> "    <document mode='index' type='" + type + "'/>\n").collect(Collectors.joining("\n")) +
+                docTypes.stream().map(type -> type.toXml()).collect(Collectors.joining("\n")) +
                "  </documents>\n" +
                "  <engine>\n" +
                "    <proton>\n" +