Merge pull request #27534 from vespa-engine/bratseth/streaming-searchble-copies

Set the recommended searchable copies with streaming

8 files changed, 167 insertions, 70 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java
index 82ad8e5d6e8..47024c1171c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java
@@ -35,7 +35,7 @@ public class RedundancyIncreaseValidator implements ChangeValidator {
     }
 
     private int redundancyOf(ContentCluster cluster) {
-        return cluster.redundancy().finalRedundancy();
+        return cluster.getRedundancy().finalRedundancy();
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java
index 5228610537f..2be0f0b8422 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.model.application.validation.first;
 
 import com.yahoo.config.application.api.ValidationId;
-import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.model.api.ConfigChangeAction;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.vespa.model.VespaModel;
@@ -10,7 +9,6 @@ import com.yahoo.vespa.model.application.validation.Validator;
 import com.yahoo.vespa.model.application.validation.change.ChangeValidator;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 
-import java.time.Instant;
 import java.util.List;
 import java.util.stream.Stream;
 
@@ -48,7 +46,7 @@ public class RedundancyValidator extends Validator implements ChangeValidator {
     }
 
     private boolean hasRedundancyOne(ContentCluster cluster) {
-        return cluster != null && cluster.redundancy().finalRedundancy() == 1 && cluster.redundancy().groups() == 1;
+        return cluster != null && cluster.getRedundancy().finalRedundancy() == 1 && cluster.getRedundancy().groups() == 1;
     }
 
     private void invalidRedundancy(ContentCluster cluster, DeployState deployState) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index ec7acaf819f..34ea41384bc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -270,6 +270,25 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
         clusters.put(sc.getClusterName(), sc);
     }
 
+    /**
+     * Returns whether the schemas in this cluster use streaming mode.
+     *
+     * @return True if this cluster only has schemas with streaming mode, False if it only has schemas
+     *         with indexing, null if it has both or none.
+     */
+    public Boolean isStreaming() {
+        boolean hasStreaming = false;
+        boolean hasIndexed = false;
+        for (var cluster : clusters.values()) {
+            if (cluster.isStreaming())
+                hasStreaming = true;
+            else
+                hasIndexed = true;
+        }
+        if (hasIndexed == hasStreaming) return null;
+        return hasStreaming;
+    }
+
     public List<SearchNode> getSearchNodes() {
         return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed;
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index 52b2ce06dfe..6078215f9b6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -171,12 +171,11 @@ public class StorageGroup {
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (obj instanceof StorageGroup) {
-            StorageGroup rhs = (StorageGroup)obj;
-            return this.index.equals(rhs.index) &&
-                    this.name.equals(rhs.name) &&
-                    this.partitions.equals(rhs.partitions);
+    public boolean equals(Object o) {
+        if (o instanceof StorageGroup other) {
+            return this.index.equals(other.index) &&
+                   this.name.equals(other.name) &&
+                   this.partitions.equals(other.partitions);
         }
         return false;
     }
@@ -208,9 +207,7 @@ public class StorageGroup {
             this.context = context;
         }
 
-        public StorageGroup buildRootGroup(DeployState deployState,
-                                           RedundancyBuilder redundancyBuilder,
-                                           ContentCluster owner) {
+        public StorageGroup buildRootGroup(DeployState deployState, ContentCluster owner, Boolean isStreaming) {
             try {
                 if (owner.isHosted())
                     validateRedundancyAndGroups(deployState.zone().environment());
@@ -229,7 +226,8 @@ public class StorageGroup {
                                             ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context)
                                             : groupBuilder.buildNonHosted(deployState, owner, Optional.empty());
 
-                Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(),
+                RedundancyBuilder redundancyBuilder = new RedundancyBuilder(clusterElement);
+                Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), isStreaming, storageGroup.subgroups.size(),
                                                                 storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(false));
                 owner.setRedundancy(redundancy);
                 if (storageGroup.partitions.isEmpty() && (redundancy.groups() > 1)) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 2592beca6c6..f792ac3a591 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -114,7 +114,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                     new SearchDefinitionBuilder().build(deployState.getDocumentModel().getDocumentManager(), documentsElement);
 
             String routingSelection = new DocumentSelectionBuilder().build(documentsElement);
-            RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement);
             Set<NewDocumentType> globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement);
 
             String clusterId = getClusterId(contentElement);
@@ -133,7 +132,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
             c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c);
             c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement);
             c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement);
-            c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c);
+            c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming());
             c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits);
             validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup);
             c.search.handleRedundancy(c.redundancy);
@@ -447,7 +446,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
 
     public final ContentSearchCluster getSearch() { return search; }
 
-    public Redundancy redundancy() { return redundancy; }
+    public Redundancy getRedundancy() { return redundancy; }
 
     public ContentCluster setRedundancy(Redundancy redundancy) {
         this.redundancy = redundancy;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
index e7bafdf52e4..d310db067a6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
@@ -20,7 +20,7 @@ public class RedundancyBuilder {
     // Always global (across groups)
     private Integer globalMinRedundancy = null;
 
-    RedundancyBuilder(ModelElement clusterXml) {
+    public RedundancyBuilder(ModelElement clusterXml) {
         ModelElement redundancyElement = clusterXml.child("redundancy");
         if (redundancyElement != null) {
             initialRedundancy = redundancyElement.integerAttribute("reply-after");
@@ -47,22 +47,40 @@ public class RedundancyBuilder {
             throw new IllegalArgumentException("Either <redundancy> or <min-redundancy> must be set");
     }
 
-    public Redundancy build(boolean isHosted, int subGroups, int leafGroups,  int totalNodes) {
+    /**
+     * @param isHosted
+     * @param isStreaming true if this cluster only has schemas with streaming mode, false if it only has schemas
+     *                    without streaming, null if it has both
+     * @param subGroups
+     * @param leafGroups
+     * @param totalNodes
+     * @return
+     */
+    public Redundancy build(boolean isHosted, Boolean isStreaming, int subGroups, int leafGroups,  int totalNodes) {
         if (isHosted) {
             if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy * leafGroups < globalMinRedundancy ))
                 initialRedundancy = finalRedundancy = (int)Math.ceil((double)globalMinRedundancy / leafGroups);
             if (readyCopies == null) {
-                if (leafGroups > 1)
-                    readyCopies = 1;
-                else
-                    readyCopies = finalRedundancy > 1 ? 2 : 1;
+                if (isStreaming == Boolean.TRUE) {
+                    readyCopies = finalRedundancy;
+                }
+                else { // If isStreaming is null (mixed mode cluster) there are no good options ...
+                    if (leafGroups > 1)
+                        readyCopies = 1;
+                    else
+                        readyCopies = finalRedundancy > 1 ? 2 : 1;
+                }
             }
             return new Redundancy(initialRedundancy, finalRedundancy, readyCopies, leafGroups, totalNodes);
         } else {
             if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy < globalMinRedundancy))
                 initialRedundancy = finalRedundancy = globalMinRedundancy;
-            if (readyCopies == null)
-                readyCopies = finalRedundancy > 1 ? Math.max(subGroups, 2) : 1;
+            if (readyCopies == null) {
+                if (isStreaming == Boolean.TRUE)
+                    readyCopies = finalRedundancy;
+                else // If isStreaming is null (mixed mode cluster) there are no good options ...
+                    readyCopies = finalRedundancy > 1 ? Math.max(subGroups, 2) : 1;
+            }
             subGroups = Math.max(1, subGroups);
             IndexedHierarchicDistributionValidator.validateThatLeafGroupsCountIsAFactorOfRedundancy(finalRedundancy, subGroups);
             IndexedHierarchicDistributionValidator.validateThatReadyCopiesIsCompatibleWithRedundancy(finalRedundancy, readyCopies, subGroups);
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 84804bc48fa..19fe9e0038d 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -1183,9 +1183,9 @@ public class ModelProvisioningTest {
 
         ContentCluster cluster = model.getContentClusters().get("bar");
         List<StorageGroup> subGroups = cluster.getRootGroup().getSubgroups();
-        assertEquals(2*3, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3
-        assertEquals(2*3, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4
-        assertEquals(2*3, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3
+        assertEquals(2*3, cluster.getRedundancy().effectiveInitialRedundancy()); // Reduced from 3*3
+        assertEquals(2*3, cluster.getRedundancy().effectiveFinalRedundancy()); // Reduced from 3*4
+        assertEquals(2*3, cluster.getRedundancy().effectiveReadyCopies()); // Reduced from 3*3
         assertEquals("2|2|*", cluster.getRootGroup().getPartitions().get()); // Reduced from 4|4|*
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(3, subGroups.size());
@@ -1257,9 +1257,9 @@ public class ModelProvisioningTest {
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
 
         ContentCluster cluster = model.getContentClusters().get("bar");
-        assertEquals(2, cluster.redundancy().effectiveInitialRedundancy());
-        assertEquals(2, cluster.redundancy().effectiveFinalRedundancy());
-        assertEquals(2, cluster.redundancy().effectiveReadyCopies());
+        assertEquals(2, cluster.getRedundancy().effectiveInitialRedundancy());
+        assertEquals(2, cluster.getRedundancy().effectiveFinalRedundancy());
+        assertEquals(2, cluster.getRedundancy().effectiveReadyCopies());
         assertEquals("1|*", cluster.getRootGroup().getPartitions().get());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(2, cluster.getRootGroup().getSubgroups().size());
@@ -1287,9 +1287,9 @@ public class ModelProvisioningTest {
 
         ContentCluster cluster = model.getContentClusters().get("bar");
         assertEquals(2, cluster.getStorageCluster().getChildren().size());
-        assertEquals(1, cluster.redundancy().effectiveInitialRedundancy());
-        assertEquals(1, cluster.redundancy().effectiveFinalRedundancy());
-        assertEquals(1, cluster.redundancy().effectiveReadyCopies());
+        assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy());
+        assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy());
+        assertEquals(1, cluster.getRedundancy().effectiveReadyCopies());
         assertEquals(2, cluster.getRootGroup().getNodes().size());
         assertEquals(0, cluster.getRootGroup().getSubgroups().size());
     }
@@ -1324,9 +1324,9 @@ public class ModelProvisioningTest {
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
 
         ContentCluster cluster = model.getContentClusters().get("bar");
-        assertEquals(4, cluster.redundancy().effectiveInitialRedundancy());
-        assertEquals(4, cluster.redundancy().effectiveFinalRedundancy());
-        assertEquals(4, cluster.redundancy().effectiveReadyCopies());
+        assertEquals(4, cluster.getRedundancy().effectiveInitialRedundancy());
+        assertEquals(4, cluster.getRedundancy().effectiveFinalRedundancy());
+        assertEquals(4, cluster.getRedundancy().effectiveReadyCopies());
         assertEquals(4, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size());
         assertEquals(4, cluster.getSearch().getIndexed().getSearchableCopies());
         assertFalse(cluster.getRootGroup().getPartitions().isPresent());
@@ -1368,9 +1368,9 @@ public class ModelProvisioningTest {
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
 
         ContentCluster cluster = model.getContentClusters().get("bar");
-        assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3
-        assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4
-        assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3
+        assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy()); // Reduced from 3*3
+        assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy()); // Reduced from 3*4
+        assertEquals(1, cluster.getRedundancy().effectiveReadyCopies()); // Reduced from 3*3
         assertFalse(cluster.getRootGroup().getPartitions().isPresent()); // 1 group - > flattened -> no distribution
         assertEquals(1, cluster.getRootGroup().getNodes().size());
         assertEquals(0, cluster.getRootGroup().getSubgroups().size());
@@ -1473,9 +1473,9 @@ public class ModelProvisioningTest {
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
 
         ContentCluster cluster = model.getContentClusters().get("bar");
-        assertEquals(1, cluster.redundancy().effectiveInitialRedundancy());
-        assertEquals(1, cluster.redundancy().effectiveFinalRedundancy());
-        assertEquals(1, cluster.redundancy().effectiveReadyCopies());
+        assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy());
+        assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy());
+        assertEquals(1, cluster.getRedundancy().effectiveReadyCopies());
 
         assertEquals(1, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size());
         assertFalse(cluster.getRootGroup().getPartitions().isPresent());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 4ce7119f5f7..73bbd6ee464 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -38,6 +38,7 @@ import com.yahoo.vespa.model.routing.DocumentProtocol;
 import com.yahoo.vespa.model.routing.Routing;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
+import com.yahoo.yolean.Exceptions;
 import org.junit.jupiter.api.Test;
 
 import java.util.Arrays;
@@ -470,7 +471,8 @@ public class ContentClusterTest extends ContentBaseTest {
             new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
             fail("Deploying without redundancy should fail");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Either <redundancy> or <min-redundancy> must be set"), e.getMessage());
+            assertEquals("In content cluster 'bar': Either <redundancy> or <min-redundancy> must be set",
+                         Exceptions.toMessageString(e));
         }
     }
 
@@ -478,12 +480,13 @@ public class ContentClusterTest extends ContentBaseTest {
     void testRedundancyFinalLessThanInitial() {
         try {
             parse(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                            "  <redundancy reply-after=\"4\">2</redundancy>\n" +
-                            "     <group>" +
-                            "       <node hostalias='node0' distribution-key='0' />" +
-                            "     </group>" +
-                            "</content>"
+                    """
+                            <content version="1.0" id="storage">
+                              <redundancy reply-after="4">2</redundancy>
+                              <group>
+                                <node hostalias='node0' distribution-key='0' />
+                              </group>
+                            </content>"""
             );
             fail("no exception thrown");
         } catch (Exception e) { /* ignore */
@@ -494,17 +497,18 @@ public class ContentClusterTest extends ContentBaseTest {
     void testReadyTooHigh() {
         try {
             parse(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                            "  <engine>" +
-                            "     <proton>" +
-                            "       <searchable-copies>3</searchable-copies>" +
-                            "     </proton>" +
-                            "  </engine>" +
-                            "  <redundancy>2</redundancy>\n" +
-                            "     <group>" +
-                            "       <node hostalias='node0' distribution-key='0' />" +
-                            "     </group>" +
-                            "</content>"
+                    """
+                            <content version="1.0" id="storage">
+                              <engine>
+                                <proton>
+                                  <searchable-copies>3</searchable-copies>
+                                </proton>
+                              </engine>
+                              <redundancy>2</redundancy>
+                              <group>
+                                <node hostalias='node0' distribution-key='0' />
+                              </group>
+                            </content>"""
             );
             fail("no exception thrown");
         } catch (Exception e) { /* ignore */
@@ -972,15 +976,17 @@ public class ContentClusterTest extends ContentBaseTest {
 
     @Test
     void reserved_document_name_throws_exception() {
-        String xml = "<content version=\"1.0\" id=\"storage\">" +
-                "  <redundancy>1</redundancy>" +
-                "  <documents>" +
-                "    <document type=\"true\" mode=\"index\"/>" +
-                "  </documents>" +
-                "  <group>" +
-                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
-                "  </group>" +
-                "</content>";
+        String xml = """
+                <content version="1.0" id="storage">
+                  <redundancy>1</redundancy>
+                  <documents>
+                    <document type="true" mode="index"/>
+                  </documents>
+                  <group>
+                    <node distribution-key="0" hostalias="mockhost"/>
+                  </group>
+                </content>
+                """;
 
         List<String> sds = ApplicationPackageUtils.generateSchemas("true");
         try {
@@ -991,6 +997,65 @@ public class ContentClusterTest extends ContentBaseTest {
         }
     }
 
+    @Test
+    void default_searchable_copies_indexing() {
+        String services = """
+                <content version="1.0" id="storage">
+                  <redundancy>3</redundancy>
+                  <documents>
+                    <document type="music" mode="index"/>
+                  </documents>
+                  <group>
+                    <node distribution-key="0" hostalias="mockhost"/>
+                    <node distribution-key="1" hostalias="mockhost"/>
+                    <node distribution-key="2" hostalias="mockhost"/>
+                  </group>
+                </content>
+                """;
+        var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("music")).create();
+        assertEquals(2, model.getContentClusters().get("storage").getRedundancy().readyCopies());
+    }
+
+    @Test
+    void default_searchable_copies_streaming() {
+        String services = """
+                <content version="1.0" id="storage">
+                  <redundancy>3</redundancy>
+                  <documents>
+                    <document type="mail" mode="streaming"/>
+                  </documents>
+                  <group>
+                    <node distribution-key="0" hostalias="mockhost"/>
+                    <node distribution-key="1" hostalias="mockhost"/>
+                    <node distribution-key="2" hostalias="mockhost"/>
+                  </group>
+                </content>
+                """;
+        var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("mail")).create();
+        assertEquals(3, model.getContentClusters().get("storage").getRedundancy().readyCopies());
+    }
+
+    /** Here there is no good choice. */
+    @Test
+    void default_searchable_copies_mixed() {
+        String services = """
+                <content version="1.0" id="storage">
+                  <redundancy>3</redundancy>
+                  <documents>
+                    <document type="music" mode="index"/>
+                    <document type="mail" mode="streaming"/>
+                  </documents>
+                  <group>
+                    <node distribution-key="0" hostalias="mockhost"/>
+                    <node distribution-key="1" hostalias="mockhost"/>
+                    <node distribution-key="2" hostalias="mockhost"/>
+                  </group>
+                </content>
+                """;
+        var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("music", "mail")).create();
+        assertEquals(2, model.getContentClusters().get("storage").getRedundancy().readyCopies());
+    }
+
     private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId,
                                                      List<String> defaultSpaceTypes, List<String> globalSpaceTypes) {
         AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId);