Add min-redundancy

13 files changed, 222 insertions, 48 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java
index b4a83efd2aa..31173f731e3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java
@@ -12,16 +12,13 @@ package com.yahoo.vespa.model.content;
  */
 public class IndexedHierarchicDistributionValidator {
 
-    private final String clusterName;
     private final StorageGroup rootGroup;
     private final Redundancy redundancy;
     private final DispatchTuning.DispatchPolicy dispatchPolicy;
 
-    public IndexedHierarchicDistributionValidator(String clusterName,
-                                                  StorageGroup rootGroup,
+    public IndexedHierarchicDistributionValidator(StorageGroup rootGroup,
                                                   Redundancy redundancy,
                                                   DispatchTuning.DispatchPolicy dispatchPolicy) {
-        this.clusterName = clusterName;
         this.rootGroup = rootGroup;
         this.redundancy = redundancy;
         this.dispatchPolicy = dispatchPolicy;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
index 84267854fbe..55d1cfbfad5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
@@ -18,11 +18,11 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig
 
     private final int groups;
 
-    /** The total number of nodes available in this cluster (assigned when this becomes known) */
+    /** The total number of nodes available in this cluster */
     private final int totalNodes;
 
-    public Redundancy(int initialRedundancy, int finalRedundancy, int readyCopies, int groups, int totalNodes) {
-        this.initialRedundancy = initialRedundancy;
+    public Redundancy(Integer initialRedundancy, int finalRedundancy, int readyCopies, int groups, int totalNodes) {
+        this.initialRedundancy = initialRedundancy != null ? initialRedundancy : finalRedundancy;
         this.finalRedundancy = finalRedundancy;
         this.readyCopies = readyCopies;
         this.groups = groups;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 3b01cd02b9c..2053ac93aba 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -537,7 +537,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce
         super.validate();
         if (search.usesHierarchicDistribution() && !isHosted) {
             // validate manually configured groups
-            new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.getDispatchPolicy()).validate();
+            new IndexedHierarchicDistributionValidator(rootGroup, redundancy, search.getIndexed().getTuning().dispatch.getDispatchPolicy()).validate();
         }
         new ReservedDocumentTypeNameValidator().validate(documentDefinitions);
         new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
index 853a90cd8fa..52966c45e75 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
@@ -7,12 +7,18 @@ import com.yahoo.vespa.model.content.Redundancy;
 
 /**
  * Builds redundancy config for a content cluster.
+ *
+ * @author bratseth
  */
 public class RedundancyBuilder {
 
-    private Integer initialRedundancy = 2;
-    private Integer finalRedundancy = 3;
-    private Integer readyCopies = 2;
+    // Per group on hosted, global otherwise
+    private Integer initialRedundancy = null;
+    private Integer finalRedundancy = null;
+    private Integer readyCopies = null;
+
+    // Always global (across groups)
+    private Integer globalMinRedundancy = null;
 
     RedundancyBuilder(ModelElement clusterXml) {
         ModelElement redundancyElement = clusterXml.child("redundancy");
@@ -27,23 +33,37 @@ public class RedundancyBuilder {
                     throw new IllegalArgumentException("Final redundancy must be higher than or equal to initial redundancy");
                 }
             }
-
             readyCopies = clusterXml.childAsInteger("engine.proton.searchable-copies");
-            if (readyCopies == null) {
-                readyCopies = Math.min(finalRedundancy, 2);
-            }
-            if (readyCopies > finalRedundancy) {
+            if (readyCopies != null && readyCopies > finalRedundancy)
                 throw new IllegalArgumentException("Number of searchable copies can not be higher than final redundancy");
-            }
         }
+
+        ModelElement minRedundancyElement = clusterXml.child("min-redundancy");
+        if (minRedundancyElement != null) {
+            globalMinRedundancy = (int)minRedundancyElement.asLong();
+        }
+
+        if (redundancyElement == null && minRedundancyElement == null)
+            throw new IllegalArgumentException("Either <redundancy> or <min-redundancy> must be set");
     }
+
     public Redundancy build(boolean isHosted, int subGroups, int leafGroups,  int totalNodes) {
         if (isHosted) {
+            if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy * leafGroups < globalMinRedundancy ))
+                initialRedundancy = finalRedundancy = (int)Math.ceil((double)globalMinRedundancy / leafGroups);
+            if (readyCopies == null)
+                readyCopies = leafGroups > 1 ? 1 : 2;
             return new Redundancy(initialRedundancy, finalRedundancy, readyCopies, leafGroups, totalNodes);
         } else {
+            if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy < globalMinRedundancy))
+                initialRedundancy = finalRedundancy = globalMinRedundancy;
+            if (readyCopies == null)
+                readyCopies = 2;
             subGroups = Math.max(1, subGroups);
             IndexedHierarchicDistributionValidator.validateThatLeafGroupsCountIsAFactorOfRedundancy(finalRedundancy, subGroups);
             IndexedHierarchicDistributionValidator.validateThatReadyCopiesIsCompatibleWithRedundancy(finalRedundancy, readyCopies, subGroups);
+            if (initialRedundancy == null)
+                initialRedundancy = finalRedundancy;
             return new Redundancy(initialRedundancy/subGroups, finalRedundancy/subGroups,
                                   readyCopies/subGroups, subGroups, totalNodes);
         }
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 703001f0107..84338d49314 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -9,6 +9,10 @@ Redundancy = element redundancy {
     xsd:nonNegativeInteger
 }
 
+MinRedundancy = element min-redundancy {
+    xsd:nonNegativeInteger
+}
+
 DistributionType = element distribution {
     attribute type { string "strict" | string "loose" | string "legacy" }
 }
@@ -104,8 +108,8 @@ Content = element content {
     attribute version { "1.0" } &
     attribute id { xsd:NCName }? &
     attribute distributor-base-port { xsd:unsignedShort }? &
-    # Mandatory
-    Redundancy &
+    Redundancy? &
+    MinRedundancy? &
     ContentSearch? &
     Dispatch? &
     ClusterTuning? &
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 6f8547c3701..4b6128ecc14 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -1510,6 +1510,7 @@ public class ModelProvisioningTest {
                       </nodes>
                    </container>
                    <content version='1.0' id='foo'>
+                     <redundancy>3</redundancy>
                       <documents>
                         <document type='type1' mode='index'/>
                       </documents>
@@ -1518,6 +1519,7 @@ public class ModelProvisioningTest {
                       </nodes>
                    </content>
                    <content version='1.0' id='bar'>
+                     <redundancy>3</redundancy>
                       <documents>
                         <document type='type1' mode='index'/>
                       </documents>
@@ -1555,6 +1557,7 @@ public class ModelProvisioningTest {
                 "      </nodes>" +
                 "   </container>" +
                 "   <content version='1.0' id='foo'>" +
+                "      <redundancy>2</redundancy>" +
                 "      <documents>" +
                 "        <document type='type1' mode='index'/>" +
                 "      </documents>" +
@@ -1584,6 +1587,7 @@ public class ModelProvisioningTest {
                 "      </nodes>" +
                 "   </container>" +
                 "   <content version='1.0' id='foo'>" +
+                "      <redundancy>2</redundancy>" +
                 "      <documents>" +
                 "        <document type='type1' mode='index'/>" +
                 "      </documents>" +
@@ -1615,6 +1619,7 @@ public class ModelProvisioningTest {
                         "      </nodes>" +
                         "   </container>" +
                         "   <content version='1.0' id='foo'>" +
+                        "      <redundancy>2</redundancy>" +
                         "      <documents>" +
                         "        <document type='type1' mode='index'/>" +
                         "      </documents>" +
@@ -1865,6 +1870,64 @@ public class ModelProvisioningTest {
         storage.getChildren().get("0").getConfig(builder);
     }
 
+    @Test
+    public void testMinRedundancyMetByGroups() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>" +
+                "<services>" +
+                "  <container version='1.0' id='container1'>" +
+                "     <nodes count='1'/>" +
+                "  </container>" +
+                "  <content version='1.0'>" +
+                "     <min-redundancy>2</min-redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='2' groups='2'/>" +
+                "   </content>" +
+                "</services>";
+        VespaModelTester tester = new VespaModelTester();
+        tester.setHosted(true);
+        tester.addHosts(6);
+        VespaModel model = tester.createModel(services, true);
+
+        var contentCluster = model.getContentClusters().get("content");
+        ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+        contentCluster.getSearch().getConfig(protonBuilder);
+        ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+        assertEquals(1, protonConfig.distribution().searchablecopies());
+        assertEquals(1, protonConfig.distribution().redundancy());
+    }
+
+    @Test
+    public void testMinRedundancyMetWithinGroup() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>" +
+                "<services>" +
+                "  <container version='1.0' id='container1'>" +
+                "     <nodes count='1'/>" +
+                "  </container>" +
+                "  <content version='1.0'>" +
+                "     <min-redundancy>2</min-redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='2' groups='1'/>" +
+                "   </content>" +
+                "</services>";
+        VespaModelTester tester = new VespaModelTester();
+        tester.setHosted(true);
+        tester.addHosts(6);
+        VespaModel model = tester.createModel(services, true);
+
+        var contentCluster = model.getContentClusters().get("content");
+        ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+        contentCluster.getSearch().getConfig(protonBuilder);
+        ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+        assertEquals(2, protonConfig.distribution().searchablecopies());
+        assertEquals(2, protonConfig.distribution().redundancy());
+    }
+
     /**
      * Deploying an application with "nodes count" standalone should give a single-node deployment,
      * also if the user has a lingering hosts file from running self-hosted.
@@ -1933,6 +1996,7 @@ public class ModelProvisioningTest {
                 "    <document-api/>" +
                 "  </container>" +
                 "  <content version='1.0' id='bar'>" +
+                "     <redundancy>3</redundancy>" +
                 "     <documents>" +
                 "       <document type='type1' mode='index'/>" +
                 "     </documents>" +
@@ -1975,6 +2039,7 @@ public class ModelProvisioningTest {
                 "    <document-api/>" +
                 "  </container>" +
                 "  <content version='1.0' id='bar'>" +
+                "     <redundancy>3</redundancy>" +
                 "     <documents>" +
                 "       <document type='type1' mode='index'/>" +
                 "     </documents>" +
@@ -2001,6 +2066,7 @@ public class ModelProvisioningTest {
                 "    <nodes><node hostalias='foo'/></nodes>"+
                 "  </container>" +
                 "  <content version='1.0' id='bar'>" +
+                "     <redundancy>3</redundancy>" +
                 "     <documents>" +
                 "       <document type='type1' mode='index'/>" +
                 "     </documents>" +
@@ -2352,6 +2418,7 @@ public class ModelProvisioningTest {
          String services = joinLines("<?xml version='1.0' encoding='utf-8' ?>",
                  "<services>",
                  "  <content version='1.0' id='test'>",
+                 "     <redundancy>2</redundancy>" +
                  "     <documents>",
                  "       <document type='type1' mode='index'/>",
                  "     </documents>",
@@ -2383,6 +2450,7 @@ public class ModelProvisioningTest {
         String services = joinLines("<?xml version='1.0' encoding='utf-8' ?>",
                 "<services>",
                 "  <content version='1.0' id='test'>",
+                "    <redundancy>1</redundancy>" +
                 "    <config name='vespa.config.search.core.proton'>",
                 "      <flush><memory><maxtlssize>2000</maxtlssize></memory></flush>",
                 "    </config>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index 32771832f86..c439c919a15 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -173,6 +173,7 @@ public class MetricsProxyContainerTest {
                            "        <nodes count='2'/>",
                            "    </container>",
                            "    <content id='my-content' version='1.0'>",
+                           "        <redundancy>2</redundancy>" +
                            "        <documents />",
                            "        <nodes count='2'/>",
                            "    </content>",
@@ -183,6 +184,7 @@ public class MetricsProxyContainerTest {
         return String.join("\n",
                            "<services>",
                            "    <content version='1.0' id='my-content'>",
+                           "        <redundancy>1</redundancy>" +
                            "        <documents />",
                            "        <nodes count='1' />",
                            "    </content>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 88d3313f58e..b2175014cd4 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -161,6 +161,83 @@ public class ContentClusterTest extends ContentBaseTest {
     }
 
     @Test
+    void testMinRedundancy() {
+        {   // Groups ensures redundancy
+            ContentCluster cc = parse("""
+                                              <content version='1.0' id='storage'>
+                                                <documents/>
+                                                <min-redundancy>2</min-redundancy>
+                                                <group name='root' distribution-key='0'>"
+                                                  <distribution partitions='1|*'/>
+                                                  <group name='g0' distribution-key='0'>
+                                                    <node hostalias='mockhost' distribution-key='0'/>
+                                                    <node hostalias='mockhost' distribution-key='1'/>
+                                                  </group>
+                                                  <group name='g1' distribution-key='1'>
+                                                    <node hostalias='mockhost' distribution-key='2'/>
+                                                    <node hostalias='mockhost' distribution-key='3'/>
+                                                  </group>
+                                                </group>
+                                              </content>
+                                              """
+            );
+            ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+            cc.getSearch().getConfig(protonBuilder);
+            ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+            assertEquals(1, protonConfig.distribution().redundancy());
+            assertEquals(1, protonConfig.distribution().searchablecopies());
+        }
+
+        {  // Redundancy must be within group
+            ContentCluster cc = parse("""
+                                              <content version='1.0' id='storage'>
+                                                <documents/>
+                                                <min-redundancy>2</min-redundancy>
+                                                <nodes>
+                                                  <node hostalias='mockhost' distribution-key='0'/>
+                                                  <node hostalias='mockhost' distribution-key='1'/>
+                                                  <node hostalias='mockhost' distribution-key='2'/>
+                                                  <node hostalias='mockhost' distribution-key='3'/>
+                                                </nodes>
+                                              </content>
+                                              """
+            );
+            ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+            cc.getSearch().getConfig(protonBuilder);
+            ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+            assertEquals(2, protonConfig.distribution().redundancy());
+            assertEquals(2, protonConfig.distribution().searchablecopies());
+        }
+
+        {   // Multiple gropups but they do not ensure redundancy
+            ContentCluster cc = parse("""
+                                              <content version='1.0' id='storage'>
+                                                <documents/>
+                                                <min-redundancy>4</min-redundancy>
+                                                <group name='root' distribution-key='0'>"
+                                                  <distribution partitions='1|*'/>
+                                                  <group name='g0' distribution-key='0'>
+                                                    <node hostalias='mockhost' distribution-key='0'/>
+                                                    <node hostalias='mockhost' distribution-key='1'/>
+                                                  </group>
+                                                  <group name='g1' distribution-key='1'>
+                                                    <node hostalias='mockhost' distribution-key='2'/>
+                                                    <node hostalias='mockhost' distribution-key='3'/>
+                                                  </group>
+                                                </group>
+                                              </content>
+                                              """
+            );
+            ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+            cc.getSearch().getConfig(protonBuilder);
+            ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+            assertEquals(2, protonConfig.distribution().redundancy());
+            assertEquals(1, protonConfig.distribution().searchablecopies());
+        }
+
+    }
+
+    @Test
     void testNoId() {
         ContentCluster c = parse(
                 "<content version=\"1.0\">\n" +
@@ -177,32 +254,6 @@ public class ContentClusterTest extends ContentBaseTest {
     }
 
     @Test
-    void testRedundancyDefaults() {
-        ContentCluster cc = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <group>" +
-                        "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
-                        "    <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" +
-                        "    <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" +
-                        "  </group>" +
-                        "</content>"
-        );
-
-        DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder();
-        cc.getConfig(distributionBuilder);
-        DistributionConfig distributionConfig = distributionBuilder.build();
-        assertEquals(3, distributionConfig.cluster("storage").redundancy());
-
-        StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder();
-        cc.getConfig(builder);
-        StorDistributionConfig config = new StorDistributionConfig(builder);
-        assertEquals(2, config.initial_redundancy());
-        assertEquals(3, config.redundancy());
-        assertEquals(2, config.ready_copies());
-    }
-
-    @Test
     void testEndToEnd() {
         String xml =
                 "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
@@ -371,7 +422,7 @@ public class ContentClusterTest extends ContentBaseTest {
             new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
             fail("Deploying without redundancy should fail");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("missing required element \"redundancy\""), e.getMessage());
+            assertTrue(e.getMessage().contains("Either <redundancy> or <min-redundancy> must be set"), e.getMessage());
         }
     }
 
@@ -427,6 +478,7 @@ public class ContentClusterTest extends ContentBaseTest {
         {
             FleetcontrollerConfig config = getFleetControllerConfig(
                     "<content version=\"1.0\" id=\"storage\">\n" +
+                           "   <redundancy>3</redundancy>" +
                             "  <documents/>" +
                             "  <group>\n" +
                             "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -443,6 +495,7 @@ public class ContentClusterTest extends ContentBaseTest {
         {
             FleetcontrollerConfig config = getFleetControllerConfig(
                     "<content version=\"1.0\" id=\"storage\">\n" +
+                            "  <redundancy>3</redundancy>" +
                             "  <documents/>" +
                             "  <group>\n" +
                             "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -464,6 +517,7 @@ public class ContentClusterTest extends ContentBaseTest {
     {
         ContentCluster cluster = parse(
                 "<content version=\"1.0\" id=\"storage\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>\n" +
                         "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -475,6 +529,7 @@ public class ContentClusterTest extends ContentBaseTest {
 
         cluster = parse(
                 "<content version=\"1.0\" id=\"storage\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>\n" +
                         "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -490,6 +545,7 @@ public class ContentClusterTest extends ContentBaseTest {
     {
         ContentCluster cluster = parse(
                 "<content version=\"1.0\" id=\"storage\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>\n" +
                         "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -504,6 +560,7 @@ public class ContentClusterTest extends ContentBaseTest {
 
         cluster = parse(
                 "<content version=\"1.0\" id=\"storage\">\n" +
+                        "  <redundancy>2</redundancy>" +
                         "  <documents/>" +
                         "  <group>\n" +
                         "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -533,6 +590,7 @@ public class ContentClusterTest extends ContentBaseTest {
     {
         ContentCluster cluster = parse(
                 "<content version=\"1.0\" id=\"storage\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <engine>" +
                         "    <proton/>" +
@@ -564,6 +622,7 @@ public class ContentClusterTest extends ContentBaseTest {
     {
         ContentCluster cluster = parse(
                 "<content version=\"1.0\" id=\"test\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <engine>" +
                         "    <proton/>" +
@@ -658,6 +717,7 @@ public class ContentClusterTest extends ContentBaseTest {
         MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder();
 
         ContentCluster cluster = parse("<content version=\"1.0\" id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "  <group>\n" +
                 "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
@@ -796,6 +856,7 @@ public class ContentClusterTest extends ContentBaseTest {
 
     private static String oneNodeClusterXml() {
         return "<content version=\"1.0\" id=\"mockcluster\">" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "  <group>" +
                 "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -817,6 +878,7 @@ public class ContentClusterTest extends ContentBaseTest {
 
     private static ContentCluster createClusterWithFlushOnShutdownOverride(boolean flushOnShutdown, boolean isHostedVespa) throws Exception {
         return createOneNodeCluster("<content version=\"1.0\" id=\"mockcluster\">" +
+                "  <redundancy>1</redundancy>" +
                 "  <documents/>" +
                 "  <engine>" +
                 "    <proton>" +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
index 06f78a1c68d..ebdd1ed1576 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
@@ -39,6 +39,7 @@ public class DistributorTest {
 
         StorServerConfig.Builder builder = new StorServerConfig.Builder();
         parse("<content id=\"foofighters\"><documents/>\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <group>" +
                 "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
                 "  </group>" +
@@ -54,6 +55,7 @@ public class DistributorTest {
     void testRevertDefaultOffForSearch() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "  <group>" +
                 "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -67,6 +69,7 @@ public class DistributorTest {
     void testSplitAndJoin() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "    <tuning>\n" +
                 "      <bucket-splitting max-documents=\"2K\" max-size=\"25M\" minimum-bits=\"8\" />\n" +
@@ -90,6 +93,7 @@ public class DistributorTest {
     void testThatGroupsAreCountedInWhenComputingSplitBits() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         ContentCluster cluster = parseCluster("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "    <tuning>" +
                 "      <distribution type=\"legacy\"/>" +
@@ -146,6 +150,7 @@ public class DistributorTest {
     void testMaxMergesPerNode() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         DistributorCluster dcluster = parse("<content id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "  <group>" +
                 "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -157,6 +162,7 @@ public class DistributorTest {
 
         builder = new StorDistributormanagerConfig.Builder();
         dcluster = parse("<content id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents/>" +
                 "  <tuning>\n" +
                 "    <merges max-nodes-per-merge=\"4\"/>\n" +
@@ -174,6 +180,7 @@ public class DistributorTest {
     void testGarbageCollectionSetExplicitly() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents garbage-collection=\"true\">\n" +
                 "    <document type=\"music\"/>\n" +
                 "  </documents>\n" +
@@ -191,6 +198,7 @@ public class DistributorTest {
     void testGarbageCollectionInterval() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents garbage-collection=\"true\" garbage-collection-interval=\"30\">\n" +
                 "    <document type=\"music\"/>\n" +
                 "  </documents>\n" +
@@ -207,6 +215,7 @@ public class DistributorTest {
     void testGarbageCollectionOffByDefault() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"storage\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents>\n" +
                 "    <document type=\"music\"/>\n" +
                 "  </documents>\n" +
@@ -224,6 +233,7 @@ public class DistributorTest {
     void testComplexGarbageCollectionSelectionForIndexedSearch() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"foo\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents garbage-collection=\"true\" selection=\"true\">" +
                 "    <document type=\"music\" selection=\"music.year &lt; now()\"/>\n" +
                 "    <document type=\"movies\" selection=\"movies.year &lt; now() - 1200\"/>\n" +
@@ -244,6 +254,7 @@ public class DistributorTest {
     void testGarbageCollectionDisabledIfForced() {
         StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder();
         parse("<cluster id=\"foo\">\n" +
+                "  <redundancy>3</redundancy>" +
                 "  <documents selection=\"true\" garbage-collection=\"false\" garbage-collection-interval=\"30\">\n" +
                 "    <document type=\"music\" selection=\"music.year &lt; now()\"/>\n" +
                 "    <document type=\"movies\" selection=\"movies.year &lt; now() - 1200\"/>\n" +
@@ -263,6 +274,7 @@ public class DistributorTest {
         StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder();
         DistributorCluster cluster =
                 parse("<cluster id=\"storage\" distributor-base-port=\"14065\">" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>" +
                         "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -279,6 +291,7 @@ public class DistributorTest {
         StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder();
         DistributorCluster cluster =
                 parse("<cluster id=\"storage\">" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>" +
                         "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -298,6 +311,7 @@ public class DistributorTest {
 
     private String generateXmlForDocTypes(DocType... docTypes) {
         return "<content id='storage'>\n" +
+               "  <redundancy>3</redundancy>" +
                 DocType.listToXml(docTypes) +
                "\n</content>";
     }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java
index 95ab1b46401..956eabc6e1c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java
@@ -33,6 +33,7 @@ public class MonitoringConfigSnoopTest {
     private String getContent() {
         return (
                 "<content version='1.0' id='search'>"+
+                "    <redundancy>3</redundancy>" +
                 "    <documents/>"+
                 "    <nodes>"+
                 "      <node hostalias='mockhost' distribution-key='0' />"+
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 57ee15a1dc4..540f625cf2b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -76,6 +76,7 @@ public class StorageClusterTest {
     private static String cluster(String clusterName, String insert) {
         return joinLines(
                 "<content id=\"" + clusterName + "\">",
+                "<redundancy>3</redundancy>" +
                 "<documents/>",
                 insert,
                 group(),
@@ -338,6 +339,7 @@ public class StorageClusterTest {
     void testCapacity() {
         String xml = joinLines(
                 "<cluster id=\"storage\">",
+                "  <redundancy>2</redundancy>" +
                 "  <documents/>",
                 "  <group>",
                 "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>",
@@ -385,6 +387,7 @@ public class StorageClusterTest {
     void testGenericPersistenceTuning() {
         String xml = joinLines(
                 "<cluster id=\"storage\">",
+                "  <redundancy>2</redundancy>" +
                 "  <documents/>",
                 "  <engine>",
                 "    <fail-partition-on-error>true</fail-partition-on-error>",
@@ -411,6 +414,7 @@ public class StorageClusterTest {
     void requireThatUserDoesNotSpecifyBothGroupAndNodes() {
         String xml = joinLines(
                 "<cluster id=\"storage\">",
+                "  <redundancy>2</redundancy>" +
                 "  <documents/>",
                 "  <engine>",
                 "    <fail-partition-on-error>true</fail-partition-on-error>",
@@ -493,6 +497,7 @@ public class StorageClusterTest {
     void requireThatNestedGroupsRequireDistribution() {
         String xml = joinLines(
                 "<cluster id=\"storage\">",
+                "  <redundancy>2</redundancy>" +
                 "  <documents/>",
                 "  <group>",
                 "    <group distribution-key=\"0\" name=\"bar\">",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
index 5407298d0a5..0b1e7959503 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
@@ -21,9 +21,10 @@ public class StorageGroupTest {
     }
 
     @Test
-    void testSingleGroup() throws Exception {
+    void testSingleGroup() {
         ContentCluster cluster = parse(
                 "<content id=\"storage\">\n" +
+                        "  <redundancy>3</redundancy>" +
                         "  <documents/>" +
                         "  <group>\n" +
                         "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\n" +
diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml
index 2697cc871e2..1246f06c58f 100644
--- a/config-model/src/test/schema-test-files/services-hosted.xml
+++ b/config-model/src/test/schema-test-files/services-hosted.xml
@@ -30,7 +30,7 @@
   </content>
 
   <content id="ml" version="1.0">
-    <redundancy>2</redundancy>
+    <min-redundancy>2</min-redundancy>
     <nodes count="[10,20]" flavor="large" groups="[1,3]" group-size="[1,2]" vespamalloc-debug-stacktrace="proton">
       <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/>
     </nodes>