Merge pull request #735 from yahoo/balder/distribute-initialcount-correctly-over-subdbs

Balder/distribute initialcount correctly over subdbs

4 files changed, 354 insertions, 280 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 56e275e74ac..7e24285c6fb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -31,6 +31,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
 
     /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */
     private IndexedSearchCluster indexedCluster;
+    private Redundancy redundancy;
 
     private final String clusterName;
     Map<String, NewDocumentType> documentDefinitions;
@@ -254,6 +255,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         if (usesHierarchicDistribution()) {
             indexedCluster.setMaxNodesDownPerFixedRow((redundancy.effectiveFinalRedundancy() / groupToSpecMap.size()) - 1);
         }
+        this.redundancy = redundancy;
     }
 
     @Override
@@ -287,6 +289,9 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         if (tuning != null) {
             tuning.getConfig(builder);
         }
+        if (redundancy != null) {
+            redundancy.getConfig(builder);
+        }
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
index 262c985e733..918bdcb8cb7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
@@ -2,19 +2,21 @@
 package com.yahoo.vespa.model.content;
 
 import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
 
 /**
  * Configuration of the redundancy of a content cluster.
  *
  * @author bratseth
  */
-public class Redundancy implements StorDistributionConfig.Producer {
+public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig.Producer {
 
     private final int initialRedundancy ;
     private final int finalRedundancy;
     private final int readyCopies;
 
     private int implicitGroups = 1;
+    private int explicitGroups = 1;
 
     /** The total number of nodes available in this cluster (assigned when this becomes known) */
     private int totalNodes = 0;
@@ -39,6 +41,7 @@ public class Redundancy implements StorDistributionConfig.Producer {
      * values returned in the config.
      */
     public void setImplicitGroups(int implicitGroups) { this.implicitGroups = implicitGroups; }
+    public void setExplicitGroups(int explicitGroups) { this.explicitGroups = explicitGroups; }
 
     public int initialRedundancy() { return initialRedundancy; }
     public int finalRedundancy() { return finalRedundancy; }
@@ -54,4 +57,11 @@ public class Redundancy implements StorDistributionConfig.Producer {
         builder.redundancy(effectiveFinalRedundancy());
         builder.ready_copies(effectiveReadyCopies());
     }
+    @Override
+    public void getConfig(ProtonConfig.Builder builder) {
+        ProtonConfig.Distribution.Builder distBuilder = new ProtonConfig.Distribution.Builder();
+        distBuilder.redundancy(finalRedundancy/explicitGroups);
+        distBuilder.searchablecopies(readyCopies/(explicitGroups*implicitGroups));
+        builder.distribution(distBuilder);
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index fa417b34844..ef05a3d6ff5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -104,8 +104,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             String routingSelection = new DocumentSelectionBuilder().build(contentElement.getChild("documents"));
             Redundancy redundancy = new RedundancyBuilder().build(contentElement);
 
-            ContentCluster c = new ContentCluster(ancestor, getClusterName(contentElement), documentDefinitions,
-                                                  routingSelection, redundancy);
+            ContentCluster c = new ContentCluster(ancestor, getClusterName(contentElement), documentDefinitions, routingSelection, redundancy);
             c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml());
             c.search = new ContentSearchCluster.Builder(documentDefinitions).build(c, contentElement.getXml());
             c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c);
@@ -113,6 +112,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             c.distributorNodes = new DistributorCluster.Builder(c).build(c, w3cContentElement);
             c.rootGroup = new StorageGroup.Builder(contentElement, c, deployLogger).buildRootGroup();
             validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup);
+            redundancy.setExplicitGroups(c.getRootGroup().getNumberOfLeafGroups());
             c.search.handleRedundancy(redundancy);
 
             IndexedSearchCluster index = c.search.getIndexed();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java
index bcb113687ec..126fcf7a583 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.config.content.core.StorServerConfig;
 import com.yahoo.vespa.config.content.FleetcontrollerConfig;
 import com.yahoo.vespa.config.content.StorDistributionConfig;
 import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
@@ -34,44 +35,99 @@ public class ClusterTest extends ContentBaseTest {
     }
 
     @Test
-    public void testRedundancy() {
-        StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder();
-        parse("" +
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                "  <documents/>" +
-                "  <engine>" +
-                "     <proton>" +
-                "       <searchable-copies>3</searchable-copies>" +
-                "     </proton>" +
-                "  </engine>" +
-                "  <redundancy reply-after=\"4\">5</redundancy>\n" +
-                "  <group>" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"3\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"4\"/>\"" +
-                "  </group>" +
-                "</content>"
-        ).getConfig(builder);
+    public void testHierarchicRedundancy() {
+        ContentCluster cc = parse("" +
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton>" +
+            "      <searchable-copies>3</searchable-copies>" +
+            "    </proton>" +
+            "  </engine>" +
+            "  <redundancy>15</redundancy>\n" +
+            "  <group name='root' distribution-key='0'>" +
+            "    <distribution partitions='1|1|*'/>" +
+            "    <group name='g-1' distribution-key='0'>" +
+            "      <node hostalias='mockhost' distribution-key='0'/>" +
+            "      <node hostalias='mockhost' distribution-key='1'/>" +
+            "      <node hostalias='mockhost' distribution-key='2'/>" +
+            "      <node hostalias='mockhost' distribution-key='3'/>" +
+            "      <node hostalias='mockhost' distribution-key='4'/>" +
+            "    </group>" +
+            "    <group name='g-2' distribution-key='1'>" +
+            "      <node hostalias='mockhost' distribution-key='5'/>" +
+            "      <node hostalias='mockhost' distribution-key='6'/>" +
+            "      <node hostalias='mockhost' distribution-key='7'/>" +
+            "      <node hostalias='mockhost' distribution-key='8'/>" +
+            "      <node hostalias='mockhost' distribution-key='9'/>" +
+            "    </group>" +
+            "    <group name='g-3' distribution-key='1'>" +
+            "      <node hostalias='mockhost' distribution-key='10'/>" +
+            "      <node hostalias='mockhost' distribution-key='11'/>" +
+            "      <node hostalias='mockhost' distribution-key='12'/>" +
+            "      <node hostalias='mockhost' distribution-key='13'/>" +
+            "      <node hostalias='mockhost' distribution-key='14'/>" +
+            "    </group>" +
+            "  </group>" +
+            "</content>"
+        );
+        StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder();
+        cc.getConfig(storBuilder);
+        StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder);
+        assertEquals(15, storConfig.initial_redundancy());
+        assertEquals(15, storConfig.redundancy());
+        assertEquals(3, storConfig.ready_copies());
+        ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+        cc.getSearch().getConfig(protonBuilder);
+        ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+        assertEquals(1, protonConfig.distribution().searchablecopies());
+        assertEquals(5, protonConfig.distribution().redundancy());
+    }
 
-        StorDistributionConfig config = new StorDistributionConfig(builder);
-        assertEquals(4, config.initial_redundancy());
-        assertEquals(5, config.redundancy());
-        assertEquals(3, config.ready_copies());
+    @Test
+    public void testRedundancy() {
+        ContentCluster cc = parse("" +
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton>" +
+            "      <searchable-copies>3</searchable-copies>" +
+            "    </proton>" +
+            "  </engine>" +
+            "  <redundancy reply-after='4'>5</redundancy>\n" +
+            "  <group>" +
+            "    <node hostalias='mockhost' distribution-key='0'/>" +
+            "    <node hostalias='mockhost' distribution-key='1'/>" +
+            "    <node hostalias='mockhost' distribution-key='2'/>" +
+            "    <node hostalias='mockhost' distribution-key='3'/>" +
+            "    <node hostalias='mockhost' distribution-key='4'/>" +
+            "  </group>" +
+            "</content>"
+        );
+        StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder();
+        cc.getConfig(storBuilder);
+        StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder);
+        assertEquals(4, storConfig.initial_redundancy());
+        assertEquals(5, storConfig.redundancy());
+        assertEquals(3, storConfig.ready_copies());
+        ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
+        cc.getSearch().getConfig(protonBuilder);
+        ProtonConfig protonConfig = new ProtonConfig(protonBuilder);
+        assertEquals(3, protonConfig.distribution().searchablecopies());
+        assertEquals(5, protonConfig.distribution().redundancy());
     }
 
     @Test
     public void testNoId() {
         ContentCluster c = parse(
-                "<content version=\"1.0\">\n" +
-                "  <redundancy>1</redundancy>\n" +
-                "  <documents/>" +
-                "  <redundancy reply-after=\"4\">5</redundancy>\n" +
-                "  <group>" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
-                "  </group>" +
-                "</content>"
+            "<content version=\"1.0\">\n" +
+            "  <redundancy>1</redundancy>\n" +
+            "  <documents/>" +
+            "  <redundancy reply-after=\"4\">5</redundancy>\n" +
+            "  <group>" +
+            "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
+            "  </group>" +
+            "</content>"
         );
 
         assertEquals("content", c.getName());
@@ -81,14 +137,14 @@ public class ClusterTest extends ContentBaseTest {
     public void testRedundancyDefaults() {
         StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder();
         parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                "  <documents/>" +
-                "  <group>" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" +
-                "    <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" +
-                "  </group>" +
-                "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <group>" +
+            "    <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" +
+            "    <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" +
+            "    <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" +
+            "  </group>" +
+            "</content>"
         ).getConfig(builder);
 
         StorDistributionConfig config = new StorDistributionConfig(builder);
@@ -99,39 +155,40 @@ public class ClusterTest extends ContentBaseTest {
 
     @Test
     public void testEndToEnd() throws Exception {
-        String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
-                "<services>\n" +
-                "\n" +
-                "  <admin version=\"2.0\">\n" +
-                "    <adminserver hostalias=\"configserver\" />\n" +
-                "    <logserver hostalias=\"logserver\" />\n" +
-                "    <slobroks>\n" +
-                "      <slobrok hostalias=\"configserver\" />\n" +
-                "      <slobrok hostalias=\"logserver\" />\n" +
-                "    </slobroks>\n" +
-                "    <cluster-controllers>\n" +
-                "      <cluster-controller hostalias=\"configserver\"/>" +
-                "      <cluster-controller hostalias=\"configserver2\"/>" +
-                "      <cluster-controller hostalias=\"configserver3\"/>" +
-                "    </cluster-controllers>\n" +
-                "  </admin>\n" +
-                "  <content version='1.0' id='bar'>" +
-                "     <redundancy>1</redundancy>\n" +
-                "     <documents>" +
-                "       <document type=\"type1\" mode=\"index\"/>\n" +
-                "       <document type=\"type2\" mode=\"index\"/>\n" +
-                "     </documents>\n" +
-                "     <group>" +
-                "       <node hostalias='node0' distribution-key='0' />" +
-                "     </group>" +
-                "    <tuning>" +
-                "      <cluster-controller>\n" +
-                "        <init-progress-time>34567</init-progress-time>" +
-                "      </cluster-controller>" +
-                "    </tuning>" +
-                "   </content>" +
-                "\n" +
-                "</services>";
+        String xml =
+            "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
+            "<services>\n" +
+            "\n" +
+            "  <admin version=\"2.0\">\n" +
+            "    <adminserver hostalias=\"configserver\" />\n" +
+            "    <logserver hostalias=\"logserver\" />\n" +
+            "    <slobroks>\n" +
+            "      <slobrok hostalias=\"configserver\" />\n" +
+            "      <slobrok hostalias=\"logserver\" />\n" +
+            "    </slobroks>\n" +
+            "    <cluster-controllers>\n" +
+            "      <cluster-controller hostalias=\"configserver\"/>" +
+            "      <cluster-controller hostalias=\"configserver2\"/>" +
+            "      <cluster-controller hostalias=\"configserver3\"/>" +
+            "    </cluster-controllers>\n" +
+            "  </admin>\n" +
+            "  <content version='1.0' id='bar'>" +
+            "     <redundancy>1</redundancy>\n" +
+            "     <documents>" +
+            "       <document type=\"type1\" mode=\"index\"/>\n" +
+            "       <document type=\"type2\" mode=\"index\"/>\n" +
+            "     </documents>\n" +
+            "     <group>" +
+            "       <node hostalias='node0' distribution-key='0' />" +
+            "     </group>" +
+            "    <tuning>" +
+            "      <cluster-controller>\n" +
+            "        <init-progress-time>34567</init-progress-time>" +
+            "      </cluster-controller>" +
+            "    </tuning>" +
+            "   </content>" +
+            "\n" +
+            "</services>";
 
         List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
         VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create();
@@ -183,32 +240,33 @@ public class ClusterTest extends ContentBaseTest {
 
     @Test
     public void testSearchTuning() throws Exception {
-        String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
-                "<services>\n" +
-                "\n" +
-                "  <admin version=\"2.0\">\n" +
-                "    <adminserver hostalias=\"node0\" />\n" +
-                "    <cluster-controllers>\n" +
-                "      <cluster-controller hostalias=\"node0\"/>" +
-                "    </cluster-controllers>\n" +
-                "  </admin>\n" +
-                "  <content version='1.0' id='bar'>" +
-                "     <redundancy>1</redundancy>\n" +
-                "     <documents>" +
-                "       <document type=\"type1\" mode='index'/>\n" +
-                "       <document type=\"type2\" mode='index'/>\n" +
-                "     </documents>\n" +
-                "     <group>" +
-                "       <node hostalias='node0' distribution-key='0'/>" +
-                "     </group>" +
-                "    <tuning>\n" +
-                "      <cluster-controller>" +
-                "        <init-progress-time>34567</init-progress-time>" +
-                "      </cluster-controller>" +
-                "    </tuning>" +
-                "   </content>" +
-                "\n" +
-                "</services>";
+        String xml =
+            "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
+            "<services>\n" +
+            "\n" +
+            "  <admin version=\"2.0\">\n" +
+            "    <adminserver hostalias=\"node0\" />\n" +
+            "    <cluster-controllers>\n" +
+            "      <cluster-controller hostalias=\"node0\"/>" +
+            "    </cluster-controllers>\n" +
+            "  </admin>\n" +
+            "  <content version='1.0' id='bar'>" +
+            "     <redundancy>1</redundancy>\n" +
+            "     <documents>" +
+            "       <document type=\"type1\" mode='index'/>\n" +
+            "       <document type=\"type2\" mode='index'/>\n" +
+            "     </documents>\n" +
+            "     <group>" +
+            "       <node hostalias='node0' distribution-key='0'/>" +
+            "     </group>" +
+            "    <tuning>\n" +
+            "      <cluster-controller>" +
+            "        <init-progress-time>34567</init-progress-time>" +
+            "      </cluster-controller>" +
+            "    </tuning>" +
+            "   </content>" +
+            "\n" +
+            "</services>";
 
         List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
@@ -232,21 +290,22 @@ public class ClusterTest extends ContentBaseTest {
 
     @Test
     public void testRedundancyRequired() throws Exception {
-        String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
-                "<services>\n" +
-                "\n" +
-                "  <admin version=\"2.0\">\n" +
-                "    <adminserver hostalias=\"node0\" />\n" +
-                "  </admin>\n" +
-                "  <content version='1.0' id='bar'>" +
-                "     <documents>" +
-                "       <document type=\"type1\" mode='index'/>\n" +
-                "     </documents>\n" +
-                "     <group>\n" +
-                "       <node hostalias='node0' distribution-key='0'/>\n" +
-                "     </group>\n" +
-                "   </content>\n" +
-                "</services>\n";
+        String xml =
+            "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" +
+            "<services>\n" +
+            "\n" +
+            "  <admin version=\"2.0\">\n" +
+            "    <adminserver hostalias=\"node0\" />\n" +
+            "  </admin>\n" +
+            "  <content version='1.0' id='bar'>" +
+            "     <documents>" +
+            "       <document type=\"type1\" mode='index'/>\n" +
+            "     </documents>\n" +
+            "     <group>\n" +
+            "       <node hostalias='node0' distribution-key='0'/>\n" +
+            "     </group>\n" +
+            "   </content>\n" +
+            "</services>\n";
 
         List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
         try{
@@ -261,12 +320,12 @@ public class ClusterTest extends ContentBaseTest {
     public void testRedundancyFinalLessThanInitial() {
         try {
             parse(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                    "  <redundancy reply-after=\"4\">2</redundancy>\n" +
-                    "     <group>" +
-                    "       <node hostalias='node0' distribution-key='0' />" +
-                    "     </group>" +
-                    "</content>"
+                "<content version=\"1.0\" id=\"storage\">\n" +
+                "  <redundancy reply-after=\"4\">2</redundancy>\n" +
+                "     <group>" +
+                "       <node hostalias='node0' distribution-key='0' />" +
+                "     </group>" +
+                "</content>"
             );
             fail("no exception thrown");
         } catch (Exception e) {
@@ -277,17 +336,17 @@ public class ClusterTest extends ContentBaseTest {
     public void testReadyTooHigh() {
         try {
             parse(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                            "  <engine>" +
-                            "     <proton>" +
-                            "       <searchable-copies>3</searchable-copies>" +
-                            "     </proton>" +
-                            "  </engine>" +
-                            "  <redundancy>2</redundancy>\n" +
-                            "     <group>" +
-                            "       <node hostalias='node0' distribution-key='0' />" +
-                            "     </group>" +
-                            "</content>"
+                "<content version=\"1.0\" id=\"storage\">\n" +
+                "  <engine>" +
+                "     <proton>" +
+                "       <searchable-copies>3</searchable-copies>" +
+                "     </proton>" +
+                "  </engine>" +
+                "  <redundancy>2</redundancy>\n" +
+                "     <group>" +
+                "       <node hostalias='node0' distribution-key='0' />" +
+                "     </group>" +
+                "</content>"
             );
             fail("no exception thrown");
         } catch (Exception e) {
@@ -308,12 +367,12 @@ public class ClusterTest extends ContentBaseTest {
     {
         {
             FleetcontrollerConfig config = getFleetControllerConfig(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                            "  <documents/>" +
-                            "  <group>\n" +
-                            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                            "  </group>\n" +
-                            "</content>"
+                "<content version=\"1.0\" id=\"storage\">\n" +
+                "  <documents/>" +
+                "  <group>\n" +
+                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+                "  </group>\n" +
+                "</content>"
             );
 
             assertEquals(0, config.min_storage_up_ratio(), 0.01);
@@ -324,17 +383,17 @@ public class ClusterTest extends ContentBaseTest {
 
         {
             FleetcontrollerConfig config = getFleetControllerConfig(
-                    "<content version=\"1.0\" id=\"storage\">\n" +
-                            "  <documents/>" +
-                            "  <group>\n" +
-                            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                            "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
-                            "    <node distribution-key=\"2\" hostalias=\"mockhost\"/>\n" +
-                            "    <node distribution-key=\"3\" hostalias=\"mockhost\"/>\n" +
-                            "    <node distribution-key=\"4\" hostalias=\"mockhost\"/>\n" +
-                            "    <node distribution-key=\"5\" hostalias=\"mockhost\"/>\n" +
-                            "  </group>\n" +
-                            "</content>"
+                "<content version=\"1.0\" id=\"storage\">\n" +
+                "  <documents/>" +
+                "  <group>\n" +
+                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+                "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
+                "    <node distribution-key=\"2\" hostalias=\"mockhost\"/>\n" +
+                "    <node distribution-key=\"3\" hostalias=\"mockhost\"/>\n" +
+                "    <node distribution-key=\"4\" hostalias=\"mockhost\"/>\n" +
+                "    <node distribution-key=\"5\" hostalias=\"mockhost\"/>\n" +
+                "  </group>\n" +
+                "</content>"
             );
 
             assertNotSame(0, config.min_storage_up_ratio());
@@ -345,12 +404,12 @@ public class ClusterTest extends ContentBaseTest {
     public void testImplicitDistributionBits()
     {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <group>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "  </group>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "</content>"
         );
 
         {
@@ -368,15 +427,15 @@ public class ClusterTest extends ContentBaseTest {
             assertEquals(8, config.minsplitcount());
         }
         cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <vds/>" +
-                        "  </engine>" +
-                        "  <group>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "  </group>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <vds/>" +
+            "  </engine>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "</content>"
         );
 
         {
@@ -399,15 +458,15 @@ public class ClusterTest extends ContentBaseTest {
     public void testExplicitDistributionBits()
     {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <group>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "  </group>\n" +
-                        "  <tuning>\n" +
-                        "    <distribution type=\"strict\"/>\n" +
-                        "  </tuning>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "  <tuning>\n" +
+            "    <distribution type=\"strict\"/>\n" +
+            "  </tuning>\n" +
+            "</content>"
         );
 
         {
@@ -425,18 +484,18 @@ public class ClusterTest extends ContentBaseTest {
             assertEquals(8, config.minsplitcount());
         }
         cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <vds/>" +
-                        "  </engine>" +
-                        "  <group>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "  </group>\n" +
-                        "  <tuning>\n" +
-                        "    <distribution type=\"loose\"/>\n" +
-                        "  </tuning>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <vds/>" +
+            "  </engine>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "  <tuning>\n" +
+            "    <distribution type=\"loose\"/>\n" +
+            "  </tuning>\n" +
+            "</content>"
         );
 
         {
@@ -459,16 +518,16 @@ public class ClusterTest extends ContentBaseTest {
     public void testGenerateSearchNodes()
     {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <proton/>" +
-                        "  </engine>" +
-                        "  <group>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
-                        "  </group>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton/>" +
+            "  </engine>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "</content>"
         );
 
         {
@@ -492,16 +551,16 @@ public class ClusterTest extends ContentBaseTest {
     public void testAlternativeNodeSyntax()
     {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"test\">\n" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <proton/>" +
-                        "  </engine>" +
-                        "  <nodes>\n" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                        "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
-                        "  </nodes>\n" +
-                        "</content>"
+            "<content version=\"1.0\" id=\"test\">\n" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton/>" +
+            "  </engine>" +
+            "  <nodes>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "    <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" +
+            "  </nodes>\n" +
+            "</content>"
         );
 
         StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder();
@@ -519,13 +578,13 @@ public class ClusterTest extends ContentBaseTest {
     public void testReadyWhenInitialOne() {
         StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder();
         parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                "  <documents/>" +
-                "  <redundancy>1</redundancy>\n" +
-                "  <group>\n" +
-                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
-                "  </group>" +
-                "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <redundancy>1</redundancy>\n" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+            "  </group>" +
+            "</content>"
         ).getConfig(builder);
 
         StorDistributionConfig config = new StorDistributionConfig(builder);
@@ -536,16 +595,16 @@ public class ClusterTest extends ContentBaseTest {
 
     public void testProvider(String tagName, StorServerConfig.Persistence_provider.Type.Enum type) {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                        "  <documents/>" +
-                        "  <redundancy>3</redundancy>" +
-                "  <engine>\n" +
-                "    <" + tagName + "/>\n" +
-                "  </engine>\n" +
-                "  <group>\n" +
-                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
-                "  </group>" +
-                "</content>"
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <documents/>" +
+            "  <redundancy>3</redundancy>" +
+            "  <engine>\n" +
+            "    <" + tagName + "/>\n" +
+            "  </engine>\n" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+            "  </group>" +
+            "</content>"
         );
 
         {
@@ -582,11 +641,11 @@ public class ClusterTest extends ContentBaseTest {
         MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder();
 
         ContentCluster cluster = parse("<content version=\"1.0\" id=\"storage\">\n" +
-                "  <documents/>" +
-                "  <group>\n" +
-                "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
-                "  </group>\n" +
-                "</content>"
+            "  <documents/>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" +
+            "  </group>\n" +
+            "</content>"
         );
         cluster.getConfig(builder);
 
@@ -642,34 +701,34 @@ public class ClusterTest extends ContentBaseTest {
     @Test
     public void testConfiguredMetrics() throws Exception {
         String xml = "" +
-                "<services>" +
-                "<content version=\"1.0\" id=\"storage\">\n" +
-                "  <redundancy>1</redundancy>\n" +
-                "  <documents>" +
-                "   <document type=\"type1\" mode='index'/>\n" +
-                "   <document type=\"type2\" mode='index'/>\n" +
-                "  </documents>" +
-                "  <group>\n" +
-                "    <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
-                "  </group>\n" +
-                "</content>" +
-                "<admin version=\"2.0\">" +
-                "  <logserver hostalias=\"node0\"/>" +
-                "  <adminserver hostalias=\"node0\"/>" +
-                "  <metric-consumers>" +
-                "    <consumer name=\"foobar\">" +
-                "      <metric name=\"storage.foo.bar\"/>" +
-                "    </consumer>" +
-                "    <consumer name=\"log\">" +
-                "      <metric name=\"extralogmetric\"/>" +
-                "      <metric name=\"extralogmetric3\"/>" +
-                "    </consumer>" +
-                "    <consumer name=\"fleetcontroller\">" +
-                "      <metric name=\"extraextra\"/>" +
-                "    </consumer>" +
-                "  </metric-consumers>" +
-                "</admin>" +
-                "</services>";
+            "<services>" +
+            "<content version=\"1.0\" id=\"storage\">\n" +
+            "  <redundancy>1</redundancy>\n" +
+            "  <documents>" +
+            "   <document type=\"type1\" mode='index'/>\n" +
+            "   <document type=\"type2\" mode='index'/>\n" +
+            "  </documents>" +
+            "  <group>\n" +
+            "    <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
+            "  </group>\n" +
+            "</content>" +
+            "<admin version=\"2.0\">" +
+            "  <logserver hostalias=\"node0\"/>" +
+            "  <adminserver hostalias=\"node0\"/>" +
+            "  <metric-consumers>" +
+            "    <consumer name=\"foobar\">" +
+            "      <metric name=\"storage.foo.bar\"/>" +
+            "    </consumer>" +
+            "    <consumer name=\"log\">" +
+            "      <metric name=\"extralogmetric\"/>" +
+            "      <metric name=\"extralogmetric3\"/>" +
+            "    </consumer>" +
+            "    <consumer name=\"fleetcontroller\">" +
+            "      <metric name=\"extraextra\"/>" +
+            "    </consumer>" +
+            "  </metric-consumers>" +
+            "</admin>" +
+            "</services>";
 
 
         List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
@@ -729,33 +788,33 @@ public class ClusterTest extends ContentBaseTest {
     @Test
     public void requireThatPreShutdownCommandIsSet() {
         ContentCluster cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <proton>" +
-                        "      <flush-on-shutdown>true</flush-on-shutdown>" +
-                        "    </proton>" +
-                        "  </engine>" +
-                        "  <group>" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
-                        "  </group>" +
-                        "</content>");
+            "<content version=\"1.0\" id=\"storage\">" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton>" +
+            "      <flush-on-shutdown>true</flush-on-shutdown>" +
+            "    </proton>" +
+            "  </engine>" +
+            "  <group>" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+            "  </group>" +
+            "</content>");
         assertThat(cluster.getSearch().getSearchNodes().size(), is(1));
         assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent());
 
         cluster = parse(
-                "<content version=\"1.0\" id=\"storage\">" +
-                        "  <documents/>" +
-                        "  <engine>" +
-                        "    <proton>" +
-                        "      <flush-on-shutdown>  \n " +
-                        "  true  </flush-on-shutdown>" +
-                        "    </proton>" +
-                        "  </engine>" +
-                        "  <group>" +
-                        "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
-                        "  </group>" +
-                        "</content>");
+            "<content version=\"1.0\" id=\"storage\">" +
+            "  <documents/>" +
+            "  <engine>" +
+            "    <proton>" +
+            "      <flush-on-shutdown>  \n " +
+            "  true  </flush-on-shutdown>" +
+            "    </proton>" +
+            "  </engine>" +
+            "  <group>" +
+            "    <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+            "  </group>" +
+            "</content>");
         assertThat(cluster.getSearch().getSearchNodes().size(), is(1));
         assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent());
     }