Draw cluster controllers from containers when necessary

17 files changed, 174 insertions, 80 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
index 99109a881a1..78e0a472e13 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
@@ -78,10 +78,7 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?>
         return multitenant;
     }
 
-    /**
-     * Returns the string representation of this Host object.
-     * @return The string representation of this Host object.
-     */
+    /** Returns the string representation of this Host object. */
     public String toString() {
         return "host '" + getHostName() + "'";
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
index 782487ee12c..002a649c9a9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -24,7 +24,7 @@ public class HostResource implements Comparable<HostResource> {
     private final Host host;
 
     // Map from "sentinel name" to service
-    private final Map<String,Service> services = new LinkedHashMap<>();
+    private final Map<String, Service> services = new LinkedHashMap<>();
     private final Map<Integer, Service> portDB = new LinkedHashMap<>();
 
     private int allocatedPorts = 0;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
index 1290b0b22d6..e39bddd5594 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
@@ -31,7 +31,7 @@ public class ClusterControllerContainer extends Container implements BundlesConf
     private final Set<String> bundles = new TreeSet<>();
 
     public ClusterControllerContainer(AbstractConfigProducer parent, int index, boolean runStandaloneZooKeeper) {
-        super(parent, "" + index);
+        super(parent, "" + index, index);
         this.index = index;
         addHandler(
                 new Handler(new ComponentModel(new BundleInstantiationSpecification(
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
index 65519637bd5..673d72c9c2f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
@@ -444,14 +444,13 @@ public class DomV20ClientsBuilder {
         }
 
         @Override
-        protected ContainerHttpGateway doBuild(AbstractConfigProducer parent,
-                Element spec) {
+        protected ContainerHttpGateway doBuild(AbstractConfigProducer parent, Element spec) {
             // TODO: remove port handling
             int port = 19020;
             if (spec != null && spec.hasAttribute("baseport")) {
                 port = Integer.parseInt(spec.getAttribute("baseport"));
             }
-            ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port);
+            ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port, index);
             List<Container> containers = new ArrayList<>();
             containers.add(httpGateway);
 
@@ -463,7 +462,7 @@ public class DomV20ClientsBuilder {
     /**
      * This class parses the feederoptions xml tag and produces Vespa config output.
      *
-     * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+     * @author Gunnar Gauslaa Bergem
      */
     private class FeederOptionsParser implements Serializable {
         private static final long serialVersionUID = 1L;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index a27739b42ef..84a9b5f7e88 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -17,7 +17,6 @@ import java.util.Optional;
  *
  * @author bratseth
  */
- // TODO: Use this for all nodes tags and unify with NodesUtil
 public class NodesSpecification {
 
     private final boolean dedicated;
@@ -101,5 +100,11 @@ public class NodesSpecification {
         return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor), groups, logger);
     }
 
+    @Override
+    public String toString() {
+        return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" +
+               (flavor.isPresent() ? " of flavor " + flavor.get() : "") +
+               (groups > 1 ? " in " + groups + " groups" : "");
+    }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index a13c7c9cec4..f1d274bc80c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -72,7 +72,10 @@ public class Container extends AbstractService implements
     private boolean httpServerEnabled = true;
     private boolean messageBusEnabled = true;
 
+    /** Whether this node has been marked as retired (e.g, will be removed) */
     private final boolean retired;
+    /** The index of this node. Non-critical: This is persisted on hosted, just a counter otherwise. */
+    private final int index;
 
     private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler");
     private final ComponentGroup<Component<?, ?>> components = new ComponentGroup(this, "components");
@@ -85,21 +88,22 @@ public class Container extends AbstractService implements
     private final int numRpcServerPorts = 2;
     private static String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage";
 
-    public Container(AbstractConfigProducer parent, String name) {
-        this(parent, name, Collections.<PortOverride>emptyList());
+    public Container(AbstractConfigProducer parent, String name, int index) {
+        this(parent, name, Collections.<PortOverride>emptyList(), index);
     }
-    public Container(AbstractConfigProducer parent, String name, boolean retired) {
-        this(parent, name, retired, Collections.<PortOverride>emptyList());
+    public Container(AbstractConfigProducer parent, String name, boolean retired, int index) {
+        this(parent, name, retired, Collections.<PortOverride>emptyList(), index);
     }
-    public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides) {
-        this(parent, name, false, portOverrides);
+    public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides, int index) {
+        this(parent, name, false, portOverrides, index);
     }
-    public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides) {
+    public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides, int index) {
         super(parent, name);
         this.name = name;
         this.parent = parent;
         this.portOverrides = Collections.unmodifiableList(new ArrayList<>(portOverrides));
         this.retired = retired;
+        this.index = index;
 
         if (getHttp() == null) {
             numHttpServerPorts = 2;
@@ -143,6 +147,9 @@ public class Container extends AbstractService implements
     public JettyHttpServer getDefaultHttpServer() {
         return defaultHttpServer;
     }
+    
+    /** Returns the index of this node. The index of a given node is stable through changes with best effort. */
+    public int index() { return index; }
 
     // We cannot set bindings yet, as baseport is not initialized
     public void addBuiltinHandlers() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java
index 88d4a0c8599..06aed4b3caa 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java
@@ -9,8 +9,8 @@ import com.yahoo.vespa.model.container.ContainerCluster;
  */
 public class ContainerHttpGateway extends Container {
 
-    public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort) {
-        super(parent, name);
+    public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort, int index) {
+        super(parent, name, index);
 
         // TODO: when this class is removed, all ports for the gateway will map to standard container ports
         //       this is just a tjuvtriks to keep the old gateway port allocation for now.
@@ -19,4 +19,5 @@ public class ContainerHttpGateway extends Container {
 
     @Override
     public String getServiceType() { return "container-httpgateway"; }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index b9611921fc9..4240ea40ebb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -368,14 +368,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     }
 
     private void addStandaloneNode(ContainerCluster cluster) {
-        Container container =  new Container(cluster, "standalone");
+        Container container =  new Container(cluster, "standalone", cluster.getContainers().size());
         cluster.addContainers(Collections.singleton(container));
     }
 
     private void addNodesFromXml(ContainerCluster cluster, Element spec) {
         Element nodesElement = XML.getChild(spec, "nodes");
         if (nodesElement == null) { // default single node on localhost
-            Container container = new Container(cluster, "container.0");
+            Container container = new Container(cluster, "container.0", 0);
             HostResource host = allocateSingleNodeHost(cluster, log);
             container.setHostResource(host);
             if ( ! container.isInitialized() ) // TODO: Fold this into initService
@@ -434,7 +434,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             Element spec, Element nodesElement, List<Container> result) {
         int nodeCount = 0;
         for (Element nodeElem: XML.getChildren(nodesElement, "node")) {
-            Container container = new ContainerServiceBuilder("container." + nodeCount).build(cluster, nodeElem);
+            Container container = new ContainerServiceBuilder("container." + nodeCount, nodeCount).build(cluster, nodeElem);
             result.add(container);
             ++nodeCount;
         }
@@ -447,7 +447,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty(), log);
             for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) {
                 String id = "container." + entry.getValue().index();
-                Container container = new Container(cluster, id, entry.getValue().retired());
+                Container container = new Container(cluster, id, entry.getValue().retired(), entry.getValue().index());
                 container.setHostResource(entry.getKey());
                 container.initService();
                 result.add(container);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
index 785ab1f7504..20a5c09b258 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
@@ -20,14 +20,16 @@ import java.util.logging.Logger;
 public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Container> {
 
     private final String id;
+    private final int index;
 
-    public ContainerServiceBuilder(String id) {
+    public ContainerServiceBuilder(String id, int index) {
         this.id = id;
+        this.index = index;
     }
 
     @Override
     protected Container doBuild(AbstractConfigProducer parent, Element nodeElem) {
-        return new Container(parent, id, readServerPortOverrides(nodeElem));
+        return new Container(parent, id, readServerPortOverrides(nodeElem), index);
     }
 
     private List<Container.PortOverride> readServerPortOverrides(Element spec) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
index 31a880ea29c..bb734efcad1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -50,7 +50,9 @@ public class Content extends ConfigModel {
 
     // Dependencies to other models
     private final AdminModel adminModel;
-    private final Collection<ContainerModel> containers; // to find or add the docproc container
+
+    // to find or add the docproc container and supplement cluster controllers with clusters having less then 3 nodes
+    private final Collection<ContainerModel> containers;
 
     @SuppressWarnings({ "UnusedDeclaration"}) // Created by reflection in ConfigModelRepo
     public Content(ConfigModelContext modelContext, AdminModel adminModel, Collection<ContainerModel> containers) {
@@ -75,14 +77,10 @@ public class Content extends ConfigModel {
 
         SimpleConfigProducer tldParent = new SimpleConfigProducer(indexedCluster, "tlds");
         for (ConfigModel model : modelRepo.asMap().values()) {
-            if ( ! (model instanceof ContainerModel)) {
-                continue;
-            }
+            if ( ! (model instanceof ContainerModel)) continue;
 
             ContainerCluster containerCluster = ((ContainerModel) model).getCluster();
-            if (containerCluster.getSearch() == null) {
-                continue; // this is not a qrs cluster
-            }
+            if (containerCluster.getSearch() == null) continue; // this is not a qrs cluster
 
             log.log(LogLevel.DEBUG, "Adding tlds for indexed cluster " + indexedCluster.getClusterName() + ", container cluster " + containerCluster.getName());
             indexedCluster.addTldsWithSameIdsAsContainers(tldParent, containerCluster);
@@ -104,9 +102,8 @@ public class Content extends ConfigModel {
 
     private static ChainSpecification getChainSpec(ComponentRegistry<DocprocChain> allChains, ComponentSpecification componentSpec) {
         DocprocChain docprocChain = allChains.getComponent(componentSpec);
-        if (docprocChain == null) {
-            throw new IllegalArgumentException("Chain '" + componentSpec + "' not found.");
-        }
+        if (docprocChain == null) throw new IllegalArgumentException("Chain '" + componentSpec + "' not found.");
+
         return docprocChain.getChainSpecification();
     }
 
@@ -114,12 +111,9 @@ public class Content extends ConfigModel {
         DocprocChain chainAlreadyPresent = containerCluster.getDocprocChains().allChains().
                 getComponent(new ComponentId(IndexingDocprocChain.NAME));
         if (chainAlreadyPresent != null) {
-            if (chainAlreadyPresent instanceof IndexingDocprocChain) {
-                return;
-            } else {
-                throw new IllegalArgumentException("A docproc chain may not have the ID '" +
-                                                   IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID.");
-            }
+            if (chainAlreadyPresent instanceof IndexingDocprocChain) return;
+            throw new IllegalArgumentException("A docproc chain may not have the ID '" +
+                                               IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID.");
         }
 
         containerCluster.getDocprocChains().add(new IndexingDocprocChain());
@@ -143,33 +137,23 @@ public class Content extends ConfigModel {
 
     public static List<Content> getContent(ConfigModelRepo pc) {
         List<Content> contents = new ArrayList<>();
-
-        for (ConfigModel model : pc.asMap().values()) {
-            if (model instanceof Content) {
+        for (ConfigModel model : pc.asMap().values())
+            if (model instanceof Content)
                 contents.add((Content)model);
-            }
-        }
-
         return contents;
     }
 
     public static List<AbstractSearchCluster> getSearchClusters(ConfigModelRepo pc) {
         List<AbstractSearchCluster> clusters = new ArrayList<>();
-
-        for (ContentCluster c : getContentClusters(pc)) {
+        for (ContentCluster c : getContentClusters(pc))
             clusters.addAll(c.getSearch().getClusters().values());
-        }
-
         return clusters;
     }
 
     public static List<ContentCluster> getContentClusters(ConfigModelRepo pc) {
        List<ContentCluster> clusters = new ArrayList<>();
-
-        for (Content c : getContent(pc)) {
+        for (Content c : getContent(pc))
             clusters.add(c.getCluster());
-        }
-
         return clusters;
     }
 
@@ -207,7 +191,7 @@ public class Content extends ConfigModel {
         @Override
         public void doBuild(Content content, Element xml, ConfigModelContext modelContext) {
             Admin admin = content.adminModel != null ? content.adminModel.getAdmin() : null; // This is null in tests only
-            content.cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(modelContext.getParentProducer(), xml);
+            content.cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(content.containers, modelContext.getParentProducer(), xml);
             buildIndexingClusters(content,
                                   modelContext.getConfigModelRepoAdder(),
                                   (ApplicationConfigProducerRoot)modelContext.getParentProducer());
@@ -310,8 +294,9 @@ public class Content extends ConfigModel {
             for (SearchNode searchNode : cluster.getSearchNodes()) {
                 HostResource host = searchNode.getHostResource();
                 if (!processedHosts.contains(host)) {
-                    String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index++);
-                    Container docprocService = new Container(indexingCluster, containerName);
+                    String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index);
+                    Container docprocService = new Container(indexingCluster, containerName, index);
+                    index++;
                     docprocService.setBasePort(host.nextAvailableBaseport(docprocService.getPortCount()));
                     docprocService.setHostResource(host);
                     docprocService.initService();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 052fc1487e2..f29cc07ee93 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
 import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
 import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
 import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
 import com.yahoo.vespa.model.content.*;
 import com.yahoo.vespa.model.content.engines.PersistenceEngine;
@@ -89,8 +90,10 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             this.admin = admin;
             this.deployLogger = deployLogger;
         }
+        
+        public ContentCluster build(Collection<ContainerModel> containers, 
+                                    AbstractConfigProducer ancestor, Element w3cContentElement) {
 
-        public ContentCluster build(AbstractConfigProducer ancestor, Element w3cContentElement) {
             ModelElement contentElement = new ModelElement(w3cContentElement);
 
             ModelElement documentsElement = contentElement.getChild("documents");
@@ -137,7 +140,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             AbstractConfigProducerRoot root = ancestor.getRoot();
             if (root == null) return c;
 
-            addClusterControllers(root, c.rootGroup, contentElement, c.clusterName, c);
+            addClusterControllers(containers, root, c.rootGroup, contentElement, c.clusterName, c);
             return c;
         }
 
@@ -253,7 +256,9 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             }
         }
 
-        private void addClusterControllers(AbstractConfigProducerRoot root, StorageGroup rootGroup, ModelElement contentElement, String contentClusterName, ContentCluster contentCluster) {
+        private void addClusterControllers(Collection<ContainerModel> containers, AbstractConfigProducerRoot root, 
+                                           StorageGroup rootGroup, ModelElement contentElement, 
+                                           String contentClusterName, ContentCluster contentCluster) {
             if (admin == null) return; // only in tests
             if (contentCluster.getPersistence() == null) return;
 
@@ -272,7 +277,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
                     NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers")).orElse(NodesSpecification.nonDedicated(3));
                 Collection<HostResource> hosts = nodesSpecification.isDedicated() ?
                                                  getControllerHosts(nodesSpecification, admin, clusterName) :
-                                                 drawContentHosts(nodesSpecification.count(), rootGroup);
+                                                 drawControllerHosts(nodesSpecification.count(), rootGroup, containers);
 
                 clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, true);
                 contentCluster.clusterControllers = clusterControllers;
@@ -312,19 +317,45 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
             return nodesSpecification.provision(admin.getHostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), Optional.empty(), deployLogger).keySet();
         }
 
-        private List<HostResource> drawContentHosts(int count, StorageGroup rootGroup) {
+        private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) {
             List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup);
-            if (hosts.size() < 3) // supply with containers if we don't have enough content hosts
-                hosts.addAll(drawContainerHosts(3 - hosts.size()));
+            if (hosts.size() < count) // supply with containers
+                hosts.addAll(drawContainerHosts(count - hosts.size(), containers));
             if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case)
                 hosts = hosts.subList(0, hosts.size()-1);
             return hosts;
         }
 
-        private List<HostResource> drawContainerHosts(int count) {
-            return new ArrayList<>(); // TODO
+        /**
+         * Draws <code>count</code> container nodes to use as cluster controllers, or as many as possible
+         * if less than <code>count</code> are available.
+         * 
+         * This will draw the same nodes each time it is 
+         * invoked if cluster names and node indexes are unchanged.
+         */
+        private List<HostResource> drawContainerHosts(int count, Collection<ContainerModel> containerClusters) {
+            if (containerClusters.isEmpty()) return Collections.emptyList();
+
+            List<HostResource> hosts = new ArrayList<>();
+            for (ContainerCluster cluster : clustersSortedByName(containerClusters))
+                hosts.addAll(hostResourcesSortedByIndex(cluster));
+            return hosts.subList(0, Math.min(hosts.size(), count));
         }
         
+        private List<ContainerCluster> clustersSortedByName(Collection<ContainerModel> containerModels) {
+            return containerModels.stream()
+                    .map(ContainerModel::getCluster)
+                    .sorted(Comparator.comparing(ContainerCluster::getName))
+                    .collect(Collectors.toList());
+        }
+
+        private List<HostResource> hostResourcesSortedByIndex(ContainerCluster cluster) {
+            return cluster.getContainers().stream()
+                    .sorted(Comparator.comparing(Container::index))
+                    .map(Container::getHostResource)
+                    .collect(Collectors.toList());
+        }
+
         /**
          * Draw <code>count</code> nodes from as many different content groups below this as possible.
          * This will only achieve maximum spread in the case where the groups are balanced and never on the same
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index b74398fc4ae..e25aa667f10 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -465,6 +465,38 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void testClusterControllersCanSupplementWithAllContainerClusters() throws ParseException {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                "<services>" +
+                "  <admin version='4.0'/>" +
+                "  <container version='1.0' id='foo1'>" +
+                "     <nodes count='2'/>" +
+                "  </container>" +
+                "  <container version='1.0' id='foo2'>" +
+                "     <nodes count='1'/>" +
+                "  </container>" +
+                "  <content version='1.0' id='bar'>" +
+                "     <redundancy>2</redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <controllers><nodes dedicated='false' count='5'/></controllers>" +
+                "     <nodes count='2'/>" +
+                "  </content>" +
+                "</services>";
+
+        int numberOfHosts = 5;
+        Hosts hosts = createHosts(numberOfHosts);
+        VespaModel model = createModel(services, hosts, true);
+        assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts));
+
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(5, clusterControllers.getContainers().size());
+    }
+
+    @Test
     public void testClusterControllersAreNotPlacedOnRetiredNodes() throws ParseException {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -602,6 +634,33 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void test2ContentNodesWithContainerClusterProducesMixedClusterControllerCluster() throws ParseException {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                "<services>" +
+                "  <container version='1.0' id='foo'>" +
+                "     <nodes count='3'/>" +
+                "  </container>" +
+                "  <content version='1.0' id='bar'>" +
+                "     <redundancy>2</redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='2'/>" +
+                "  </content>" +
+                "</services>";
+
+        int numberOfHosts = 5;
+        Hosts hosts = createHosts(numberOfHosts);
+        VespaModel model = createModel(services, hosts, true);
+        assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts));
+
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(3, clusterControllers.getContainers().size());
+    }
+
+    @Test
     public void testExplicitDedicatedClusterControllers() throws ParseException {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 3365177409a..c6de6835d49 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -19,6 +19,7 @@ import com.yahoo.vespa.model.container.search.ContainerSearch;
 import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
 import org.junit.Test;
 
+import java.util.Collections;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
@@ -164,7 +165,7 @@ public class ContainerClusterTest {
     }
 
     private static void addContainer(ContainerCluster cluster, String name, String hostName) {
-        Container container = new Container(cluster, name);
+        Container container = new Container(cluster, name, 0);
         container.setHostResource(new HostResource(new Host(null, hostName)));
         container.initService();
         cluster.addContainer(container);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 95761de7331..990c0e10927 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -17,6 +17,8 @@ import org.junit.Test;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 
+import java.util.Collections;
+
 import static org.junit.Assert.*;
 
 public class StorageClusterTest {
@@ -31,7 +33,7 @@ public class StorageClusterTest {
         );
         Document doc = XML.getDocument(xml);
         Element clusterElem = doc.getDocumentElement();
-        ContentCluster cluster = new ContentCluster.Builder(null, null).build(root, clusterElem);
+        ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, clusterElem);
 
         root.freezeModelTopology();
         return cluster.getStorageNodes();
@@ -182,7 +184,7 @@ public class StorageClusterTest {
                         "</cluster>"
         );
 
-        ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+        ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
 
         for (int i = 0; i < 3; ++i) {
             StorageNode node = cluster.getStorageNodes().getChildren().get("" + i);
@@ -205,7 +207,7 @@ public class StorageClusterTest {
                         "</cluster>"
         );
 
-        ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+        ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
 
         StorageNode node = cluster.getStorageNodes().getChildren().get("0");
 
@@ -249,7 +251,7 @@ public class StorageClusterTest {
                         "</cluster>"
         );
 
-        ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+        ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
 
         PersistenceConfig.Builder builder = new PersistenceConfig.Builder();
         cluster.getStorageNodes().getConfig(builder);
@@ -279,7 +281,7 @@ public class StorageClusterTest {
         );
 
         try {
-            new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+            new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
             assertTrue(false);
         } catch (Exception e) {
 
@@ -303,7 +305,7 @@ public class StorageClusterTest {
                 "</cluster>"
         );
         try {
-            new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+            new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
             fail("Did not get exception with duplicate group names");
         } catch (RuntimeException e) {
             assertEquals("Cluster 'storage' has multiple groups with name 'bar' in the same subgroup. " +
@@ -332,7 +334,7 @@ public class StorageClusterTest {
                 "</cluster>"
         );
         // Should not throw.
-        new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+        new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
     }
 
     @Test
@@ -351,7 +353,7 @@ public class StorageClusterTest {
                         "</cluster>"
         );
         try {
-            new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+            new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
             fail("Did not get exception with missing distribution element");
         } catch (RuntimeException e) {
             assertEquals("'distribution' attribute is required with multiple subgroups", e.getMessage());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
index da6636255ba..f1768ab6927 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java
@@ -8,6 +8,8 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import org.junit.Test;
 import org.w3c.dom.Document;
 
+import java.util.Collections;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -15,9 +17,10 @@ import static org.junit.Assert.assertTrue;
  * Test for storage groups.
  */
 public class StorageGroupTest {
+
     ContentCluster parse(String xml) {
         Document doc = XML.getDocument(xml);
-        return new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement());
+        return new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement());
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
index 7da263696c1..808c9a97287 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import org.w3c.dom.Document;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -72,7 +73,7 @@ public class ContentClusterBuilder {
 
     public ContentCluster build(MockRoot root) {
         Document doc = XML.getDocument(getXml());
-        return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement());
+        return new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, doc.getDocumentElement());
     }
 
     public String getXml() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
index c8b35bd0a37..f2f9ff482c1 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import org.w3c.dom.Document;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
@@ -45,7 +46,7 @@ public class ContentClusterUtils {
 
     public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception {
         Document doc = XML.getDocument(clusterXml);
-        return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement());
+        return new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, doc.getDocumentElement());
     }
 
     public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception {