Revert "Remove support for non-dedicated cc's on hosted"

7 files changed, 235 insertions, 46 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 59930a77a21..3722876a53f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -37,6 +37,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private Zone zone;
     private final Set<ContainerEndpoint> endpoints = Collections.emptySet();
     private boolean useDedicatedNodeForLogserver = false;
+    private boolean dedicatedClusterControllerCluster = true;
     private boolean useThreePhaseUpdates = false;
     private double defaultTermwiseLimit = 1.0;
     private String jvmGCOptions = null;
@@ -77,6 +78,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean isBootstrap() { return false; }
     @Override public boolean isFirstTimeDeployment() { return false; }
     @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; }
+    @Override public boolean dedicatedClusterControllerCluster() { return hostedVespa && dedicatedClusterControllerCluster; }
     @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
     @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
     @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; }
@@ -168,6 +170,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setDedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
+        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
+        return this;
+    }
+
     public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
         this.endpointCertificateSecrets = endpointCertificateSecrets;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 9d2ad10b6df..53df9045dfe 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -142,6 +142,7 @@ public class InMemoryProvisioner implements HostProvisioner {
     public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) {
         if (cluster.group().isPresent() && requested.groups() > 1)
             throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
+
         int capacity = failOnOutOfCapacity || required
                        ? requested.nodes()
                        : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
@@ -199,9 +200,7 @@ public class InMemoryProvisioner implements HostProvisioner {
         for (int i = allocation.size() - 1; i >= 0; i--) {
             NodeResources currentResources = allocation.get(0).advertisedResources();
             if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue;
-            if ( (! sharedHosts && ! currentResources.satisfies(requestedResources))
-                 ||
-                 (sharedHosts && ! currentResources.compatibleWith(requestedResources))) {
+            if ( ! currentResources.compatibleWith(requestedResources)) {
                 HostSpec removed = allocation.remove(i);
                 freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool
             }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index af99ee28e29..3f346f20144 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -78,7 +78,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
     private final FileDistributionConfigProducer fileDistribution;
     private final boolean multitenant;
 
-    public Admin(AbstractConfigProducer<?> parent,
+    public Admin(AbstractConfigProducer parent,
                  Monitoring monitoring,
                  Metrics metrics,
                  boolean multitenant,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index f8f10c33a43..79ce9343560 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -293,8 +293,26 @@ public class ContentCluster extends AbstractConfigProducer implements
             if (contentCluster.getPersistence() == null) return;
 
             ClusterControllerContainerCluster clusterControllers;
-            if (admin.multitenant()) {
-                clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
+
+            ContentCluster overlappingCluster = findOverlappingCluster(context.getParentProducer().getRoot(), contentCluster);
+
+            if (overlappingCluster != null && overlappingCluster.getClusterControllers() != null) {
+                // Borrow the cluster controllers of the other cluster in this case.
+                // This condition only occurs on non-hosted systems with a shared config server,
+                // a combination which only exists in system tests.
+                clusterControllers = overlappingCluster.getClusterControllers();
+            }
+            else if (admin.multitenant()) {
+                if (context.properties().dedicatedClusterControllerCluster())
+                    clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
+                else {
+                    clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone", deployState),
+                                                                  drawControllerHosts(3, rootGroup),
+                                                                  contentClusterName + "-controllers",
+                                                                  true,
+                                                                  context.getDeployState());
+                    contentCluster.clusterControllers = clusterControllers;
+                }
             }
             else {
                 clusterControllers = admin.getClusterControllers();
@@ -316,6 +334,21 @@ public class ContentCluster extends AbstractConfigProducer implements
             }
         }
 
+        /** Returns any other content cluster which shares nodes with this, or null if none are built */
+        private ContentCluster findOverlappingCluster(AbstractConfigProducerRoot root, ContentCluster contentCluster) {
+            for (ContentCluster otherContentCluster : root.getChildrenByTypeRecursive(ContentCluster.class)) {
+                if (otherContentCluster != contentCluster && overlaps(contentCluster, otherContentCluster))
+                    return otherContentCluster;
+            }
+            return null;
+        }
+
+        private boolean overlaps(ContentCluster c1, ContentCluster c2) {
+            Set<HostResource> c1Hosts = c1.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+            Set<HostResource> c2Hosts = c2.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+            return ! Sets.intersection(c1Hosts, c2Hosts).isEmpty();
+        }
+
         public static final NodeResources clusterControllerResources = new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
 
         private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin,
@@ -342,6 +375,49 @@ public class ContentCluster extends AbstractConfigProducer implements
             return admin.getClusterControllers();
         }
 
+        private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup) {
+            List<HostResource> hosts = drawControllerHosts(count, false, rootGroup);
+            List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup);
+
+            // preserve the cluster state in case all pre-existing controllers are on retired nodes
+            List<HostResource> all = new ArrayList<>(hosts);
+            all.addAll(retiredHosts);
+            return all;
+        }
+
+        private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup) {
+            List<HostResource> hosts = drawContentHostsRecursively(count, retired, rootGroup);
+            if (hosts.size() % 2 == 0 && ! hosts.isEmpty()) // ZK clusters of even sizes are less available (even in the size=2 case)
+                hosts = hosts.subList(0, hosts.size()-1);
+            return hosts;
+        }
+
+        /**
+         * Draw <code>count</code> nodes from as many different content groups below this as possible.
+         * This will only achieve maximum spread in the case where the groups are balanced and never on the same
+         * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy.
+         */
+        // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes
+        //       which overlaps with previously drawn nodes as that will prevent rolling upgrade
+        private List<HostResource> drawContentHostsRecursively(int count, boolean retired, StorageGroup group) {
+            Set<HostResource> hosts = new HashSet<>();
+            if (group.getNodes().isEmpty()) {
+                int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size());
+                for (StorageGroup subgroup : group.getSubgroups())
+                    hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, retired, subgroup));
+            }
+            else {
+                hosts.addAll(group.getNodes().stream()
+                     .filter(node -> node.isRetired() == retired)
+                     .map(StorageNode::getHostResource).collect(toList()));
+            }
+
+            List<HostResource> sortedHosts = new ArrayList<>(hosts);
+            sortedHosts.sort(HostResource::comparePrimarilyByIndexTo);
+            sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size()));
+            return sortedHosts;
+        }
+
         private ClusterControllerContainerCluster createClusterControllers(AbstractConfigProducer<?> parent,
                                                                            Collection<HostResource> hosts,
                                                                            String name,
@@ -349,10 +425,13 @@ public class ContentCluster extends AbstractConfigProducer implements
                                                                            DeployState deployState) {
             var clusterControllers = new ClusterControllerContainerCluster(parent, name, name, deployState);
             List<ClusterControllerContainer> containers = new ArrayList<>();
+            // Add a cluster controller on each config server (there is always at least one).
             if (clusterControllers.getContainers().isEmpty()) {
                 int index = 0;
                 for (HostResource host : hosts) {
-                    int ccIndex = host.spec().membership().map(ClusterMembership::index).orElse(index);
+                    int ccIndex = deployState.getProperties().dedicatedClusterControllerCluster()
+                                  ? host.spec().membership().map(ClusterMembership::index).orElse(index)
+                                  : index;
                     boolean retired = host.spec().membership().map(ClusterMembership::retired).orElse(false);
                     var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, multitenant, deployState, retired);
                     clusterControllerContainer.setHostResource(host);
@@ -432,7 +511,11 @@ public class ContentCluster extends AbstractConfigProducer implements
 
     public PersistenceEngine.PersistenceFactory getPersistence() { return persistenceFactory; }
 
-    /** Returns a list of th document types declared at the cluster level. */
+    /**
+     * The list of documentdefinitions declared at the cluster level.
+     *
+     * @return the set of documenttype names
+     */
     public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; }
 
     public boolean isGloballyDistributed(NewDocumentType docType) {
@@ -670,7 +753,6 @@ public class ContentCluster extends AbstractConfigProducer implements
      * a previous generation of it only by restarting the consuming processes.
      */
     public void setDeferChangesUntilRestart(boolean deferChangesUntilRestart) {
-        // TODO
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 73bcc3fe1d4..5a06379a1c2 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -264,7 +264,8 @@ public class ModelProvisioningTest {
                     "   </content>" +
                     "</services>";
             VespaModelTester tester = new VespaModelTester();
-            tester.addHosts(5);
+            tester.dedicatedClusterControllerCluster(false);
+            tester.addHosts(2);
             VespaModel model = tester.createModel(xmlWithNodes, true);
             assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
             assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size());
@@ -329,7 +330,8 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(5);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(2);
         VespaModel model = tester.createModel(xmlWithNodes, true);
 
         assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
@@ -581,8 +583,9 @@ public class ModelProvisioningTest {
                         "   </content>" +
                         "</services>";
 
-        int numberOfHosts = 67;
+        int numberOfHosts = 64;
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, true);
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
@@ -601,17 +604,23 @@ public class ModelProvisioningTest {
         assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts));
         assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
         assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
-        assertEquals(3, admin.getClusterControllers().getContainers().size());
+        assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers());
 
         // Check content clusters
         ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(3, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-54", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-51", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-48", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(9, cluster.getRootGroup().getSubgroups().size());
         assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex());
         assertEquals(3, cluster.getRootGroup().getSubgroups().get(0).getNodes().size());
         assertEquals(0, cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0"));
-        assertEquals("node-1-3-10-57", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1"));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2));
@@ -620,13 +629,13 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3"));
-        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4"));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5"));
         // ...
-        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
         // ...
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8"));
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3));
@@ -638,6 +647,12 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26"));
 
         cluster = model.getContentClusters().get("baz");
+        clusterControllers = cluster.getClusterControllers();
+        assertEquals(3, clusterControllers.getContainers().size());
+        assertEquals("baz-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-27", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-26", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-25", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(27, cluster.getRootGroup().getSubgroups().size());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
@@ -717,6 +732,41 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void testClusterControllersWithGroupSize2() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                "<services>" +
+                "  <admin version='4.0'/>" +
+                "  <container version='1.0' id='foo'>" +
+                "     <nodes count='10'/>" +
+                "  </container>" +
+                "  <content version='1.0' id='bar'>" +
+                "     <redundancy>2</redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='8' groups='4'/>" +
+                "  </content>" +
+                "</services>";
+
+        int numberOfHosts = 18;
+        VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(numberOfHosts);
+        VespaModel model = tester.createModel(services, true);
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
+
+        // Check content clusters
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-08", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-06", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-04", clusterControllers.getContainers().get(2).getHostName());
+    }
+
+    @Test
     public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -800,6 +850,32 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void test2ContentNodesProduces1ClusterController() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                        "<services>" +
+                        "  <content version='1.0' id='bar'>" +
+                        "     <redundancy>2</redundancy>" +
+                        "     <documents>" +
+                        "       <document type='type1' mode='index'/>" +
+                        "     </documents>" +
+                        "     <nodes count='2'/>" +
+                        "  </content>" +
+                        "</services>";
+
+        int numberOfHosts = 2;
+        VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(numberOfHosts);
+        VespaModel model = tester.createModel(services, true);
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
+
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(1, clusterControllers.getContainers().size());
+    }
+
+    @Test
     public void testDedicatedClusterControllers() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -989,11 +1065,16 @@ public class ModelProvisioningTest {
 
         int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, false);
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
 
         ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(1, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-01", clusterControllers.getContainers().get(0).getHostName());
         assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3
         assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4
         assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3
@@ -1148,14 +1229,14 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 21;
+        int totalHosts = 18;
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(new NodeResources(0.1, 0.2, 300, 0.3, NodeResources.DiskSpeed.slow), 1);// Logserver
         tester.addHosts(new NodeResources(0.1, 0.3, 1, 0.5), 2); // Slobrok
         tester.addHosts(new NodeResources(12, 10, 30, 0.3), 4); // Container
         tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo
         tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar
-        tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller
         VespaModel model = tester.createModel(services, true, 0);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1182,11 +1263,11 @@ public class ModelProvisioningTest {
 
         int totalHosts = 10;
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(new NodeResources(11.5,  10,           30, 0.3), 6);
-        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
-        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(totalHosts + 3, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
 
     @Test
@@ -1209,11 +1290,11 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 29;
+        int totalHosts = 26;
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(new NodeResources(13.5, 100,         1000, 0.3), 6);
-        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
-        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
         VespaModel model = tester.createModel(services, true, true);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1434,9 +1515,10 @@ public class ModelProvisioningTest {
                 "  </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(6);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(3);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(6, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(3, model.getRoot().hostSystem().getHosts().size());
         assertEquals(2, model.getAdmin().getSlobroks().size());
         assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
         assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes());
@@ -1551,7 +1633,7 @@ public class ModelProvisioningTest {
         assertEquals(1, model.getRoot().hostSystem().getHosts().size());
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1597,7 +1679,7 @@ public class ModelProvisioningTest {
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(1, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1706,10 +1788,10 @@ public class ModelProvisioningTest {
                 " </services>";
 
         VespaModel model = createNonProvisionedMultitenantModel(services);
-        assertEquals(1, model.getRoot().hostSystem().getHosts().size());
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1734,7 +1816,8 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(9);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(6);
         VespaModel model = tester.createModel(servicesXml, true);
 
         Map<String, Boolean> tests = Map.of("qrs", false,
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 4aadc0e3f05..219afb14ea5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1021,17 +1021,26 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5);
     }
 
-    void assertZookeeperServerImplementation(String expectedClassName,
-                                             ClusterControllerContainerCluster clusterControllerCluster) {
-        for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) {
+    void assertZookeeperServerImplementation(String expectedClassName) {
+        VespaModel model = createEnd2EndOneNode(new TestProperties().setMultitenant(true));
+
+        ContentCluster cc = model.getContentClusters().get("storage");
+        for (ClusterControllerContainer c : cc.getClusterControllers().getContainers()) {
             var builder = new ComponentsConfig.Builder();
             c.getConfig(builder);
             assertEquals(1, new ComponentsConfig(builder).components().stream()
-                                                         .filter(component -> component.classId().equals(expectedClassName))
-                                                         .count());
+                    .filter(component -> component.classId().equals(expectedClassName))
+                    .count());
         }
     }
 
+    @Test
+    public void reconfigurableZookeeperServerComponentsForClusterController() {
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer");
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer");
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl");
+    }
+
     private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) {
         VespaModel model = createEnd2EndOneNode(props);
 
@@ -1128,7 +1137,17 @@ public class ContentClusterTest extends ContentBaseTest {
         assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers());
         assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers());
 
+        Map<String, ContentCluster> clustersWithOwnCCC = createEnd2EndOneNode(new TestProperties().setMultitenant(true), twoContentServices).getContentClusters();
         ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers();
+        assertEquals("Union of components in own clusters is equal to those in shared cluster",
+                     clusterControllers.getAllComponents().stream()
+                                    .map(Component::getComponentId)
+                                    .collect(toList()),
+                     clustersWithOwnCCC.values().stream()
+                                       .flatMap(cluster -> Optional.ofNullable(cluster.getClusterControllers()).stream()
+                                                                   .flatMap(c -> c.getAllComponents().stream()))
+                                       .map(Component::getComponentId)
+                                       .collect(toList()));
 
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("storage").size());
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("dev-null").size());
@@ -1138,13 +1157,6 @@ public class ContentClusterTest extends ContentBaseTest {
         twoContentModel.getConfig(devNullBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-dev-null-configurer");
         assertEquals(0.618, storageBuilder.build().min_distributor_up_ratio(), 1e-9);
         assertEquals(0.418, devNullBuilder.build().min_distributor_up_ratio(), 1e-9);
-
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer",
-                                            clusterControllers);
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer",
-                                            clusterControllers);
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl",
-                                            clusterControllers);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index b72ae088484..eafbca09009 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -52,6 +52,7 @@ public class VespaModelTester {
     private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>();
     private ApplicationId applicationId = ApplicationId.defaultId();
     private boolean useDedicatedNodeForLogserver = false;
+    private boolean dedicatedClusterControllerCluster = true;
 
     public VespaModelTester() {
         this(new NullConfigModelRegistry());
@@ -101,6 +102,10 @@ public class VespaModelTester {
         this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver;
     }
 
+    public void dedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
+        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
+    }
+
     /** Creates a model which uses 0 as start index and fails on out of capacity */
     public VespaModel createModel(String services, String ... retiredHostNames) {
         return createModel(Zone.defaultZone(), services, true, retiredHostNames);
@@ -170,7 +175,8 @@ public class VespaModelTester {
                 .setMultitenant(true)
                 .setHostedVespa(hosted)
                 .setApplicationId(applicationId)
-                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver);
+                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver)
+                .setDedicatedClusterControllerCluster(dedicatedClusterControllerCluster);
 
         DeployState.Builder deployState = deployStatebuilder
                 .applicationPackage(appPkg)