Revert "Bratseth/clustercontroller cleanup take 2"

12 files changed, 253 insertions, 64 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 59930a77a21..3722876a53f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -37,6 +37,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private Zone zone;
     private final Set<ContainerEndpoint> endpoints = Collections.emptySet();
     private boolean useDedicatedNodeForLogserver = false;
+    private boolean dedicatedClusterControllerCluster = true;
     private boolean useThreePhaseUpdates = false;
     private double defaultTermwiseLimit = 1.0;
     private String jvmGCOptions = null;
@@ -77,6 +78,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean isBootstrap() { return false; }
     @Override public boolean isFirstTimeDeployment() { return false; }
     @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; }
+    @Override public boolean dedicatedClusterControllerCluster() { return hostedVespa && dedicatedClusterControllerCluster; }
     @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
     @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
     @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; }
@@ -168,6 +170,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setDedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
+        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
+        return this;
+    }
+
     public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
         this.endpointCertificateSecrets = endpointCertificateSecrets;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 188504edd18..0f824fcacf9 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -142,6 +142,7 @@ public class InMemoryProvisioner implements HostProvisioner {
     public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) {
         if (cluster.group().isPresent() && requested.groups() > 1)
             throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
+
         int capacity = failOnOutOfCapacity || required
                        ? requested.nodes()
                        : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
@@ -199,9 +200,7 @@ public class InMemoryProvisioner implements HostProvisioner {
         for (int i = allocation.size() - 1; i >= 0; i--) {
             NodeResources currentResources = allocation.get(0).advertisedResources();
             if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue;
-            if ( (! sharedHosts && ! currentResources.satisfies(requestedResources))
-                 ||
-                 (sharedHosts && ! currentResources.compatibleWith(requestedResources))) {
+            if ( ! currentResources.compatibleWith(requestedResources)) {
                 HostSpec removed = allocation.remove(i);
                 freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool
             }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index af99ee28e29..3f346f20144 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -78,7 +78,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
     private final FileDistributionConfigProducer fileDistribution;
     private final boolean multitenant;
 
-    public Admin(AbstractConfigProducer<?> parent,
+    public Admin(AbstractConfigProducer parent,
                  Monitoring monitoring,
                  Metrics metrics,
                  boolean multitenant,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index c836209e8b3..79ce9343560 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -291,9 +291,28 @@ public class ContentCluster extends AbstractConfigProducer implements
                                            DeployState deployState) {
             if (admin == null) return; // only in tests
             if (contentCluster.getPersistence() == null) return;
+
             ClusterControllerContainerCluster clusterControllers;
-            if (context.properties().hostedVespa()) {
-                clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
+
+            ContentCluster overlappingCluster = findOverlappingCluster(context.getParentProducer().getRoot(), contentCluster);
+
+            if (overlappingCluster != null && overlappingCluster.getClusterControllers() != null) {
+                // Borrow the cluster controllers of the other cluster in this case.
+                // This condition only occurs on non-hosted systems with a shared config server,
+                // a combination which only exists in system tests.
+                clusterControllers = overlappingCluster.getClusterControllers();
+            }
+            else if (admin.multitenant()) {
+                if (context.properties().dedicatedClusterControllerCluster())
+                    clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
+                else {
+                    clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone", deployState),
+                                                                  drawControllerHosts(3, rootGroup),
+                                                                  contentClusterName + "-controllers",
+                                                                  true,
+                                                                  context.getDeployState());
+                    contentCluster.clusterControllers = clusterControllers;
+                }
             }
             else {
                 clusterControllers = admin.getClusterControllers();
@@ -303,8 +322,7 @@ public class ContentCluster extends AbstractConfigProducer implements
                         context.getDeployState().getDeployLogger().log(Level.INFO,
                                                                        "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly.");
                     }
-                    boolean runStandaloneZooKeeper = admin.multitenant(); // When multitenant we'll not run on config servers so we need to add a Zk instance
-                    clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", runStandaloneZooKeeper, context.getDeployState());
+                    clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", false, context.getDeployState());
                     admin.setClusterControllers(clusterControllers);
                 }
             }
@@ -316,6 +334,21 @@ public class ContentCluster extends AbstractConfigProducer implements
             }
         }
 
+        /** Returns any other content cluster which shares nodes with this, or null if none are built */
+        private ContentCluster findOverlappingCluster(AbstractConfigProducerRoot root, ContentCluster contentCluster) {
+            for (ContentCluster otherContentCluster : root.getChildrenByTypeRecursive(ContentCluster.class)) {
+                if (otherContentCluster != contentCluster && overlaps(contentCluster, otherContentCluster))
+                    return otherContentCluster;
+            }
+            return null;
+        }
+
+        private boolean overlaps(ContentCluster c1, ContentCluster c2) {
+            Set<HostResource> c1Hosts = c1.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+            Set<HostResource> c2Hosts = c2.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+            return ! Sets.intersection(c1Hosts, c2Hosts).isEmpty();
+        }
+
         public static final NodeResources clusterControllerResources = new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
 
         private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin,
@@ -342,19 +375,65 @@ public class ContentCluster extends AbstractConfigProducer implements
             return admin.getClusterControllers();
         }
 
+        private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup) {
+            List<HostResource> hosts = drawControllerHosts(count, false, rootGroup);
+            List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup);
+
+            // preserve the cluster state in case all pre-existing controllers are on retired nodes
+            List<HostResource> all = new ArrayList<>(hosts);
+            all.addAll(retiredHosts);
+            return all;
+        }
+
+        private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup) {
+            List<HostResource> hosts = drawContentHostsRecursively(count, retired, rootGroup);
+            if (hosts.size() % 2 == 0 && ! hosts.isEmpty()) // ZK clusters of even sizes are less available (even in the size=2 case)
+                hosts = hosts.subList(0, hosts.size()-1);
+            return hosts;
+        }
+
+        /**
+         * Draw <code>count</code> nodes from as many different content groups below this as possible.
+         * This will only achieve maximum spread in the case where the groups are balanced and never on the same
+         * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy.
+         */
+        // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes
+        //       which overlaps with previously drawn nodes as that will prevent rolling upgrade
+        private List<HostResource> drawContentHostsRecursively(int count, boolean retired, StorageGroup group) {
+            Set<HostResource> hosts = new HashSet<>();
+            if (group.getNodes().isEmpty()) {
+                int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size());
+                for (StorageGroup subgroup : group.getSubgroups())
+                    hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, retired, subgroup));
+            }
+            else {
+                hosts.addAll(group.getNodes().stream()
+                     .filter(node -> node.isRetired() == retired)
+                     .map(StorageNode::getHostResource).collect(toList()));
+            }
+
+            List<HostResource> sortedHosts = new ArrayList<>(hosts);
+            sortedHosts.sort(HostResource::comparePrimarilyByIndexTo);
+            sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size()));
+            return sortedHosts;
+        }
+
         private ClusterControllerContainerCluster createClusterControllers(AbstractConfigProducer<?> parent,
                                                                            Collection<HostResource> hosts,
                                                                            String name,
-                                                                           boolean runStandaloneZooKeeper,
+                                                                           boolean multitenant,
                                                                            DeployState deployState) {
             var clusterControllers = new ClusterControllerContainerCluster(parent, name, name, deployState);
             List<ClusterControllerContainer> containers = new ArrayList<>();
+            // Add a cluster controller on each config server (there is always at least one).
             if (clusterControllers.getContainers().isEmpty()) {
                 int index = 0;
                 for (HostResource host : hosts) {
-                    int ccIndex = host.spec().membership().map(ClusterMembership::index).orElse(index);
+                    int ccIndex = deployState.getProperties().dedicatedClusterControllerCluster()
+                                  ? host.spec().membership().map(ClusterMembership::index).orElse(index)
+                                  : index;
                     boolean retired = host.spec().membership().map(ClusterMembership::retired).orElse(false);
-                    var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, runStandaloneZooKeeper, deployState, retired);
+                    var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, multitenant, deployState, retired);
                     clusterControllerContainer.setHostResource(host);
                     clusterControllerContainer.initService(deployState.getDeployLogger());
                     clusterControllerContainer.setProp("clustertype", "admin");
@@ -432,7 +511,11 @@ public class ContentCluster extends AbstractConfigProducer implements
 
     public PersistenceEngine.PersistenceFactory getPersistence() { return persistenceFactory; }
 
-    /** Returns a list of th document types declared at the cluster level. */
+    /**
+     * The list of documentdefinitions declared at the cluster level.
+     *
+     * @return the set of documenttype names
+     */
     public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; }
 
     public boolean isGloballyDistributed(NewDocumentType docType) {
@@ -670,7 +753,6 @@ public class ContentCluster extends AbstractConfigProducer implements
      * a previous generation of it only by restarting the consuming processes.
      */
     public void setDeferChangesUntilRestart(boolean deferChangesUntilRestart) {
-        // TODO
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 86668fe3098..c6ddc6b0c36 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -264,7 +264,8 @@ public class ModelProvisioningTest {
                     "   </content>" +
                     "</services>";
             VespaModelTester tester = new VespaModelTester();
-            tester.addHosts(5);
+            tester.dedicatedClusterControllerCluster(false);
+            tester.addHosts(2);
             VespaModel model = tester.createModel(xmlWithNodes, true);
             assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
             assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size());
@@ -329,7 +330,8 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(5);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(2);
         VespaModel model = tester.createModel(xmlWithNodes, true);
 
         assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
@@ -581,8 +583,9 @@ public class ModelProvisioningTest {
                         "   </content>" +
                         "</services>";
 
-        int numberOfHosts = 67;
+        int numberOfHosts = 64;
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, true);
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
@@ -601,17 +604,23 @@ public class ModelProvisioningTest {
         assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts));
         assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
         assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
-        assertEquals(3, admin.getClusterControllers().getContainers().size());
+        assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers());
 
         // Check content clusters
         ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(3, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-54", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-51", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-48", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(9, cluster.getRootGroup().getSubgroups().size());
         assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex());
         assertEquals(3, cluster.getRootGroup().getSubgroups().get(0).getNodes().size());
         assertEquals(0, cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0"));
-        assertEquals("node-1-3-10-57", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1"));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2));
@@ -620,13 +629,13 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3"));
-        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4"));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5"));
         // ...
-        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
         // ...
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8"));
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3));
@@ -638,6 +647,12 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26"));
 
         cluster = model.getContentClusters().get("baz");
+        clusterControllers = cluster.getClusterControllers();
+        assertEquals(3, clusterControllers.getContainers().size());
+        assertEquals("baz-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-27", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-26", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-25", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(27, cluster.getRootGroup().getSubgroups().size());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
@@ -717,6 +732,41 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void testClusterControllersWithGroupSize2() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                "<services>" +
+                "  <admin version='4.0'/>" +
+                "  <container version='1.0' id='foo'>" +
+                "     <nodes count='10'/>" +
+                "  </container>" +
+                "  <content version='1.0' id='bar'>" +
+                "     <redundancy>2</redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='8' groups='4'/>" +
+                "  </content>" +
+                "</services>";
+
+        int numberOfHosts = 18;
+        VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(numberOfHosts);
+        VespaModel model = tester.createModel(services, true);
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
+
+        // Check content clusters
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-08", clusterControllers.getContainers().get(0).getHostName());
+        assertEquals("node-1-3-10-06", clusterControllers.getContainers().get(1).getHostName());
+        assertEquals("node-1-3-10-04", clusterControllers.getContainers().get(2).getHostName());
+    }
+
+    @Test
     public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -800,6 +850,32 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void test2ContentNodesProduces1ClusterController() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                        "<services>" +
+                        "  <content version='1.0' id='bar'>" +
+                        "     <redundancy>2</redundancy>" +
+                        "     <documents>" +
+                        "       <document type='type1' mode='index'/>" +
+                        "     </documents>" +
+                        "     <nodes count='2'/>" +
+                        "  </content>" +
+                        "</services>";
+
+        int numberOfHosts = 2;
+        VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(numberOfHosts);
+        VespaModel model = tester.createModel(services, true);
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
+
+        ContentCluster cluster = model.getContentClusters().get("bar");
+        ContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(1, clusterControllers.getContainers().size());
+    }
+
+    @Test
     public void testDedicatedClusterControllers() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -1055,11 +1131,16 @@ public class ModelProvisioningTest {
 
         int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, false);
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
 
         ContentCluster cluster = model.getContentClusters().get("bar");
+        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
+        assertEquals(1, clusterControllers.getContainers().size());
+        assertEquals("bar-controllers", clusterControllers.getName());
+        assertEquals("node-1-3-10-01", clusterControllers.getContainers().get(0).getHostName());
         assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3
         assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4
         assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3
@@ -1214,14 +1295,14 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 21;
+        int totalHosts = 18;
         VespaModelTester tester = new VespaModelTester();
+        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(new NodeResources(0.1, 0.2, 300, 0.3, NodeResources.DiskSpeed.slow), 1);// Logserver
         tester.addHosts(new NodeResources(0.1, 0.3, 1, 0.5), 2); // Slobrok
         tester.addHosts(new NodeResources(12, 10, 30, 0.3), 4); // Container
         tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo
         tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar
-        tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller
         VespaModel model = tester.createModel(services, true, 0);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1248,11 +1329,11 @@ public class ModelProvisioningTest {
 
         int totalHosts = 10;
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(new NodeResources(11.5,  10,           30, 0.3), 6);
-        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
-        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(totalHosts + 3, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
 
     @Test
@@ -1275,11 +1356,11 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 29;
+        int totalHosts = 26;
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(new NodeResources(13.5, 100,         1000, 0.3), 6);
-        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
-        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
         VespaModel model = tester.createModel(services, true, true);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1500,9 +1581,10 @@ public class ModelProvisioningTest {
                 "  </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(6);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(3);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(6, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(3, model.getRoot().hostSystem().getHosts().size());
         assertEquals(2, model.getAdmin().getSlobroks().size());
         assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
         assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes());
@@ -1617,7 +1699,7 @@ public class ModelProvisioningTest {
         assertEquals(1, model.getRoot().hostSystem().getHosts().size());
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1663,7 +1745,7 @@ public class ModelProvisioningTest {
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(1, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1772,10 +1854,10 @@ public class ModelProvisioningTest {
                 " </services>";
 
         VespaModel model = createNonProvisionedMultitenantModel(services);
-        assertEquals(1, model.getRoot().hostSystem().getHosts().size());
+        assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
+        ContainerCluster controller = content.getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1800,7 +1882,8 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(9);
+        tester.dedicatedClusterControllerCluster(false);
+        tester.addHosts(6);
         VespaModel model = tester.createModel(servicesXml, true);
 
         Map<String, Boolean> tests = Map.of("qrs", false,
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index 3f211a595b9..a11d23dbfbb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -144,7 +144,7 @@ public class MetricsProxyContainerTest {
     @Test
     public void vespa_services_config_has_all_services() {
         VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent());
-        assertEquals(7, vespaServicesConfig.service().size());
+        assertEquals(6, vespaServicesConfig.service().size());
 
         for (var service : vespaServicesConfig.service()) {
             if (service.configId().equals("admin/cluster-controllers/0")) {
@@ -185,7 +185,7 @@ public class MetricsProxyContainerTest {
     private static String servicesWithContent() {
         return String.join("\n",
                            "<services>",
-                           "    <admin version='2.0'>",
+                           "    <admin version='4.0'>",
                            "        <adminserver hostalias='node1'/>",
                            "    </admin>",
                            "    <content version='1.0' id='my-content'>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
index d92ace2939a..6ad74231cae 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
@@ -25,13 +25,13 @@ public class QuotaValidatorTest {
 
     @Test
     public void test_deploy_under_quota() {
-        var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(5, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         tester.deploy(null, getServices("testCluster", 5), Environment.prod, null);
     }
 
     @Test
     public void test_deploy_above_quota_clustersize() {
-        var tester = new ValidationTester(14, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(11, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 11), Environment.prod, null);
             fail();
@@ -42,32 +42,32 @@ public class QuotaValidatorTest {
 
     @Test
     public void test_deploy_above_quota_budget() {
-        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
         } catch (RuntimeException e) {
             assertEquals("Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!",
-                         ValidationTester.censorNumbers(e.getMessage()));
+                    ValidationTester.censorNumbers(e.getMessage()));
         }
     }
 
     @Test
     public void test_deploy_above_quota_budget_in_publiccd() {
-        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone));
+        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
         } catch (RuntimeException e) {
             assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!",
-                         ValidationTester.censorNumbers(e.getMessage()));
+                    ValidationTester.censorNumbers(e.getMessage()));
         }
     }
 
     @Test
     public void test_deploy_with_negative_budget() {
         var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1));
-        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
index 02f1195d50e..b7fa72d8a64 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
@@ -34,9 +34,9 @@ public class ValidationTester {
     private final TestProperties properties;
     private final InMemoryProvisioner hostProvisioner;
 
-    /** Creates a validation tester with 1 node available (in addition to cluster controllers) */
+    /** Creates a validation tester with 1 node available */
     public ValidationTester() {
-        this(4);
+        this(1);
     }
 
     /** Creates a validation tester with number of nodes available and the given test properties */
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
index f90762e8fc0..bade5a746f7 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
@@ -19,7 +19,7 @@ public class ClusterSizeReductionValidatorTest {
 
     @Test
     public void testSizeReductionValidation() {
-        ValidationTester tester = new ValidationTester(33);
+        ValidationTester tester = new ValidationTester(30);
 
         VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst();
         try {
@@ -45,7 +45,7 @@ public class ClusterSizeReductionValidatorTest {
 
     @Test
     public void testOverridingSizereductionValidation() {
-        ValidationTester tester = new ValidationTester(33);
+        ValidationTester tester = new ValidationTester(30);
 
         VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst();
         tester.deploy(previous, getServices(14), Environment.prod, sizeReductionOverride); // Allowed due to override
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
index 30dab17635a..392c37f50f3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.fail;
  */
 public class ContentClusterRemovalValidatorTest {
 
-    private final ValidationTester tester = new ValidationTester(5);
+    private final ValidationTester tester = new ValidationTester(2);
 
     @Test
     public void testContentRemovalValidation() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 4aadc0e3f05..219afb14ea5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1021,17 +1021,26 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5);
     }
 
-    void assertZookeeperServerImplementation(String expectedClassName,
-                                             ClusterControllerContainerCluster clusterControllerCluster) {
-        for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) {
+    void assertZookeeperServerImplementation(String expectedClassName) {
+        VespaModel model = createEnd2EndOneNode(new TestProperties().setMultitenant(true));
+
+        ContentCluster cc = model.getContentClusters().get("storage");
+        for (ClusterControllerContainer c : cc.getClusterControllers().getContainers()) {
             var builder = new ComponentsConfig.Builder();
             c.getConfig(builder);
             assertEquals(1, new ComponentsConfig(builder).components().stream()
-                                                         .filter(component -> component.classId().equals(expectedClassName))
-                                                         .count());
+                    .filter(component -> component.classId().equals(expectedClassName))
+                    .count());
         }
     }
 
+    @Test
+    public void reconfigurableZookeeperServerComponentsForClusterController() {
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer");
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer");
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl");
+    }
+
     private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) {
         VespaModel model = createEnd2EndOneNode(props);
 
@@ -1128,7 +1137,17 @@ public class ContentClusterTest extends ContentBaseTest {
         assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers());
         assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers());
 
+        Map<String, ContentCluster> clustersWithOwnCCC = createEnd2EndOneNode(new TestProperties().setMultitenant(true), twoContentServices).getContentClusters();
         ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers();
+        assertEquals("Union of components in own clusters is equal to those in shared cluster",
+                     clusterControllers.getAllComponents().stream()
+                                    .map(Component::getComponentId)
+                                    .collect(toList()),
+                     clustersWithOwnCCC.values().stream()
+                                       .flatMap(cluster -> Optional.ofNullable(cluster.getClusterControllers()).stream()
+                                                                   .flatMap(c -> c.getAllComponents().stream()))
+                                       .map(Component::getComponentId)
+                                       .collect(toList()));
 
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("storage").size());
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("dev-null").size());
@@ -1138,13 +1157,6 @@ public class ContentClusterTest extends ContentBaseTest {
         twoContentModel.getConfig(devNullBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-dev-null-configurer");
         assertEquals(0.618, storageBuilder.build().min_distributor_up_ratio(), 1e-9);
         assertEquals(0.418, devNullBuilder.build().min_distributor_up_ratio(), 1e-9);
-
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer",
-                                            clusterControllers);
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer",
-                                            clusterControllers);
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl",
-                                            clusterControllers);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index b72ae088484..eafbca09009 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -52,6 +52,7 @@ public class VespaModelTester {
     private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>();
     private ApplicationId applicationId = ApplicationId.defaultId();
     private boolean useDedicatedNodeForLogserver = false;
+    private boolean dedicatedClusterControllerCluster = true;
 
     public VespaModelTester() {
         this(new NullConfigModelRegistry());
@@ -101,6 +102,10 @@ public class VespaModelTester {
         this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver;
     }
 
+    public void dedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
+        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
+    }
+
     /** Creates a model which uses 0 as start index and fails on out of capacity */
     public VespaModel createModel(String services, String ... retiredHostNames) {
         return createModel(Zone.defaultZone(), services, true, retiredHostNames);
@@ -170,7 +175,8 @@ public class VespaModelTester {
                 .setMultitenant(true)
                 .setHostedVespa(hosted)
                 .setApplicationId(applicationId)
-                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver);
+                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver)
+                .setDedicatedClusterControllerCluster(dedicatedClusterControllerCluster);
 
         DeployState.Builder deployState = deployStatebuilder
                 .applicationPackage(appPkg)