Merge pull request #17544 from vespa-engine/bratseth/clustercontroller-cleanup-take-2

Bratseth/clustercontroller cleanup take 2

12 files changed, 64 insertions, 253 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 3722876a53f..59930a77a21 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -37,7 +37,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private Zone zone;
     private final Set<ContainerEndpoint> endpoints = Collections.emptySet();
     private boolean useDedicatedNodeForLogserver = false;
-    private boolean dedicatedClusterControllerCluster = true;
     private boolean useThreePhaseUpdates = false;
     private double defaultTermwiseLimit = 1.0;
     private String jvmGCOptions = null;
@@ -78,7 +77,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean isBootstrap() { return false; }
     @Override public boolean isFirstTimeDeployment() { return false; }
     @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; }
-    @Override public boolean dedicatedClusterControllerCluster() { return hostedVespa && dedicatedClusterControllerCluster; }
     @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
     @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
     @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; }
@@ -170,11 +168,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
-    public TestProperties setDedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
-        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
-        return this;
-    }
-
     public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
         this.endpointCertificateSecrets = endpointCertificateSecrets;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 0f824fcacf9..188504edd18 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -142,7 +142,6 @@ public class InMemoryProvisioner implements HostProvisioner {
     public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) {
         if (cluster.group().isPresent() && requested.groups() > 1)
             throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
-
         int capacity = failOnOutOfCapacity || required
                        ? requested.nodes()
                        : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
@@ -200,7 +199,9 @@ public class InMemoryProvisioner implements HostProvisioner {
         for (int i = allocation.size() - 1; i >= 0; i--) {
             NodeResources currentResources = allocation.get(0).advertisedResources();
             if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue;
-            if ( ! currentResources.compatibleWith(requestedResources)) {
+            if ( (! sharedHosts && ! currentResources.satisfies(requestedResources))
+                 ||
+                 (sharedHosts && ! currentResources.compatibleWith(requestedResources))) {
                 HostSpec removed = allocation.remove(i);
                 freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool
             }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 3f346f20144..af99ee28e29 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -78,7 +78,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
     private final FileDistributionConfigProducer fileDistribution;
     private final boolean multitenant;
 
-    public Admin(AbstractConfigProducer parent,
+    public Admin(AbstractConfigProducer<?> parent,
                  Monitoring monitoring,
                  Metrics metrics,
                  boolean multitenant,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 79ce9343560..c836209e8b3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -291,28 +291,9 @@ public class ContentCluster extends AbstractConfigProducer implements
                                            DeployState deployState) {
             if (admin == null) return; // only in tests
             if (contentCluster.getPersistence() == null) return;
-
             ClusterControllerContainerCluster clusterControllers;
-
-            ContentCluster overlappingCluster = findOverlappingCluster(context.getParentProducer().getRoot(), contentCluster);
-
-            if (overlappingCluster != null && overlappingCluster.getClusterControllers() != null) {
-                // Borrow the cluster controllers of the other cluster in this case.
-                // This condition only occurs on non-hosted systems with a shared config server,
-                // a combination which only exists in system tests.
-                clusterControllers = overlappingCluster.getClusterControllers();
-            }
-            else if (admin.multitenant()) {
-                if (context.properties().dedicatedClusterControllerCluster())
-                    clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
-                else {
-                    clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone", deployState),
-                                                                  drawControllerHosts(3, rootGroup),
-                                                                  contentClusterName + "-controllers",
-                                                                  true,
-                                                                  context.getDeployState());
-                    contentCluster.clusterControllers = clusterControllers;
-                }
+            if (context.properties().hostedVespa()) {
+                clusterControllers = getDedicatedSharedControllers(contentElement, admin, context, deployState);
             }
             else {
                 clusterControllers = admin.getClusterControllers();
@@ -322,7 +303,8 @@ public class ContentCluster extends AbstractConfigProducer implements
                         context.getDeployState().getDeployLogger().log(Level.INFO,
                                                                        "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly.");
                     }
-                    clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", false, context.getDeployState());
+                    boolean runStandaloneZooKeeper = admin.multitenant(); // When multitenant we'll not run on config servers so we need to add a Zk instance
+                    clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", runStandaloneZooKeeper, context.getDeployState());
                     admin.setClusterControllers(clusterControllers);
                 }
             }
@@ -334,21 +316,6 @@ public class ContentCluster extends AbstractConfigProducer implements
             }
         }
 
-        /** Returns any other content cluster which shares nodes with this, or null if none are built */
-        private ContentCluster findOverlappingCluster(AbstractConfigProducerRoot root, ContentCluster contentCluster) {
-            for (ContentCluster otherContentCluster : root.getChildrenByTypeRecursive(ContentCluster.class)) {
-                if (otherContentCluster != contentCluster && overlaps(contentCluster, otherContentCluster))
-                    return otherContentCluster;
-            }
-            return null;
-        }
-
-        private boolean overlaps(ContentCluster c1, ContentCluster c2) {
-            Set<HostResource> c1Hosts = c1.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
-            Set<HostResource> c2Hosts = c2.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
-            return ! Sets.intersection(c1Hosts, c2Hosts).isEmpty();
-        }
-
         public static final NodeResources clusterControllerResources = new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
 
         private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin,
@@ -375,65 +342,19 @@ public class ContentCluster extends AbstractConfigProducer implements
             return admin.getClusterControllers();
         }
 
-        private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup) {
-            List<HostResource> hosts = drawControllerHosts(count, false, rootGroup);
-            List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup);
-
-            // preserve the cluster state in case all pre-existing controllers are on retired nodes
-            List<HostResource> all = new ArrayList<>(hosts);
-            all.addAll(retiredHosts);
-            return all;
-        }
-
-        private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup) {
-            List<HostResource> hosts = drawContentHostsRecursively(count, retired, rootGroup);
-            if (hosts.size() % 2 == 0 && ! hosts.isEmpty()) // ZK clusters of even sizes are less available (even in the size=2 case)
-                hosts = hosts.subList(0, hosts.size()-1);
-            return hosts;
-        }
-
-        /**
-         * Draw <code>count</code> nodes from as many different content groups below this as possible.
-         * This will only achieve maximum spread in the case where the groups are balanced and never on the same
-         * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy.
-         */
-        // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes
-        //       which overlaps with previously drawn nodes as that will prevent rolling upgrade
-        private List<HostResource> drawContentHostsRecursively(int count, boolean retired, StorageGroup group) {
-            Set<HostResource> hosts = new HashSet<>();
-            if (group.getNodes().isEmpty()) {
-                int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size());
-                for (StorageGroup subgroup : group.getSubgroups())
-                    hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, retired, subgroup));
-            }
-            else {
-                hosts.addAll(group.getNodes().stream()
-                     .filter(node -> node.isRetired() == retired)
-                     .map(StorageNode::getHostResource).collect(toList()));
-            }
-
-            List<HostResource> sortedHosts = new ArrayList<>(hosts);
-            sortedHosts.sort(HostResource::comparePrimarilyByIndexTo);
-            sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size()));
-            return sortedHosts;
-        }
-
         private ClusterControllerContainerCluster createClusterControllers(AbstractConfigProducer<?> parent,
                                                                            Collection<HostResource> hosts,
                                                                            String name,
-                                                                           boolean multitenant,
+                                                                           boolean runStandaloneZooKeeper,
                                                                            DeployState deployState) {
             var clusterControllers = new ClusterControllerContainerCluster(parent, name, name, deployState);
             List<ClusterControllerContainer> containers = new ArrayList<>();
-            // Add a cluster controller on each config server (there is always at least one).
             if (clusterControllers.getContainers().isEmpty()) {
                 int index = 0;
                 for (HostResource host : hosts) {
-                    int ccIndex = deployState.getProperties().dedicatedClusterControllerCluster()
-                                  ? host.spec().membership().map(ClusterMembership::index).orElse(index)
-                                  : index;
+                    int ccIndex = host.spec().membership().map(ClusterMembership::index).orElse(index);
                     boolean retired = host.spec().membership().map(ClusterMembership::retired).orElse(false);
-                    var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, multitenant, deployState, retired);
+                    var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, runStandaloneZooKeeper, deployState, retired);
                     clusterControllerContainer.setHostResource(host);
                     clusterControllerContainer.initService(deployState.getDeployLogger());
                     clusterControllerContainer.setProp("clustertype", "admin");
@@ -511,11 +432,7 @@ public class ContentCluster extends AbstractConfigProducer implements
 
     public PersistenceEngine.PersistenceFactory getPersistence() { return persistenceFactory; }
 
-    /**
-     * The list of documentdefinitions declared at the cluster level.
-     *
-     * @return the set of documenttype names
-     */
+    /** Returns a list of th document types declared at the cluster level. */
     public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; }
 
     public boolean isGloballyDistributed(NewDocumentType docType) {
@@ -753,6 +670,7 @@ public class ContentCluster extends AbstractConfigProducer implements
      * a previous generation of it only by restarting the consuming processes.
      */
     public void setDeferChangesUntilRestart(boolean deferChangesUntilRestart) {
+        // TODO
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index c6ddc6b0c36..86668fe3098 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -264,8 +264,7 @@ public class ModelProvisioningTest {
                     "   </content>" +
                     "</services>";
             VespaModelTester tester = new VespaModelTester();
-            tester.dedicatedClusterControllerCluster(false);
-            tester.addHosts(2);
+            tester.addHosts(5);
             VespaModel model = tester.createModel(xmlWithNodes, true);
             assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
             assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size());
@@ -330,8 +329,7 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(2);
+        tester.addHosts(5);
         VespaModel model = tester.createModel(xmlWithNodes, true);
 
         assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
@@ -583,9 +581,8 @@ public class ModelProvisioningTest {
                         "   </content>" +
                         "</services>";
 
-        int numberOfHosts = 64;
+        int numberOfHosts = 67;
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, true);
         assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
@@ -604,23 +601,17 @@ public class ModelProvisioningTest {
         assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts));
         assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
         assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
-        assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers());
+        assertEquals(3, admin.getClusterControllers().getContainers().size());
 
         // Check content clusters
         ContentCluster cluster = model.getContentClusters().get("bar");
-        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
-        assertEquals(3, clusterControllers.getContainers().size());
-        assertEquals("bar-controllers", clusterControllers.getName());
-        assertEquals("node-1-3-10-54", clusterControllers.getContainers().get(0).getHostName());
-        assertEquals("node-1-3-10-51", clusterControllers.getContainers().get(1).getHostName());
-        assertEquals("node-1-3-10-48", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(9, cluster.getRootGroup().getSubgroups().size());
         assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex());
         assertEquals(3, cluster.getRootGroup().getSubgroups().get(0).getNodes().size());
         assertEquals(0, cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0"));
-        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-57", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1"));
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2));
@@ -629,13 +620,13 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3"));
-        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4"));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5));
         assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5"));
         // ...
-        assertEquals("node-1-3-10-48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
+        assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
         // ...
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8"));
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3));
@@ -647,12 +638,6 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26"));
 
         cluster = model.getContentClusters().get("baz");
-        clusterControllers = cluster.getClusterControllers();
-        assertEquals(3, clusterControllers.getContainers().size());
-        assertEquals("baz-controllers", clusterControllers.getName());
-        assertEquals("node-1-3-10-27", clusterControllers.getContainers().get(0).getHostName());
-        assertEquals("node-1-3-10-26", clusterControllers.getContainers().get(1).getHostName());
-        assertEquals("node-1-3-10-25", clusterControllers.getContainers().get(2).getHostName());
         assertEquals(0, cluster.getRootGroup().getNodes().size());
         assertEquals(27, cluster.getRootGroup().getSubgroups().size());
         assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
@@ -732,41 +717,6 @@ public class ModelProvisioningTest {
     }
 
     @Test
-    public void testClusterControllersWithGroupSize2() {
-        String services =
-                "<?xml version='1.0' encoding='utf-8' ?>\n" +
-                "<services>" +
-                "  <admin version='4.0'/>" +
-                "  <container version='1.0' id='foo'>" +
-                "     <nodes count='10'/>" +
-                "  </container>" +
-                "  <content version='1.0' id='bar'>" +
-                "     <redundancy>2</redundancy>" +
-                "     <documents>" +
-                "       <document type='type1' mode='index'/>" +
-                "     </documents>" +
-                "     <nodes count='8' groups='4'/>" +
-                "  </content>" +
-                "</services>";
-
-        int numberOfHosts = 18;
-        VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(numberOfHosts);
-        VespaModel model = tester.createModel(services, true);
-        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
-
-        // Check content clusters
-        ContentCluster cluster = model.getContentClusters().get("bar");
-        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
-        assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size());
-        assertEquals("bar-controllers", clusterControllers.getName());
-        assertEquals("node-1-3-10-08", clusterControllers.getContainers().get(0).getHostName());
-        assertEquals("node-1-3-10-06", clusterControllers.getContainers().get(1).getHostName());
-        assertEquals("node-1-3-10-04", clusterControllers.getContainers().get(2).getHostName());
-    }
-
-    @Test
     public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -850,32 +800,6 @@ public class ModelProvisioningTest {
     }
 
     @Test
-    public void test2ContentNodesProduces1ClusterController() {
-        String services =
-                "<?xml version='1.0' encoding='utf-8' ?>\n" +
-                        "<services>" +
-                        "  <content version='1.0' id='bar'>" +
-                        "     <redundancy>2</redundancy>" +
-                        "     <documents>" +
-                        "       <document type='type1' mode='index'/>" +
-                        "     </documents>" +
-                        "     <nodes count='2'/>" +
-                        "  </content>" +
-                        "</services>";
-
-        int numberOfHosts = 2;
-        VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(numberOfHosts);
-        VespaModel model = tester.createModel(services, true);
-        assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
-
-        ContentCluster cluster = model.getContentClusters().get("bar");
-        ContainerCluster clusterControllers = cluster.getClusterControllers();
-        assertEquals(1, clusterControllers.getContainers().size());
-    }
-
-    @Test
     public void testDedicatedClusterControllers() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -1131,16 +1055,11 @@ public class ModelProvisioningTest {
 
         int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(numberOfHosts);
         VespaModel model = tester.createModel(services, false);
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts));
 
         ContentCluster cluster = model.getContentClusters().get("bar");
-        ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers();
-        assertEquals(1, clusterControllers.getContainers().size());
-        assertEquals("bar-controllers", clusterControllers.getName());
-        assertEquals("node-1-3-10-01", clusterControllers.getContainers().get(0).getHostName());
         assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3
         assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4
         assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3
@@ -1295,14 +1214,14 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 18;
+        int totalHosts = 21;
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
         tester.addHosts(new NodeResources(0.1, 0.2, 300, 0.3, NodeResources.DiskSpeed.slow), 1);// Logserver
         tester.addHosts(new NodeResources(0.1, 0.3, 1, 0.5), 2); // Slobrok
         tester.addHosts(new NodeResources(12, 10, 30, 0.3), 4); // Container
         tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo
         tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar
+        tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller
         VespaModel model = tester.createModel(services, true, 0);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1329,11 +1248,11 @@ public class ModelProvisioningTest {
 
         int totalHosts = 10;
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6);
-        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+        tester.addHosts(new NodeResources(11.5,  10,           30, 0.3), 6);
+        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
+        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(totalHosts + 3, model.getRoot().hostSystem().getHosts().size());
     }
 
     @Test
@@ -1356,11 +1275,11 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
 
-        int totalHosts = 26;
+        int totalHosts = 29;
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6);
-        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+        tester.addHosts(new NodeResources(13.5, 100,         1000, 0.3), 6);
+        tester.addHosts(new NodeResources(85,   200, 1000_000_000, 0.3), 20);
+        tester.addHosts(new NodeResources( 0.5,   2,           10, 0.3), 3);
         VespaModel model = tester.createModel(services, true, true);
         assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
     }
@@ -1581,10 +1500,9 @@ public class ModelProvisioningTest {
                 "  </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(3);
+        tester.addHosts(6);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(3, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(6, model.getRoot().hostSystem().getHosts().size());
         assertEquals(2, model.getAdmin().getSlobroks().size());
         assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
         assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes());
@@ -1699,7 +1617,7 @@ public class ModelProvisioningTest {
         assertEquals(1, model.getRoot().hostSystem().getHosts().size());
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster controller = content.getClusterControllers();
+        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1745,7 +1663,7 @@ public class ModelProvisioningTest {
         assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(1, content.getRootGroup().getNodes().size());
-        ContainerCluster controller = content.getClusterControllers();
+        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1854,10 +1772,10 @@ public class ModelProvisioningTest {
                 " </services>";
 
         VespaModel model = createNonProvisionedMultitenantModel(services);
-        assertThat(model.getRoot().hostSystem().getHosts().size(), is(1));
+        assertEquals(1, model.getRoot().hostSystem().getHosts().size());
         ContentCluster content = model.getContentClusters().get("storage");
         assertEquals(2, content.getRootGroup().getNodes().size());
-        ContainerCluster controller = content.getClusterControllers();
+        ContainerCluster<?> controller = model.getAdmin().getClusterControllers();
         assertEquals(1, controller.getContainers().size());
     }
 
@@ -1882,8 +1800,7 @@ public class ModelProvisioningTest {
                 "   </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.dedicatedClusterControllerCluster(false);
-        tester.addHosts(6);
+        tester.addHosts(9);
         VespaModel model = tester.createModel(servicesXml, true);
 
         Map<String, Boolean> tests = Map.of("qrs", false,
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index a11d23dbfbb..3f211a595b9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -144,7 +144,7 @@ public class MetricsProxyContainerTest {
     @Test
     public void vespa_services_config_has_all_services() {
         VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent());
-        assertEquals(6, vespaServicesConfig.service().size());
+        assertEquals(7, vespaServicesConfig.service().size());
 
         for (var service : vespaServicesConfig.service()) {
             if (service.configId().equals("admin/cluster-controllers/0")) {
@@ -185,7 +185,7 @@ public class MetricsProxyContainerTest {
     private static String servicesWithContent() {
         return String.join("\n",
                            "<services>",
-                           "    <admin version='4.0'>",
+                           "    <admin version='2.0'>",
                            "        <adminserver hostalias='node1'/>",
                            "    </admin>",
                            "    <content version='1.0' id='my-content'>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
index 6ad74231cae..d92ace2939a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
@@ -25,13 +25,13 @@ public class QuotaValidatorTest {
 
     @Test
     public void test_deploy_under_quota() {
-        var tester = new ValidationTester(5, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         tester.deploy(null, getServices("testCluster", 5), Environment.prod, null);
     }
 
     @Test
     public void test_deploy_above_quota_clustersize() {
-        var tester = new ValidationTester(11, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(14, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 11), Environment.prod, null);
             fail();
@@ -42,32 +42,32 @@ public class QuotaValidatorTest {
 
     @Test
     public void test_deploy_above_quota_budget() {
-        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
         } catch (RuntimeException e) {
             assertEquals("Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!",
-                    ValidationTester.censorNumbers(e.getMessage()));
+                         ValidationTester.censorNumbers(e.getMessage()));
         }
     }
 
     @Test
     public void test_deploy_above_quota_budget_in_publiccd() {
-        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone));
+        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
         } catch (RuntimeException e) {
             assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!",
-                    ValidationTester.censorNumbers(e.getMessage()));
+                         ValidationTester.censorNumbers(e.getMessage()));
         }
     }
 
     @Test
     public void test_deploy_with_negative_budget() {
         var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1));
-        var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
+        var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone));
         try {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
index b7fa72d8a64..02f1195d50e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
@@ -34,9 +34,9 @@ public class ValidationTester {
     private final TestProperties properties;
     private final InMemoryProvisioner hostProvisioner;
 
-    /** Creates a validation tester with 1 node available */
+    /** Creates a validation tester with 1 node available (in addition to cluster controllers) */
     public ValidationTester() {
-        this(1);
+        this(4);
     }
 
     /** Creates a validation tester with number of nodes available and the given test properties */
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
index bade5a746f7..f90762e8fc0 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
@@ -19,7 +19,7 @@ public class ClusterSizeReductionValidatorTest {
 
     @Test
     public void testSizeReductionValidation() {
-        ValidationTester tester = new ValidationTester(30);
+        ValidationTester tester = new ValidationTester(33);
 
         VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst();
         try {
@@ -45,7 +45,7 @@ public class ClusterSizeReductionValidatorTest {
 
     @Test
     public void testOverridingSizereductionValidation() {
-        ValidationTester tester = new ValidationTester(30);
+        ValidationTester tester = new ValidationTester(33);
 
         VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst();
         tester.deploy(previous, getServices(14), Environment.prod, sizeReductionOverride); // Allowed due to override
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
index 392c37f50f3..30dab17635a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.fail;
  */
 public class ContentClusterRemovalValidatorTest {
 
-    private final ValidationTester tester = new ValidationTester(2);
+    private final ValidationTester tester = new ValidationTester(5);
 
     @Test
     public void testContentRemovalValidation() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 219afb14ea5..4aadc0e3f05 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1021,26 +1021,17 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5);
     }
 
-    void assertZookeeperServerImplementation(String expectedClassName) {
-        VespaModel model = createEnd2EndOneNode(new TestProperties().setMultitenant(true));
-
-        ContentCluster cc = model.getContentClusters().get("storage");
-        for (ClusterControllerContainer c : cc.getClusterControllers().getContainers()) {
+    void assertZookeeperServerImplementation(String expectedClassName,
+                                             ClusterControllerContainerCluster clusterControllerCluster) {
+        for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) {
             var builder = new ComponentsConfig.Builder();
             c.getConfig(builder);
             assertEquals(1, new ComponentsConfig(builder).components().stream()
-                    .filter(component -> component.classId().equals(expectedClassName))
-                    .count());
+                                                         .filter(component -> component.classId().equals(expectedClassName))
+                                                         .count());
         }
     }
 
-    @Test
-    public void reconfigurableZookeeperServerComponentsForClusterController() {
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer");
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer");
-        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl");
-    }
-
     private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) {
         VespaModel model = createEnd2EndOneNode(props);
 
@@ -1137,17 +1128,7 @@ public class ContentClusterTest extends ContentBaseTest {
         assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers());
         assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers());
 
-        Map<String, ContentCluster> clustersWithOwnCCC = createEnd2EndOneNode(new TestProperties().setMultitenant(true), twoContentServices).getContentClusters();
         ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers();
-        assertEquals("Union of components in own clusters is equal to those in shared cluster",
-                     clusterControllers.getAllComponents().stream()
-                                    .map(Component::getComponentId)
-                                    .collect(toList()),
-                     clustersWithOwnCCC.values().stream()
-                                       .flatMap(cluster -> Optional.ofNullable(cluster.getClusterControllers()).stream()
-                                                                   .flatMap(c -> c.getAllComponents().stream()))
-                                       .map(Component::getComponentId)
-                                       .collect(toList()));
 
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("storage").size());
         assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("dev-null").size());
@@ -1157,6 +1138,13 @@ public class ContentClusterTest extends ContentBaseTest {
         twoContentModel.getConfig(devNullBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-dev-null-configurer");
         assertEquals(0.618, storageBuilder.build().min_distributor_up_ratio(), 1e-9);
         assertEquals(0.418, devNullBuilder.build().min_distributor_up_ratio(), 1e-9);
+
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer",
+                                            clusterControllers);
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer",
+                                            clusterControllers);
+        assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl",
+                                            clusterControllers);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index eafbca09009..b72ae088484 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -52,7 +52,6 @@ public class VespaModelTester {
     private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>();
     private ApplicationId applicationId = ApplicationId.defaultId();
     private boolean useDedicatedNodeForLogserver = false;
-    private boolean dedicatedClusterControllerCluster = true;
 
     public VespaModelTester() {
         this(new NullConfigModelRegistry());
@@ -102,10 +101,6 @@ public class VespaModelTester {
         this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver;
     }
 
-    public void dedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) {
-        this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster;
-    }
-
     /** Creates a model which uses 0 as start index and fails on out of capacity */
     public VespaModel createModel(String services, String ... retiredHostNames) {
         return createModel(Zone.defaultZone(), services, true, retiredHostNames);
@@ -175,8 +170,7 @@ public class VespaModelTester {
                 .setMultitenant(true)
                 .setHostedVespa(hosted)
                 .setApplicationId(applicationId)
-                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver)
-                .setDedicatedClusterControllerCluster(dedicatedClusterControllerCluster);
+                .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver);
 
         DeployState.Builder deployState = deployStatebuilder
                 .applicationPackage(appPkg)