Use configured config server cluster size in orchestrator

11 files changed, 204 insertions, 113 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index bd8e31cd283..0cf654b23b5 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.google.inject.Inject;
+import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
@@ -17,7 +18,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
-import com.yahoo.vespa.orchestrator.model.ApplicationApiImpl;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.model.VespaModelUtil;
 import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
@@ -56,19 +57,22 @@ public class OrchestratorImpl implements Orchestrator {
     private final int serviceMonitorConvergenceLatencySeconds;
     private final ClusterControllerClientFactory clusterControllerClientFactory;
     private final Clock clock;
+    private final ApplicationApiFactory applicationApiFactory;
 
     @Inject
     public OrchestratorImpl(ClusterControllerClientFactory clusterControllerClientFactory,
                             StatusService statusService,
                             OrchestratorConfig orchestratorConfig,
-                            InstanceLookupService instanceLookupService)
+                            InstanceLookupService instanceLookupService,
+                            ConfigserverConfig configServerConfig)
     {
-        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory),
-                clusterControllerClientFactory,
-                statusService,
-                instanceLookupService,
-                orchestratorConfig.serviceMonitorConvergenceLatencySeconds(),
-                Clock.systemUTC());
+        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, new ApplicationApiFactory(configServerConfig.zookeeperserver().size())),
+             clusterControllerClientFactory,
+             statusService,
+             instanceLookupService,
+             orchestratorConfig.serviceMonitorConvergenceLatencySeconds(),
+             Clock.systemUTC(),
+             new ApplicationApiFactory(configServerConfig.zookeeperserver().size()));
     }
 
     public OrchestratorImpl(Policy policy,
@@ -76,7 +80,8 @@ public class OrchestratorImpl implements Orchestrator {
                             StatusService statusService,
                             InstanceLookupService instanceLookupService,
                             int serviceMonitorConvergenceLatencySeconds,
-                            Clock clock)
+                            Clock clock,
+                            ApplicationApiFactory applicationApiFactory)
     {
         this.policy = policy;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
@@ -84,6 +89,7 @@ public class OrchestratorImpl implements Orchestrator {
         this.serviceMonitorConvergenceLatencySeconds = serviceMonitorConvergenceLatencySeconds;
         this.instanceLookupService = instanceLookupService;
         this.clock = clock;
+        this.applicationApiFactory = applicationApiFactory;
     }
 
     @Override
@@ -171,10 +177,8 @@ public class OrchestratorImpl implements Orchestrator {
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
         try (MutableStatusRegistry statusRegistry = statusService
                 .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) {
-            ApplicationApi applicationApi = new ApplicationApiImpl(
-                    nodeGroup,
-                    statusRegistry,
-                    clusterControllerClientFactory);
+            ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, statusRegistry,
+                                                                         clusterControllerClientFactory);
 
             policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi);
         }
@@ -196,9 +200,8 @@ public class OrchestratorImpl implements Orchestrator {
                 return;
             }
 
-            ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup,
-                                                                   hostStatusRegistry,
-                                                                   clusterControllerClientFactory);
+            ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusRegistry,
+                                                                         clusterControllerClientFactory);
             policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi);
         }
     }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
new file mode 100644
index 00000000000..5e6ec59c28d
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
@@ -0,0 +1,24 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.model;
+
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+
+/**
+ * @author mpolden
+ */
+public class ApplicationApiFactory {
+
+    private final int numberOfConfigServers;
+
+    public ApplicationApiFactory(int numberOfConfigServers) {
+        this.numberOfConfigServers = numberOfConfigServers;
+    }
+
+    public ApplicationApi create(NodeGroup nodeGroup,
+                                 MutableStatusRegistry hostStatusService,
+                                 ClusterControllerClientFactory clusterControllerClientFactory) {
+        return new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory, numberOfConfigServers);
+    }
+
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
index db2fb9b4a18..eb5dcf790ba 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
-import com.yahoo.vespa.orchestrator.status.StatusService;
 
 import java.util.Collection;
 import java.util.Comparator;
@@ -26,6 +25,9 @@ import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
 
+/**
+ * @author hakonhall
+ */
 public class ApplicationApiImpl implements ApplicationApi {
 
     private final ApplicationInstance applicationInstance;
@@ -36,7 +38,8 @@ public class ApplicationApiImpl implements ApplicationApi {
 
     public ApplicationApiImpl(NodeGroup nodeGroup,
                               MutableStatusRegistry hostStatusService,
-                              ClusterControllerClientFactory clusterControllerClientFactory) {
+                              ClusterControllerClientFactory clusterControllerClientFactory,
+                              int numberOfConfigServers) {
         this.applicationInstance = nodeGroup.getApplication();
         this.nodeGroup = nodeGroup;
         this.hostStatusService = hostStatusService;
@@ -44,7 +47,7 @@ public class ApplicationApiImpl implements ApplicationApi {
         this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(),
                                                                      hostName -> hostStatusService.getSuspendedHosts().contains(hostName)
                                                                                  ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS));
-        this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory);
+        this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory, numberOfConfigServers);
     }
 
     @Override
@@ -109,10 +112,9 @@ public class ApplicationApiImpl implements ApplicationApi {
                 .collect(Collectors.toList());
     }
 
-    private List<ClusterApi> makeClustersInOrder
-            (NodeGroup nodeGroup,
-             Map<HostName, HostStatus> hostStatusMap,
-             ClusterControllerClientFactory clusterControllerClientFactory) {
+    private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, Map<HostName, HostStatus> hostStatusMap,
+                                                 ClusterControllerClientFactory clusterControllerClientFactory,
+                                                 int numberOfConfigServers) {
         Set<ServiceCluster> clustersInGroup = getServiceClustersInGroup(nodeGroup);
         return clustersInGroup.stream()
                 .map(serviceCluster -> new ClusterApiImpl(
@@ -120,7 +122,8 @@ public class ApplicationApiImpl implements ApplicationApi {
                         serviceCluster,
                         nodeGroup,
                         hostStatusMap,
-                        clusterControllerClientFactory))
+                        clusterControllerClientFactory,
+                        numberOfConfigServers))
                 .sorted(ApplicationApiImpl::compareClusters)
                 .collect(Collectors.toList());
     }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index 7b72b4e970d..2aaebb18aa6 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -19,6 +19,9 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+/**
+ * @author hakonhall
+ */
 class ClusterApiImpl implements ClusterApi {
 
     private final ApplicationApi applicationApi;
@@ -31,10 +34,14 @@ class ClusterApiImpl implements ClusterApi {
     private final Set<ServiceInstance> servicesNotInGroup;
     private final Set<ServiceInstance> servicesDownAndNotInGroup;
 
-    /**
-     * There are supposed to be (at least) 3 config servers in a production-like environment.
-     * However the number of config servers in the zone-config-servers application/cluster may only be 2,
-     * if only 2 have been provisioned so far, or 1 is being reprovisioned. In these cases it is
+    /*
+     * There are two sources for the number of config servers in a cluster. The config server config and the node
+     * repository.
+     *
+     * The actual number of config servers in the zone-config-servers application/cluster may be less than
+     * the configured number.
+     *
+     * For example: If only 2/3 have been provisioned so far, or 1 is being reprovisioned. In these cases it is
      * important for the Orchestrator to count that third config server as down.
      */
     private final int missingServices;
@@ -44,7 +51,8 @@ class ClusterApiImpl implements ClusterApi {
                           ServiceCluster serviceCluster,
                           NodeGroup nodeGroup,
                           Map<HostName, HostStatus> hostStatusMap,
-                          ClusterControllerClientFactory clusterControllerClientFactory) {
+                          ClusterControllerClientFactory clusterControllerClientFactory,
+                          int numberOfConfigServers) {
         this.applicationApi = applicationApi;
         this.serviceCluster = serviceCluster;
         this.nodeGroup = nodeGroup;
@@ -64,8 +72,8 @@ class ClusterApiImpl implements ClusterApi {
         servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet());
 
         int serviceInstances = serviceCluster.serviceInstances().size();
-        if (serviceCluster.isConfigServerCluster() && serviceInstances < 3) {
-            missingServices = 3 - serviceInstances;
+        if (serviceCluster.isConfigServerCluster() && serviceInstances < numberOfConfigServers) {
+            missingServices = numberOfConfigServers - serviceInstances;
             descriptionOfMissingServices = missingServices + " missing config server" + (missingServices > 1 ? "s" : "");
         } else {
             missingServices = 0;
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
index 96930c1cda2..a9b9736ebfb 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
@@ -7,16 +7,13 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
-import com.yahoo.vespa.orchestrator.model.ApplicationApiImpl;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
 import com.yahoo.vespa.orchestrator.model.ClusterApi;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.model.StorageNode;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
-import com.yahoo.vespa.orchestrator.status.StatusService;
-
-import java.util.logging.Logger;
 
 /**
  * @author oyving
@@ -31,10 +28,12 @@ public class HostedVespaPolicy implements Policy {
 
     private final HostedVespaClusterPolicy clusterPolicy;
     private final ClusterControllerClientFactory clusterControllerClientFactory;
+    private final ApplicationApiFactory applicationApiFactory;
 
-    public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory) {
+    public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory, ApplicationApiFactory applicationApiFactory) {
         this.clusterPolicy = clusterPolicy;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
+        this.applicationApiFactory = applicationApiFactory;
     }
 
     @Override
@@ -107,7 +106,7 @@ public class HostedVespaPolicy implements Policy {
             HostName hostName,
             MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostName);
-        ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory);
+        ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusService, clusterControllerClientFactory);
         releaseSuspensionGrant(context, applicationApi);
     }
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 4d7a0f18918..45d4c531898 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.orchestrator;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
@@ -16,17 +17,18 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.StatusService;
 import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
 import com.yahoo.vespa.service.monitor.ServiceModel;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -61,6 +63,8 @@ import static org.mockito.Mockito.spy;
  */
 public class OrchestratorImplTest {
 
+    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3);
+
     private ApplicationId app1;
     private ApplicationId app2;
     private HostName app1_host1;
@@ -69,7 +73,7 @@ public class OrchestratorImplTest {
     private ClusterControllerClientFactoryMock clustercontroller;
 
     @Before
-    public void setUp() throws Exception {
+    public void setUp() {
         // Extract applications and hosts from dummy instance lookup service
         Iterator<ApplicationInstance> iterator = DummyInstanceLookupService.getApplications().iterator();
         ApplicationInstanceReference app1_ref = iterator.next().reference();
@@ -78,21 +82,17 @@ public class OrchestratorImplTest {
         app2 = OrchestratorUtil.toApplicationId(iterator.next().reference());
 
         clustercontroller = new ClusterControllerClientFactoryMock();
-        orchestrator = new OrchestratorImpl(
-                clustercontroller,
-                new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()),
-                new OrchestratorConfig(new OrchestratorConfig.Builder()),
-                new DummyInstanceLookupService());
+        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clustercontroller, applicationApiFactory),
+                                            clustercontroller,
+                                            new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()),
+                                            new DummyInstanceLookupService(),
+                                            0,
+                                            new ManualClock(),
+                                            applicationApiFactory);
 
         clustercontroller.setAllDummyNodesAsUp();
     }
 
-    @After
-    public void tearDown() throws Exception {
-        orchestrator = null;
-        clustercontroller = null;
-    }
-
     @Test
     public void application_has_initially_no_remarks() throws Exception {
         assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS));
@@ -112,7 +112,7 @@ public class OrchestratorImplTest {
     }
 
     @Test
-    public void appliations_list_returns_empty_initially() throws Exception {
+    public void appliations_list_returns_empty_initially() {
         assertThat(orchestrator.getAllSuspendedApplications(), is(empty()));
     }
 
@@ -343,11 +343,13 @@ public class OrchestratorImplTest {
         InstanceLookupService lookupService = new ServiceMonitorInstanceLookupService(
                 () -> new ServiceModel(Map.of(reference, applicationInstance)));
 
-        orchestrator = new OrchestratorImpl(
-                clusterControllerClientFactory,
-                statusService,
-                new OrchestratorConfig(new OrchestratorConfig.Builder()),
-                lookupService);
+        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory),
+                                            clusterControllerClientFactory,
+                                            statusService,
+                                            lookupService,
+                                            0,
+                                            new ManualClock(),
+                                            applicationApiFactory);
 
         orchestrator.setNodeStatus(hostName, HostStatus.ALLOWED_TO_BE_DOWN);
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
index 5a11d3aa640..d5734a73de0 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
@@ -20,7 +20,7 @@ public class ApplicationApiImplTest {
 
     @Test
     public void testApplicationId() {
-        ApplicationApiImpl applicationApi =
+        ApplicationApi applicationApi =
                 modelUtils.createApplicationApiImpl(modelUtils.createApplicationInstance(new ArrayList<>()));
         assertEquals("tenant:application-name:default", applicationApi.applicationId().serializedForm());
     }
@@ -179,7 +179,7 @@ public class ApplicationApiImplTest {
 
         modelUtils.createNode("host1", hostStatus);
 
-        ApplicationApiImpl applicationApi = modelUtils.createApplicationApiImpl(applicationInstance, hostName1);
+        ApplicationApi applicationApi = modelUtils.createApplicationApiImpl(applicationInstance, hostName1);
         List<HostName> upStorageNodes = expectUp ? Arrays.asList(hostName1) : new ArrayList<>();
 
         List<HostName> actualStorageNodes = applicationApi.getUpStorageNodesInGroupInClusterOrder().stream()
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index c17e885df02..8e42ccdc79d 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
@@ -18,10 +19,13 @@ import org.junit.Test;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.hamcrest.core.StringContains.containsString;
 import static org.junit.Assert.assertEquals;
@@ -32,9 +36,13 @@ import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+/**
+ * @author hakonhall
+ */
 public class ClusterApiImplTest {
-    final ApplicationApi applicationApi = mock(ApplicationApi.class);
-    final ModelTestUtils modelUtils = new ModelTestUtils();
+
+    private final ApplicationApi applicationApi = mock(ApplicationApi.class);
+    private final ModelTestUtils modelUtils = new ModelTestUtils();
 
     @Test
     public void testServicesDownAndNotInGroup() {
@@ -68,7 +76,7 @@ public class ClusterApiImplTest {
                 serviceCluster,
                 new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5),
                 modelUtils.getHostStatusMap(),
-                modelUtils.getClusterControllerClientFactory());
+                modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo());
         assertFalse(clusterApi.isStorageCluster());
@@ -87,39 +95,7 @@ public class ClusterApiImplTest {
 
     /** Make a ClusterApiImpl for the cfg1 config server, with cfg3 missing from the cluster (not provisioned). */
     private ClusterApiImpl makeCfg1ClusterApi(ServiceStatus cfg1ServiceStatus, ServiceStatus cfg2ServiceStatus) {
-        HostName cfg1Hostname = new HostName("cfg1");
-        HostName cfg2Hostname = new HostName("cfg2");
-
-        ServiceCluster serviceCluster = modelUtils.createServiceCluster(
-                ClusterId.CONFIG_SERVER.s(),
-                ServiceType.CONFIG_SERVER,
-                Arrays.asList(
-                        modelUtils.createServiceInstance("cs1", cfg1Hostname, cfg1ServiceStatus),
-                        modelUtils.createServiceInstance("cs2", cfg2Hostname, cfg2ServiceStatus))
-        );
-
-        Set<ServiceCluster> serviceClusterSet = new HashSet<>(Set.of(serviceCluster));
-
-        ApplicationInstance application = new ApplicationInstance(
-                TenantId.HOSTED_VESPA,
-                ApplicationInstanceId.CONFIG_SERVER,
-                serviceClusterSet);
-
-        serviceCluster.setApplicationInstance(application);
-
-        when(applicationApi.applicationId()).thenReturn(OrchestratorUtil.toApplicationId(application.reference()));
-
-        ClusterApiImpl clusterApi = new ClusterApiImpl(
-                applicationApi,
-                serviceCluster,
-                new NodeGroup(application, cfg1Hostname),
-                modelUtils.getHostStatusMap(),
-                modelUtils.getClusterControllerClientFactory());
-
-        assertEquals(1, clusterApi.missingServices());
-        assertFalse(clusterApi.noServicesOutsideGroupIsDown());
-
-        return clusterApi;
+        return makeConfigClusterApi(ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, cfg1ServiceStatus, cfg2ServiceStatus);
     }
 
     @Test
@@ -149,6 +125,19 @@ public class ClusterApiImplTest {
     }
 
     @Test
+    public void testSingleConfigServerCanSuspend() {
+        for (var status : EnumSet.of(ServiceStatus.UP, ServiceStatus.DOWN)) {
+            var clusterApi = makeConfigClusterApi(1, status);
+            var policy = new HostedVespaClusterPolicy();
+            try {
+                policy.verifyGroupGoingDownIsFine(clusterApi);
+            } catch (HostStateChangeDeniedException e) {
+                fail("Expected suspension to succeed");
+            }
+        }
+    }
+
+    @Test
     public void testNoServices() {
         HostName hostName1 = new HostName("host1");
         HostName hostName2 = new HostName("host2");
@@ -196,7 +185,7 @@ public class ClusterApiImplTest {
                 serviceCluster,
                 new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes),
                 modelUtils.getHostStatusMap(),
-                modelUtils.getClusterControllerClientFactory());
+                modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp());
         assertEquals(expectedNoServicesOutsideGroupIsDown, clusterApi.noServicesOutsideGroupIsDown());
@@ -226,10 +215,52 @@ public class ClusterApiImplTest {
                 serviceCluster,
                 new NodeGroup(applicationInstance, hostName1, hostName3),
                 new HashMap<>(),
-                modelUtils.getClusterControllerClientFactory());
+                modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertTrue(clusterApi.isStorageCluster());
         assertEquals(Optional.of(hostName1), clusterApi.storageNodeInGroup().map(storageNode -> storageNode.hostName()));
         assertEquals(Optional.of(hostName1), clusterApi.upStorageNodeInGroup().map(storageNode -> storageNode.hostName()));
     }
+
+    private ClusterApiImpl makeConfigClusterApi(int clusterSize, ServiceStatus first, ServiceStatus... rest) {
+        var serviceStatusList = new ArrayList<ServiceStatus>();
+        serviceStatusList.add(first);
+        serviceStatusList.addAll(List.of(rest));
+        var hostnames = IntStream.rangeClosed(1, serviceStatusList.size())
+                                 .mapToObj(i -> new HostName("cfg" + i))
+                                 .collect(Collectors.toList());
+        var instances = new ArrayList<ServiceInstance>();
+        for (int i = 0; i < hostnames.size(); i++) {
+            instances.add(modelUtils.createServiceInstance("cs" + i + 1, hostnames.get(i), serviceStatusList.get(i)));
+        }
+        ServiceCluster serviceCluster = modelUtils.createServiceCluster(
+                ClusterId.CONFIG_SERVER.s(),
+                ServiceType.CONFIG_SERVER,
+                instances
+        );
+
+        Set<ServiceCluster> serviceClusterSet = Set.of(serviceCluster);
+
+        ApplicationInstance application = new ApplicationInstance(
+                TenantId.HOSTED_VESPA,
+                ApplicationInstanceId.CONFIG_SERVER,
+                serviceClusterSet);
+
+        serviceCluster.setApplicationInstance(application);
+
+        when(applicationApi.applicationId()).thenReturn(OrchestratorUtil.toApplicationId(application.reference()));
+
+        ClusterApiImpl clusterApi = new ClusterApiImpl(
+                applicationApi,
+                serviceCluster,
+                new NodeGroup(application, hostnames.get(0)),
+                modelUtils.getHostStatusMap(),
+                modelUtils.getClusterControllerClientFactory(), clusterSize);
+
+        assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices());
+        assertEquals(clusterSize == serviceStatusList.size(), clusterApi.noServicesOutsideGroupIsDown());
+
+        return clusterApi;
+    }
+
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 544ac27c92f..729b3ae79ff 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.model;
 
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
@@ -20,10 +21,10 @@ import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
 import com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService;
-import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
-import com.yahoo.vespa.orchestrator.config.OrchestratorConfig.Builder;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
@@ -40,16 +41,28 @@ import java.util.Set;
 
 import static org.mockito.Mockito.mock;
 
+/**
+ * @author hakonhall
+ */
 class ModelTestUtils {
 
+    public static final int NUMBER_OF_CONFIG_SERVERS = 3;
+
     private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>();
     private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
     private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>();
     private final StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer());
-    private final Orchestrator orchestrator = new OrchestratorImpl(clusterControllerClientFactory,
+    private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory()),
+                                                                   clusterControllerClientFactory,
                                                                    statusService,
-                                                                   new OrchestratorConfig(new Builder()),
-                                                                   new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)));
+                                                                   new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)),
+                                                                   0,
+                                                                   new ManualClock(),
+                                                                   applicationApiFactory());
+
+    ApplicationApiFactory applicationApiFactory() {
+        return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS);
+    }
 
     Map<HostName, HostStatus> getHostStatusMap() {
         return hostStatusMap;
@@ -71,14 +84,14 @@ class ModelTestUtils {
         return hostName;
     }
 
-    ApplicationApiImpl createApplicationApiImpl(
+    ApplicationApi createApplicationApiImpl(
             ApplicationInstance applicationInstance,
             HostName... hostnames) {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostnames);
         MutableStatusRegistry registry = statusService.lockApplicationInstance_forCurrentThreadOnly(
                 OrchestratorContext.createContextForSingleAppOp(Clock.systemUTC()),
                 applicationInstance.reference());
-        return new ApplicationApiImpl(nodeGroup, registry, clusterControllerClientFactory);
+        return applicationApiFactory().create(nodeGroup, registry, clusterControllerClientFactory);
     }
 
     ApplicationInstance createApplicationInstance(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
index ed36cd7e317..b27e37ac034 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
 import com.yahoo.vespa.orchestrator.model.ClusterApi;
 import com.yahoo.vespa.orchestrator.model.StorageNode;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
@@ -33,6 +34,7 @@ public class HostedVespaPolicyTest {
 
     private final ClusterControllerClientFactory clientFactory = mock(ClusterControllerClientFactory.class);
     private final ClusterControllerClient client = mock(ClusterControllerClient.class);
+    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3);
 
     @Before
     public void setUp() {
@@ -42,7 +44,7 @@ public class HostedVespaPolicyTest {
     @Test
     public void testGrantSuspension() throws HostStateChangeDeniedException {
         final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class);
-        final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory);
+        final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory);
         final ApplicationApi applicationApi = mock(ApplicationApi.class);
         when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default"));
 
@@ -94,7 +96,7 @@ public class HostedVespaPolicyTest {
     @Test
     public void testAcquirePermissionToRemove() throws OrchestrationException {
         final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class);
-        final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory);
+        final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory);
         final ApplicationApi applicationApi = mock(ApplicationApi.class);
         when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default"));
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 32758970d75..fec1554396d 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
 import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.Policy;
@@ -52,6 +53,7 @@ import java.time.Clock;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
@@ -76,6 +78,7 @@ public class HostResourceTest {
     private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId");
     private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZookeeperStatusService(
             new MockCurator(), mock(Metric.class), new TestTimer());
+    private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3);
 
     private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class);
     static {
@@ -131,7 +134,8 @@ public class HostResourceTest {
             new ClusterControllerClientFactoryMock(),
             EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService,
             SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
-            clock
+            clock,
+            applicationApiFactory
     );
 
     private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl(
@@ -139,7 +143,8 @@ public class HostResourceTest {
             new ClusterControllerClientFactoryMock(),
             EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService,
             SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
-            clock
+            clock,
+            applicationApiFactory
     );
 
     private final UriInfo uriInfo = mock(UriInfo.class);
@@ -172,8 +177,7 @@ public class HostResourceTest {
     @Test
     public void returns_200_empty_batch() {
         HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
-        BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname",
-                                                                          Collections.emptyList());;
+        BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of());
         assertThat(response.success());
     }
 
@@ -242,7 +246,8 @@ public class HostResourceTest {
                 new ClusterControllerClientFactoryMock(),
                 EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService,
                 SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
-                clock);
+                clock,
+                applicationApiFactory);
 
         try {
             HostResource hostResource = new HostResource(alwaysRejectResolver, uriInfo);
@@ -261,7 +266,8 @@ public class HostResourceTest {
                 EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
                 mockInstanceLookupService,
                 SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
-                clock);
+                clock,
+                applicationApiFactory);
 
         try {
             HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver);