Merge pull request #13675 from vespa-engine/bratseth/1-spare

From 0 to 1 spare

17 files changed, 76 insertions, 30 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index bec35e7ee4f..ff65dc18181 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -8,6 +8,7 @@ import com.yahoo.component.Version;
 import com.yahoo.concurrent.maintenance.JobControl;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
@@ -103,6 +104,7 @@ public class NodeRepository extends AbstractComponent {
     private final JobControl jobControl;
     private final Applications applications;
     private final boolean canProvisionHosts;
+    private final int spareCount;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -121,7 +123,8 @@ public class NodeRepository extends AbstractComponent {
              zone,
              new DnsNameResolver(),
              DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(),
-             provisionServiceProvider.getHostProvisioner().isPresent());
+             provisionServiceProvider.getHostProvisioner().isPresent(),
+             zone.environment() == Environment.prod ? 1 : 0);
     }
 
     /**
@@ -136,7 +139,8 @@ public class NodeRepository extends AbstractComponent {
                           NameResolver nameResolver,
                           DockerImage dockerImage,
                           boolean useCuratorClientCache,
-                          boolean canProvisionHosts) {
+                          boolean canProvisionHosts,
+                          int spareCount) {
         this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache);
         this.zone = zone;
         this.clock = clock;
@@ -150,6 +154,7 @@ public class NodeRepository extends AbstractComponent {
         this.jobControl = new JobControl(db);
         this.applications = new Applications(db);
         this.canProvisionHosts = canProvisionHosts;
+        this.spareCount = spareCount;
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (State state : State.values())
@@ -192,6 +197,9 @@ public class NodeRepository extends AbstractComponent {
 
     public HostResourcesCalculator resourcesCalculator() { return resourcesCalculator; }
 
+    /** The number of nodes we should ensure has free capacity for node failures whenever possible */
+    public int spareCount() { return spareCount; }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
index 54899372397..c8c66a77d60 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
@@ -48,7 +48,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {
                                    Duration interval) {
         this(deployer, nodeRepository, metric, interval,
              10_000 // Should take less than a few minutes
-        );
+            );
     }
 
     public SpareCapacityMaintainer(Deployer deployer,
@@ -103,10 +103,11 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {
 
         Node node = nodeWhichCantMove.get();
         NodeList allNodes = nodeRepository().list();
-        // Allocation will assign the two most empty nodes as "spares", which will not be allocated on
+        // Allocation will assign the spareCount most empty nodes as "spares", which will not be allocated on
         // unless needed for node failing. Our goal here is to make room on these spares for the given node
         HostCapacity hostCapacity = new HostCapacity(allNodes, nodeRepository().resourcesCalculator());
-        Set<Node> spareHosts = hostCapacity.findSpareHosts(allNodes.hosts().satisfies(node.resources()).asList(), 2);
+        Set<Node> spareHosts = hostCapacity.findSpareHosts(allNodes.hosts().satisfies(node.resources()).asList(),
+                                                           nodeRepository().spareCount());
         List<Node> hosts = allNodes.hosts().except(spareHosts).asList();
 
         CapacitySolver capacitySolver = new CapacitySolver(hostCapacity, maxIterations);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index f9d8213072e..faae52c72ec 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -46,8 +46,6 @@ import java.util.logging.Logger;
 public class NodeRepositoryProvisioner implements Provisioner {
 
     private static final Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName());
-    private static final int SPARE_CAPACITY_PROD = 0;
-    private static final int SPARE_CAPACITY_NONPROD = 0;
 
     private final NodeRepository nodeRepository;
     private final AllocationOptimizer allocationOptimizer;
@@ -69,13 +67,14 @@ public class NodeRepositoryProvisioner implements Provisioner {
                                                                .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService, flagSource));
         this.nodeResourceLimits = new NodeResourceLimits(nodeRepository);
         this.preparer = new Preparer(nodeRepository,
-                                     zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD,
+                                     nodeRepository.spareCount(),
                                      provisionServiceProvider.getHostProvisioner(),
                                      flagSource,
                                      loadBalancerProvisioner);
         this.activator = new Activator(nodeRepository, loadBalancerProvisioner);
     }
 
+
     /**
      * Returns a list of nodes in the prepared or active state, matching the given constraints.
      * The nodes are ordered by increasing index number.
@@ -133,8 +132,6 @@ public class NodeRepositoryProvisioner implements Provisioner {
         loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction));
     }
 
-    int getSpareCapacityProd() { return SPARE_CAPACITY_PROD; }
-
     /**
      * Returns the target cluster resources, a value between the min and max in the requested capacity,
      * and updates the application store with the received min and max.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 151bd80a7b7..bf56be4598c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -64,7 +64,9 @@ public class MockNodeRepository extends NodeRepository {
               Zone.defaultZone(),
               new MockNameResolver().mockAnyLookup(),
               DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-              true, false);
+              true,
+              false,
+              0);
         this.flavors = flavors;
 
         curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
index 52c68cd74b2..034a9fb27d0 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
@@ -40,7 +40,9 @@ public class NodeRepositoryTester {
                                             Zone.defaultZone(),
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true, false);
+                                            true,
+                                            false,
+                                            0);
     }
     
     public NodeRepository nodeRepository() { return nodeRepository; }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index 62e9a227109..cb3c7af797f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -63,7 +63,9 @@ public class CapacityCheckerTester {
                                             zone,
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true, false);
+                                            true,
+                                            false,
+                                            0);
     }
 
     private void updateCapacityChecker() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 24a0020df4f..7965293cf87 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -258,7 +258,9 @@ public class FailedExpirerTest {
                                                      zone,
                                                      new MockNameResolver().mockAnyLookup(),
                                                      DockerImage.fromString("docker-image"),
-                                                     true, false);
+                                                     true,
+                                                     false,
+                                                     0);
             this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
             this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30));
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index 754870e798e..59540223797 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -40,7 +40,9 @@ public class MaintenanceTester {
                                                                     zone,
                                                                     new MockNameResolver().mockAnyLookup(),
                                                                     DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                                    true, false);
+                                                                    true,
+                                                                    false,
+                                                                    0);
 
     public MaintenanceTester() {
         curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 727232e5c7c..567cec7ff3e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -84,7 +84,9 @@ public class MetricsReporterTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true, false);
+                                                           true,
+                                                           false,
+                                                           0);
         Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant);
         nodeRepository.addNodes(List.of(node), Agent.system);
         Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy);
@@ -150,7 +152,9 @@ public class MetricsReporterTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true, false);
+                                                           true,
+                                                           false,
+                                                           0);
 
         // Allow 4 containers
         Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 9cf03c6f33b..a97675f5d5e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -82,7 +82,9 @@ public class NodeFailTester {
                                             zone,
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true, false);
+                                            true,
+                                            false,
+                                            0);
         provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
         hostLivenessTracker = new TestHostLivenessTracker(clock);
         orchestrator = new OrchestratorMock();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index b9c57e013c3..e342e3b7302 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -61,7 +61,9 @@ public class OperatorChangeApplicationMaintainerTest {
                                                  zone,
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                 true, false);
+                                                 true,
+                                                 false,
+                                                 0);
         this.fixture = new Fixture(zone, nodeRepository);
 
         createReadyNodes(15, this.fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index b6ffe4ebe26..6d2b9a47e9f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -67,7 +67,9 @@ public class PeriodicApplicationMaintainerTest {
                                                  zone,
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                 true, false);
+                                                 true,
+                                                 false,
+                                                 0);
         this.fixture = new Fixture(zone, nodeRepository);
 
         createReadyNodes(15, fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index 2c0dac10f9f..705a038a79f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -52,7 +52,9 @@ public class ReservationExpirerTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true, false);
+                                                           true,
+                                                           false,
+                                                           0);
         NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource());
 
         List<Node> nodes = new ArrayList<>(2);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 29451d7b690..6a85654a2dd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -71,7 +71,9 @@ public class RetiredExpirerTest {
                                                                      zone,
                                                                      new MockNameResolver().mockAnyLookup(),
                                                                      DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                                     true, false);
+                                                                     true,
+                                                                     false,
+                                                                     0);
     private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
     private final Orchestrator orchestrator = mock(Orchestrator.class);
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
index fb84dc0a32a..ccb35e758bb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
@@ -137,7 +137,7 @@ public class SpareCapacityMaintainerTest {
         // By moving the 4 small nodes from host 2 we free up sufficient space on the third host to act as a spare for
         // application 0
         var tester = new SpareCapacityMaintainerTester();
-        setupMultipleHosts(tester, 5);
+        setupMultipleHosts(tester, 4);
 
         tester.maintainer.maintain();
         assertEquals(1, tester.deployer.redeployments);
@@ -148,7 +148,7 @@ public class SpareCapacityMaintainerTest {
     @Test
     public void testMultipleNodesMustMoveFromOneHostButInsufficientCapacity() {
         var tester = new SpareCapacityMaintainerTester();
-        setupMultipleHosts(tester, 4);
+        setupMultipleHosts(tester, 3);
 
         tester.maintainer.maintain();
         assertEquals(0, tester.deployer.redeployments);
@@ -244,7 +244,10 @@ public class SpareCapacityMaintainerTest {
                                                 new ManualClock(),
                                                 new Zone(Environment.prod, RegionName.from("us-east-3")),
                                                 new MockNameResolver().mockAnyLookup(),
-                                                DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, false);
+                                                DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
+                                                true,
+                                                false,
+                                                1);
             deployer = new MockDeployer(nodeRepository);
             maintainer = new SpareCapacityMaintainer(deployer, nodeRepository, metric, Duration.ofDays(1), maxIterations);
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
index 98ec01e8e95..100cf5704eb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
@@ -60,7 +60,11 @@ public class DynamicDockerAllocationTest {
      */
     @Test
     public void relocate_nodes_from_spare_hosts() {
-        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
+        int spareCount = 1;
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east")))
+                                                                    .flavorsConfig(flavorsConfig())
+                                                                    .spareCount(spareCount)
+                                                                    .build();
         tester.makeReadyNodes(4, "host-small", NodeType.host, 32);
         tester.deployZoneApp();
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
@@ -89,7 +93,7 @@ public class DynamicDockerAllocationTest {
                 hostsWithChildren.add(node.parentHostname().get());
             }
         }
-        assertEquals(4 - tester.provisioner().getSpareCapacityProd(), hostsWithChildren.size());
+        assertEquals(4 - spareCount, hostsWithChildren.size());
 
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index e73aeb05ce3..204d1171919 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -91,7 +91,8 @@ public class ProvisioningTester {
                               Orchestrator orchestrator,
                               HostProvisioner hostProvisioner,
                               LoadBalancerServiceMock loadBalancerService,
-                              FlagSource flagSource) {
+                              FlagSource flagSource,
+                              int spareCount) {
         this.curator = curator;
         this.nodeFlavors = nodeFlavors;
         this.clock = new ManualClock();
@@ -104,7 +105,8 @@ public class ProvisioningTester {
                                                  nameResolver,
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
                                                  true,
-                                                 provisionServiceProvider.getHostProvisioner().isPresent());
+                                                 provisionServiceProvider.getHostProvisioner().isPresent(),
+                                                 spareCount);
         this.orchestrator = orchestrator;
         this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource);
         this.capacityPolicies = new CapacityPolicies(nodeRepository);
@@ -541,6 +543,7 @@ public class ProvisioningTester {
         private HostProvisioner hostProvisioner;
         private LoadBalancerServiceMock loadBalancerService;
         private FlagSource flagSource;
+        private int spareCount = 0;
 
         public Builder curator(Curator curator) {
             this.curator = curator;
@@ -598,6 +601,11 @@ public class ProvisioningTester {
             return this;
         }
 
+        public Builder spareCount(int spareCount) {
+            this.spareCount = spareCount;
+            return this;
+        }
+
         public ProvisioningTester build() {
             Orchestrator orchestrator = Optional.ofNullable(this.orchestrator)
                     .orElseGet(() -> {
@@ -618,7 +626,8 @@ public class ProvisioningTester {
                                           orchestrator,
                                           hostProvisioner,
                                           Optional.ofNullable(loadBalancerService).orElseGet(LoadBalancerServiceMock::new),
-                                          Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new));
+                                          Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new),
+                                          spareCount);
         }
 
         private static FlavorsConfig asConfig(List<Flavor> flavors) {