Merge pull request #10399 from vespa-engine/bratseth/avoid-node-changes-on-startup

Don't retire if we cannot fail

4 files changed, 143 insertions, 94 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index 8b79a303dbd..f6822f64570 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -100,7 +100,6 @@ class NodeAllocation {
             Node offered = offeredPriority.node;
 
             if (offered.allocation().isPresent()) {
-                boolean wantToRetireNode = false;
                 ClusterMembership membership = offered.allocation().get().membership();
                 if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application
                 if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type
@@ -108,15 +107,19 @@ class NodeAllocation {
                 if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal
                 if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure)
 
-                // conditions on which we want to retire nodes that were allocated previously
-                if ( violatesParentHostPolicy(this.nodes, offered)) wantToRetireNode = true;
-                if ( ! hasCompatibleFlavor(offered)) wantToRetireNode = true;
-                if ( offered.status().wantToRetire()) wantToRetireNode = true;
-                if ( requestedNodes.isExclusive() &&
-                     ! hostsOnly(application.tenant(), offered.parentHostname())) wantToRetireNode = true;
+                if (requestedNodes.considerRetiring()) {
+                    boolean wantToRetireNode = false;
+                    if (violatesParentHostPolicy(this.nodes, offered)) wantToRetireNode = true;
+                    if ( ! hasCompatibleFlavor(offered)) wantToRetireNode = true;
+                    if (offered.status().wantToRetire()) wantToRetireNode = true;
+                    if (requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), offered.parentHostname()))
+                        wantToRetireNode = true;
 
-                if (( ! saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) {
-                    accepted.add(acceptNode(offeredPriority, wantToRetireNode));
+                    if (( ! saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered))
+                        accepted.add(acceptNode(offeredPriority, wantToRetireNode));
+                }
+                else {
+                    accepted.add(acceptNode(offeredPriority, false));
                 }
             }
             else if ( ! saturated() && hasCompatibleFlavor(offered)) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
index 865b643b93b..da8d6a2ccf6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
@@ -41,6 +41,9 @@ public interface NodeSpec {
     /** Returns whether this should throw an exception if the requested nodes are not fully available */
     boolean canFail();
 
+    /** Returns whether we should retire nodes at all when fulfilling this spec */
+    boolean considerRetiring();
+
     /** Returns the ideal number of nodes that should be retired to fulfill this spec */
     int idealRetiredCount(int acceptedCount, int currentRetiredCount);
 
@@ -110,6 +113,12 @@ public interface NodeSpec {
         public boolean canFail() { return canFail; }
 
         @Override
+        public boolean considerRetiring() {
+            // If we cannot fail we cannot retire as we may end up without sufficient replacement capacity
+            return canFail();
+        }
+
+        @Override
         public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { return acceptedCount - this.count; }
 
         @Override
@@ -168,6 +177,9 @@ public interface NodeSpec {
         public boolean canFail() { return false; }
 
         @Override
+        public boolean considerRetiring() { return true; }
+
+        @Override
         public int idealRetiredCount(int acceptedCount, int currentRetiredCount) {
              // All nodes marked with wantToRetire get marked as retired just before this function is called,
              // the job of this function is to throttle the retired count. If no nodes are marked as retired
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
index cfad4757446..f0bd43b6bae 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
@@ -35,8 +35,10 @@ public class NodeTypeProvisioningTest {
 
     private final ApplicationId application = tester.makeApplicationId(); // application using proxy nodes
     private final Capacity capacity = Capacity.fromRequiredNodeType(NodeType.proxy);
-    private final ClusterSpec clusterSpec = ClusterSpec.request(
-            ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
+    private final ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
+                                                                ClusterSpec.Id.from("test"),
+                                                                Version.fromString("6.42"),
+                                                                false);
 
     @Before
     public void setup() {
@@ -156,13 +158,16 @@ public class NodeTypeProvisioningTest {
 
     @Test
     public void retire_multiple_proxy_simultaneously() {
-        MockDeployer deployer = new MockDeployer(
-                tester.provisioner(),
-                tester.clock(),
-                Collections.singletonMap(
-                        application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
-        RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer,
-                tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10));
+        MockDeployer deployer = new MockDeployer(tester.provisioner(),
+                                                 tester.clock(),
+                                                 Collections.singletonMap(application,
+                                                                          new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
+        RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(),
+                                                            tester.orchestrator(),
+                                                            deployer,
+                                                            tester.clock(),
+                                                            Duration.ofDays(30),
+                                                            Duration.ofMinutes(10));
         final int numNodesToRetire = 5;
 
         { // Deploy
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 89edb9cf579..851a734674f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -16,6 +16,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.maintenance.ReservationExpirer;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
@@ -47,6 +48,8 @@ import static org.junit.Assert.fail;
  */
 public class ProvisioningTest {
 
+    private static final NodeResources defaultResources = new NodeResources(1, 2, 3, 4);
+
     @Test
     public void application_deployment_constant_application_size() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
@@ -54,37 +57,37 @@ public class ProvisioningTest {
         ApplicationId application1 = tester.makeApplicationId();
         ApplicationId application2 = tester.makeApplicationId();
 
-        tester.makeReadyNodes(21, "d-1-1-1");
+        tester.makeReadyNodes(21, defaultResources);
 
         // deploy
-        SystemState state1 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state1.allHosts);
 
         // redeploy
-        SystemState state2 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state2 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         state2.assertEquals(state1);
         tester.activate(application1, state2.allHosts);
 
         // deploy another application
-        SystemState state1App2 = prepare(application2, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1App2 = prepare(application2, 2, 2, 3, 3, defaultResources, tester);
         assertFalse("Hosts to different apps are disjunct", state1App2.allHosts.removeAll(state1.allHosts));
         tester.activate(application2, state1App2.allHosts);
 
         // prepare twice
-        SystemState state3 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
-        SystemState state4 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state3 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
+        SystemState state4 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         state3.assertEquals(state2);
         state4.assertEquals(state3);
         tester.activate(application1, state4.allHosts);
 
         // remove nodes before deploying
-        SystemState state5 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state5 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         HostSpec removed = tester.removeOne(state5.allHosts);
         tester.activate(application1, state5.allHosts);
         assertEquals(removed.hostname(), tester.nodeRepository().getNodes(application1, Node.State.inactive).get(0).hostname());
 
         // remove some of the clusters
-        SystemState state6 = prepare(application1, 0, 2, 0, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state6 = prepare(application1, 0, 2, 0, 3, defaultResources, tester);
         tester.activate(application1, state6.allHosts);
         assertEquals(5, tester.getNodes(application1, Node.State.active).size());
         assertEquals(5, tester.getNodes(application1, Node.State.inactive).size());
@@ -103,14 +106,14 @@ public class ProvisioningTest {
         HostSpec failed = tester.removeOne(state1App2.allHosts);
         tester.fail(failed);
         assertEquals(9, tester.getNodes(application2, Node.State.active).size());
-        SystemState state2App2 = prepare(application2, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state2App2 = prepare(application2, 2, 2, 3, 3, defaultResources, tester);
         assertFalse("Hosts to different apps are disjunct", state2App2.allHosts.removeAll(state1.allHosts));
         assertEquals("A new node was reserved to replace the failed one", 10, state2App2.allHosts.size());
         assertFalse("The new host is not the failed one", state2App2.allHosts.contains(failed));
         tester.activate(application2, state2App2.allHosts);
 
         // deploy first app again
-        SystemState state7 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state7 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         state7.assertEquals(state1);
         tester.activate(application1, state7.allHosts);
         assertEquals(0, tester.getNodes(application1, Node.State.inactive).size());
@@ -132,12 +135,12 @@ public class ProvisioningTest {
     public void nodeVersionIsReturnedIfSet() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
 
-        tester.makeReadyNodes(4, new NodeResources(1, 1, 1, 0.3), NodeType.host, 1);
+        tester.makeReadyNodes(4, defaultResources, NodeType.host, 1);
         tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
 
         // deploy
         ApplicationId application1 = tester.makeApplicationId();
-        SystemState state1 = prepare(application1, 1, 1, 1, 1, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1 = prepare(application1, 1, 1, 1, 1, defaultResources, tester);
         tester.activate(application1, state1.allHosts);
 
         HostSpec host1 = state1.container0.iterator().next();
@@ -146,7 +149,7 @@ public class ProvisioningTest {
         tester.nodeRepository().write(node1.with(node1.status().withVespaVersion(Version.fromString("1.2.3"))), () -> {});
 
         // redeploy
-        SystemState state2 = prepare(application1, 1, 1, 1, 1, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state2 = prepare(application1, 1, 1, 1, 1, defaultResources, tester);
         tester.activate(application1, state2.allHosts);
 
         host1 = state2.container0.iterator().next();
@@ -159,20 +162,20 @@ public class ProvisioningTest {
 
         ApplicationId application1 = tester.makeApplicationId();
 
-        tester.makeReadyNodes(24, "d-1-1-1");
+        tester.makeReadyNodes(24, defaultResources);
 
         // deploy
-        SystemState state1 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state1.allHosts);
 
         // redeploy with increased sizes
-        SystemState state2 = prepare(application1, 3, 4, 4, 5, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state2 = prepare(application1, 3, 4, 4, 5, defaultResources, tester);
         state2.assertExtends(state1);
         assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size());
         tester.activate(application1, state2.allHosts);
 
         // decrease again
-        SystemState state3 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state3 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state3.allHosts);
         assertEquals("Superfluous container nodes are deactivated",
                      3-2 + 4-2, tester.getNodes(application1, Node.State.inactive).size());
@@ -180,7 +183,7 @@ public class ProvisioningTest {
                      4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
 
         // increase even more, and remove one node before deploying
-        SystemState state4 = prepare(application1, 4, 5, 5, 6, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state4 = prepare(application1, 4, 5, 5, 6, defaultResources, tester);
         assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size());
         assertEquals("Earlier retired nodes are not unretired before activate",
                      4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
@@ -196,7 +199,7 @@ public class ProvisioningTest {
                      0, tester.getNodes(application1, Node.State.active).retired().size());
 
         // decrease again
-        SystemState state5 = prepare(application1, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state5 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state5.allHosts);
         assertEquals("Superfluous container nodes are also deactivated",
                      4-2 + 5-2 + 1, tester.getNodes(application1, Node.State.inactive).size()); //
@@ -204,13 +207,13 @@ public class ProvisioningTest {
                      5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size());
 
         // increase content slightly
-        SystemState state6 = prepare(application1, 2, 2, 4, 3, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state6 = prepare(application1, 2, 2, 4, 3, defaultResources, tester);
         tester.activate(application1, state6.allHosts);
         assertEquals("One content node is unretired",
                      5-4 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size());
 
         // Then reserve more
-        SystemState state7 = prepare(application1, 8, 2, 2, 2, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state7 = prepare(application1, 8, 2, 2, 2, defaultResources, tester);
 
         // delete app
         NestedTransaction removeTransaction = new NestedTransaction();
@@ -222,14 +225,14 @@ public class ProvisioningTest {
 
     @Test
     public void application_deployment_multiple_flavors() {
+        NodeResources small = new NodeResources(1, 1, 1, 0.3);
+        NodeResources large = new NodeResources(2, 2, 2, 0.3);
+
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
         ApplicationId application1 = tester.makeApplicationId();
 
-        tester.makeReadyNodes(12, "d-1-1-1");
-
-        NodeResources small = new NodeResources(1, 1, 1, 0.3);
-        NodeResources large = new NodeResources(2, 2, 2, 0.3);
+        tester.makeReadyNodes(12, small);
 
         // deploy
         SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester);
@@ -239,7 +242,7 @@ public class ProvisioningTest {
         SystemState state2 = prepare(application1, 2, 2, 3, 3, small, tester);
         tester.activate(application1, state2.allHosts);
 
-        tester.makeReadyNodes(16, "d-2-2-2");
+        tester.makeReadyNodes(16, large);
 
         // redeploy with increased sizes and new flavor
         SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester);
@@ -284,25 +287,22 @@ public class ProvisioningTest {
 
         ApplicationId application1 = tester.makeApplicationId();
 
-        tester.makeReadyNodes(5, "d-1-1-1");
+        tester.makeReadyNodes(5, defaultResources);
 
         // deploy
-        SystemState state1 = prepare(application1, 2, 0, 3, 0,
-                                     new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester);
         tester.activate(application1, state1.allHosts);
 
         // redeploy a too large application
         try {
-            SystemState state2 = prepare(application1, 3, 0, 3, 0,
-                                         new NodeResources(1, 1, 1, 0.3), tester);
+            SystemState state2 = prepare(application1, 3, 0, 3, 0, defaultResources, tester);
             fail("Expected out of capacity exception");
         }
         catch (OutOfCapacityException expected) {
         }
 
         // deploy first state again
-        SystemState state3 = prepare(application1, 2, 0, 3, 0,
-                                     new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state3 = prepare(application1, 2, 0, 3, 0, defaultResources, tester);
         tester.activate(application1, state3.allHosts);
     }
 
@@ -310,12 +310,11 @@ public class ProvisioningTest {
     public void dev_deployment_node_size() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
 
-        tester.makeReadyNodes(4, new NodeResources(1, 1, 1, 0.3), NodeType.host, 1);
+        tester.makeReadyNodes(4, defaultResources, NodeType.host, 1);
         tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
 
         ApplicationId application = tester.makeApplicationId();
-        SystemState state = prepare(application, 2, 2, 3, 3,
-                                    new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester);
         assertEquals(4, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -324,11 +323,11 @@ public class ProvisioningTest {
     public void deploy_specific_vespa_version() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
 
-        tester.makeReadyNodes(4, new NodeResources(1, 1, 1, 0.3), NodeType.host, 1);
+        tester.makeReadyNodes(4, defaultResources, NodeType.host, 1);
         tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
 
         ApplicationId application = tester.makeApplicationId();
-        SystemState state = prepare(application, 2, 2, 3, 3, new NodeResources(1, 1, 1, 0.3), Version.fromString("6.91"), tester);
+        SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, Version.fromString("6.91"), tester);
         assertEquals(4, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -338,9 +337,8 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(4, "d-1-1-1");
-        SystemState state = prepare(application, 2, 2, 3, 3,
-                                    new NodeResources(1, 1, 1, 0.3), tester);
+        tester.makeReadyNodes(4, defaultResources);
+        SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester);
         assertEquals(4, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -350,9 +348,8 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(10, "d-1-1-1");
-        prepare(application, 1, 2, 3, 3,
-                new NodeResources(1, 1, 1, 0.3), tester);
+        tester.makeReadyNodes(10, defaultResources);
+        prepare(application, 1, 2, 3, 3, defaultResources, tester);
     }
 
     /** Dev always uses the zone default flavor */
@@ -370,15 +367,16 @@ public class ProvisioningTest {
         tester.activate(application, state.allHosts);
     }
 
-    /** Test always uses the zone default flavor */
+    /** Test always uses the zone default resources */
     @Test
-    public void test_deployment_flavor() {
+    public void test_deployment_resources() {
+        NodeResources large = new NodeResources(2, 2, 2, 0.3);
+
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(4, "d-2-2-2");
-        SystemState state = prepare(application, 2, 2, 3, 3,
-                                    new NodeResources(2, 2, 2, 0.3), tester);
+        tester.makeReadyNodes(4, large);
+        SystemState state = prepare(application, 2, 2, 3, 3, large, tester);
         assertEquals(4, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -388,9 +386,8 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(14, "d-1-1-1");
-        SystemState state = prepare(application, 1, 1, 1, 64,
-                                    new NodeResources(1, 1, 1, 0.3), tester); // becomes 1, 1, 1, 1, 6
+        tester.makeReadyNodes(14, defaultResources);
+        SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6
         assertEquals(9, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -399,10 +396,9 @@ public class ProvisioningTest {
     public void activate_after_reservation_timeout() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
-        tester.makeReadyNodes(10, "d-1-1-1");
+        tester.makeReadyNodes(10, defaultResources);
         ApplicationId application = tester.makeApplicationId();
-        SystemState state = prepare(application, 2, 2, 3, 3,
-                                    new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester);
 
         // Simulate expiry
         NestedTransaction deactivateTransaction = new NestedTransaction();
@@ -422,11 +418,10 @@ public class ProvisioningTest {
     public void out_of_capacity() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
-        tester.makeReadyNodes(9, "d-1-1-1"); // need 2+2+3+3=10
+        tester.makeReadyNodes(9, defaultResources); // need 2+2+3+3=10
         ApplicationId application = tester.makeApplicationId();
         try {
-            prepare(application, 2, 2, 3, 3,
-                    new NodeResources(1, 1, 1, 0.3), tester);
+            prepare(application, 2, 2, 3, 3, defaultResources, tester);
             fail("Expected exception");
         }
         catch (OutOfCapacityException e) {
@@ -437,7 +432,7 @@ public class ProvisioningTest {
     @Test
     public void out_of_capacity_but_cannot_fail() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
-        tester.makeReadyNodes(4, "d-1-1-1");
+        tester.makeReadyNodes(4, defaultResources);
         ApplicationId application = tester.makeApplicationId();
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content,
                                                   ClusterSpec.Id.from("music"),
@@ -453,12 +448,11 @@ public class ProvisioningTest {
 
         ApplicationId application = tester.makeApplicationId();
         // Flag all nodes for retirement
-        List<Node> readyNodes = tester.makeReadyNodes(5, "d-1-1-1");
+        List<Node> readyNodes = tester.makeReadyNodes(5, defaultResources);
         readyNodes.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true))));
 
         try {
-            prepare(application, 2, 0, 2, 0,
-                    new NodeResources(1, 1, 1, 0.3), tester);
+            prepare(application, 2, 0, 2, 0, defaultResources, tester);
             fail("Expected exception");
         } catch (OutOfCapacityException e) {
             assertTrue(e.getMessage().startsWith("Could not satisfy request"));
@@ -466,19 +460,57 @@ public class ProvisioningTest {
     }
 
     @Test
+    public void want_to_retire_but_cannot_fail() {
+        Capacity capacity =       Capacity.fromCount(5, Optional.of(defaultResources), false, true);
+        Capacity capacityFORCED = Capacity.fromCount(5, Optional.of(defaultResources), false, false);
+
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+
+        ApplicationId application = tester.makeApplicationId();
+
+        // Create 10 nodes
+        tester.makeReadyNodes(10, defaultResources);
+        // Allocate 5 nodes
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content,
+                                                  ClusterSpec.Id.from("music"),
+                                                  new com.yahoo.component.Version(4, 5, 6),
+                                                  false);
+        tester.activate(application, tester.prepare(application, cluster, capacity, 1));
+        assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).nonretired().size());
+        assertEquals(0, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size());
+
+        // Mark the nodes as want to retire
+        tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true))));
+        // redeploy without allow failing
+        tester.activate(application, tester.prepare(application, cluster, capacityFORCED, 1));
+
+        // Nodes are not retired since that is unsafe when we cannot fail
+        assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).nonretired().size());
+        assertEquals(0, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size());
+        // ... but we still want to
+        tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> assertTrue(node.status().wantToRetire()));
+
+        // redeploy with allowing failing
+        tester.activate(application, tester.prepare(application, cluster, capacity, 1));
+        // ... old nodes are now retired
+        assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).nonretired().size());
+        assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size());
+    }
+
+    @Test
     public void highest_node_indexes_are_retired_first() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
         ApplicationId application1 = tester.makeApplicationId();
 
-        tester.makeReadyNodes(14, "d-1-1-1");
+        tester.makeReadyNodes(14, defaultResources);
 
         // deploy
-        SystemState state1 = prepare(application1, 3, 3, 4, 4, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester);
         tester.activate(application1, state1.allHosts);
 
         // decrease cluster sizes
-        SystemState state2 = prepare(application1, 2, 2, 2, 2, new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state2 = prepare(application1, 2, 2, 2, 2, defaultResources, tester);
         tester.activate(application1, state2.allHosts);
 
         // content0
@@ -499,12 +531,11 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(10, "d-1-1-1");
+        tester.makeReadyNodes(10, defaultResources);
 
         // Deploy application
         {
-            SystemState state = prepare(application, 2, 0, 2, 0,
-                                        new NodeResources(1, 1, 1, 0.3), tester);
+            SystemState state = prepare(application, 2, 0, 2, 0,defaultResources, tester);
             tester.activate(application, state.allHosts);
             assertEquals(4, tester.getNodes(application, Node.State.active).size());
         }
@@ -514,7 +545,7 @@ public class ProvisioningTest {
             List<Node> nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2);
             nodesToRetire.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true))));
 
-            SystemState state = prepare(application, 2, 0, 2, 0, new NodeResources(1, 1, 1, 0.3), tester);
+            SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester);
             tester.activate(application, state.allHosts);
 
             List<Node> retiredNodes = tester.getNodes(application).retired().asList();
@@ -528,24 +559,22 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
         ApplicationId application = tester.makeApplicationId();
-        tester.makeReadyNodes(2, "d-1-1-1");
+        tester.makeReadyNodes(2, defaultResources);
 
         // Deploy fails with out of capacity
         try {
-            prepare(application, 2, 0, 2, 0,
-                    new NodeResources(1, 1, 1, 0.3), tester);
+            prepare(application, 2, 0, 2, 0, defaultResources, tester);
             fail("Expected exception");
         } catch (OutOfCapacityException ignored) {}
         assertEquals("Reserved a subset of required nodes", 2,
                      tester.getNodes(application, Node.State.reserved).size());
 
         // Enough nodes become available
-        tester.makeReadyNodes(2, "d-1-1-1");
+        tester.makeReadyNodes(2, defaultResources);
 
         // Deploy is retried after a few minutes
         tester.clock().advance(Duration.ofMinutes(2));
-        SystemState state = prepare(application, 2, 0, 2, 0,
-                                    new NodeResources(1, 1, 1, 0.3), tester);
+        SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester);
         List<Node> reserved = tester.getNodes(application, Node.State.reserved).asList();
         assertEquals("Reserved required nodes", 4, reserved.size());
         assertTrue("Time of event is updated for all nodes",
@@ -571,7 +600,7 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
         ApplicationId application = tester.makeApplicationId();
         try {
-            prepare(application, 1, 0, 1, 0, true, new NodeResources(1, 1, 1, 0.3), Version.fromString("6.42"), tester);
+            prepare(application, 1, 0, 1, 0, true, defaultResources, Version.fromString("6.42"), tester);
             fail("Expected exception");
         } catch (IllegalArgumentException ignored) {}
     }