Merge pull request #16417 from vespa-engine/bratseth/less-inactive

Bratseth/less inactive 

25 files changed, 158 insertions, 105 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
index 14658e57c1b..f4b79129ada 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
@@ -77,10 +77,8 @@ public final class ClusterSpec {
      */
     public boolean isExclusive() { return exclusive; }
 
-    /** Whether this cluster has state */
-    public boolean isStateful() {
-        return stateful;
-    }
+    /** Returns whether this cluster has state */
+    public boolean isStateful() { return stateful; }
 
     public ClusterSpec with(Optional<Group> newGroup) {
         return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo, stateful);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index b0b61e8a6b2..19c1fa090c9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -69,6 +69,16 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> {
         return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().isContainer());
     }
 
+    /** Returns the subset of nodes that run a stateless service */
+    public NodeList stateless() {
+        return matching(node -> node.allocation().isPresent() && ! node.allocation().get().membership().cluster().isStateful());
+    }
+
+    /** Returns the subset of nodes that run a stateful service */
+    public NodeList stateful() {
+        return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().isStateful());
+    }
+
     /** Returns the subset of nodes that are currently changing their Vespa version */
     public NodeList changingVersion() {
         return matching(node -> node.status().vespaVersion().isPresent() &&
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 4eb38fa650e..8e14b61db9a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer;
 import com.yahoo.vespa.hosted.provision.maintenance.PeriodicApplicationMaintainer;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.NodeAcl;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
@@ -410,7 +411,7 @@ public class NodeRepository extends AbstractComponent {
         for (Node node : nodes) {
             if ( ! node.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))
                 illegal("Cannot add " + node + ": This is not a docker node");
-            if ( ! node.allocation().isPresent())
+            if (node.allocation().isEmpty())
                 illegal("Cannot add " + node + ": Docker containers needs to be allocated");
             Optional<Node> existing = getNode(node.hostname());
             if (existing.isPresent())
@@ -514,7 +515,13 @@ public class NodeRepository extends AbstractComponent {
      * transaction commits.
      */
     public List<Node> deactivate(List<Node> nodes, ApplicationTransaction transaction) {
-        return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction.nested());
+        var stateless = NodeList.copyOf(nodes).stateless();
+        var stateful  = NodeList.copyOf(nodes).stateful();
+        List<Node> written = new ArrayList<>();
+        written.addAll(deallocate(stateless.asList(), Agent.application, "Deactivated by application", transaction.nested()));
+        written.addAll(db.writeTo(State.inactive, stateful.asList(), Agent.application, Optional.empty(), transaction.nested()));
+        return written;
+
     }
 
     /** Removes this application: Active nodes are deactivated while all non-active nodes are set dirty. */
@@ -531,22 +538,11 @@ public class NodeRepository extends AbstractComponent {
     }
 
     /** Move nodes to the dirty state */
-    public List<Node> setDirty(List<Node> nodes, Agent agent, String reason) {
-        return performOn(NodeListFilter.from(nodes), (node, lock) -> setDirty(node, agent, reason));
-    }
-
-    /**
-     * Set a node dirty, allowed if it is in the provisioned, inactive, failed or parked state.
-     * Use this to clean newly provisioned nodes or to recycle failed nodes which have been repaired or put on hold.
-     *
-     * @throws IllegalArgumentException if the node has hardware failure
-     */
-    public Node setDirty(Node node, Agent agent, String reason) {
-        return db.writeTo(State.dirty, node, agent, Optional.of(reason));
+    public List<Node> deallocate(List<Node> nodes, Agent agent, String reason) {
+        return performOn(NodeListFilter.from(nodes), (node, lock) -> deallocate(node, agent, reason));
     }
 
-
-    public List<Node> dirtyRecursively(String hostname, Agent agent, String reason) {
+    public List<Node> deallocateRecursively(String hostname, Agent agent, String reason) {
         Node nodeToDirty = getNode(hostname).orElseThrow(() ->
                 new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found"));
 
@@ -568,7 +564,37 @@ public class NodeRepository extends AbstractComponent {
             illegal("Could not deallocate " + nodeToDirty + ": " +
                     hostnamesNotAllowedToDirty + " are not in states [provisioned, failed, parked, breakfixed]");
 
-        return nodesToDirty.stream().map(node -> setDirty(node, agent, reason)).collect(Collectors.toList());
+        return nodesToDirty.stream().map(node -> deallocate(node, agent, reason)).collect(Collectors.toList());
+    }
+
+    /**
+     * Set a node dirty  or parked, allowed if it is in the provisioned, inactive, failed or parked state.
+     * Use this to clean newly provisioned nodes or to recycle failed nodes which have been repaired or put on hold.
+     */
+    public Node deallocate(Node node, Agent agent, String reason) {
+        NestedTransaction transaction = new NestedTransaction();
+        Node deallocated = deallocate(node, agent, reason, transaction);
+        transaction.commit();
+        return deallocated;
+    }
+
+    public List<Node> deallocate(List<Node> nodes, Agent agent, String reason, NestedTransaction transaction) {
+        return nodes.stream().map(node -> deallocate(node, agent, reason, transaction)).collect(Collectors.toList());
+    }
+
+    public Node deallocate(Node node, Agent agent, String reason, NestedTransaction transaction) {
+        if (node.state() != State.parked && agent != Agent.operator
+            && (node.status().wantToDeprovision() || retiredByOperator(node)))
+            return park(node.hostname(), false, agent, reason, transaction);
+        else
+            return db.writeTo(State.dirty, List.of(node), agent, Optional.of(reason), transaction).get(0);
+    }
+
+    private static boolean retiredByOperator(Node node) {
+        return node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire)
+                                                   .map(History.Event::agent)
+                                                   .map(agent -> agent == Agent.operator)
+                                                   .orElse(false);
     }
 
     /**
@@ -597,7 +623,14 @@ public class NodeRepository extends AbstractComponent {
      * @throws NoSuchNodeException if the node is not found
      */
     public Node park(String hostname, boolean keepAllocation, Agent agent, String reason) {
-        return move(hostname, keepAllocation, State.parked, agent, Optional.of(reason));
+        NestedTransaction transaction = new NestedTransaction();
+        Node parked = park(hostname, keepAllocation, agent, reason, transaction);
+        transaction.commit();
+        return parked;
+    }
+
+    public Node park(String hostname, boolean keepAllocation, Agent agent, String reason, NestedTransaction transaction) {
+        return move(hostname, keepAllocation, State.parked, agent, Optional.of(reason), transaction);
     }
 
     /**
@@ -644,6 +677,14 @@ public class NodeRepository extends AbstractComponent {
     }
 
     private Node move(String hostname, boolean keepAllocation, State toState, Agent agent, Optional<String> reason) {
+        NestedTransaction transaction = new NestedTransaction();
+        Node moved = move(hostname, keepAllocation, toState, agent, reason, transaction);
+        transaction.commit();
+        return moved;
+    }
+
+    private Node move(String hostname, boolean keepAllocation, State toState, Agent agent, Optional<String> reason,
+                      NestedTransaction transaction) {
         Node node = getNode(hostname).orElseThrow(() ->
                 new NoSuchNodeException("Could not move " + hostname + " to " + toState + ": Node not found"));
 
@@ -651,10 +692,17 @@ public class NodeRepository extends AbstractComponent {
             node = node.withoutAllocation();
         }
 
-        return move(node, toState, agent, reason);
+        return move(node, toState, agent, reason, transaction);
     }
 
     private Node move(Node node, State toState, Agent agent, Optional<String> reason) {
+        NestedTransaction transaction = new NestedTransaction();
+        Node moved = move(node, toState, agent, reason, transaction);
+        transaction.commit();
+        return moved;
+    }
+
+    private Node move(Node node, State toState, Agent agent, Optional<String> reason, NestedTransaction transaction) {
         if (toState == Node.State.active && node.allocation().isEmpty())
             illegal("Could not set " + node + " active. It has no allocation.");
 
@@ -667,7 +715,7 @@ public class NodeRepository extends AbstractComponent {
                         illegal("Could not set " + node + " active: Same cluster and index as " + currentActive);
                 }
             }
-            return db.writeTo(toState, node, agent, reason);
+            return db.writeTo(toState, List.of(node), agent, reason, transaction).get(0);
         }
     }
 
@@ -981,4 +1029,5 @@ public class NodeRepository extends AbstractComponent {
     private void illegal(String message) {
         throw new IllegalArgumentException(message);
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index b55f49722cb..1a47af6b929 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -112,7 +112,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer {
                 nodesToRecycle.add(candidate);
             }
         }
-        nodeRepository.setDirty(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer");
+        nodeRepository.deallocate(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer");
     }
 
     /** Returns whether the current node fail count should be used as an indicator of hardware issue */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
index 392146f6ead..231d2ac08b1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
@@ -14,14 +14,9 @@ import java.util.List;
 /**
  * Maintenance job which moves inactive nodes to dirty or parked after timeout.
  *
- * The timeout is in place for two reasons:
- *
- * - To ensure that the new application configuration has time to
- *   propagate before the node is used for something else.
- *
- * - To provide a grace period in which nodes can be brought back to active
- *   if they were deactivated in error. As inactive nodes retain their state
- *   they can be brought back to active and correct state faster than a new node.
+ * The timeout is in place to provide a grace period in which nodes can be brought back to active
+ * if they were deactivated in error. As inactive nodes retain their state
+ * they can be brought back to active and correct state faster than a new node.
  *
  * Nodes with following flags set are not reusable and will be moved to parked
  * instead of dirty:
@@ -44,11 +39,7 @@ public class InactiveExpirer extends Expirer {
     @Override
     protected void expire(List<Node> expired) {
         expired.forEach(node -> {
-            if (node.status().wantToDeprovision() || retiredByOperator(node)) {
-                nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer");
-            } else {
-                nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer");
-            }
+            nodeRepository.deallocate(node, Agent.InactiveExpirer, "Expired by InactiveExpirer");
         });
     }
 
@@ -58,11 +49,4 @@ public class InactiveExpirer extends Expirer {
                || node.allocation().get().owner().instance().isTester();
     }
 
-    private static boolean retiredByOperator(Node node) {
-        return node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire)
-                                                   .map(History.Event::agent)
-                                                   .map(agent -> agent == Agent.operator)
-                                                   .orElse(false);
-    }
-
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
index 66ddf7f9ffe..52c487c28cf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
@@ -186,7 +186,7 @@ class MaintenanceDeployment implements Closeable {
 
                     // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host
                     expectedNewNode.flatMap(node -> nodeRepository.getNode(node.hostname(), Node.State.reserved))
-                                   .ifPresent(node -> nodeRepository.setDirty(node, agent, "Expired by " + agent));
+                                   .ifPresent(node -> nodeRepository.deallocate(node, agent, "Expired by " + agent));
                 }
             }
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
index c6427123d09..1967615de02 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
@@ -25,6 +25,6 @@ public class ReservationExpirer extends Expirer {
     }
 
     @Override
-    protected void expire(List<Node> expired) { nodeRepository().setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); }
+    protected void expire(List<Node> expired) { nodeRepository().deallocate(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
index 9b5dac49e32..08bfd104863 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
@@ -142,7 +142,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler {
             return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked");
         }
         else if (path.startsWith("/nodes/v2/state/dirty/")) {
-            List<Node> dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API");
+            List<Node> dirtiedNodes = nodeRepository.deallocateRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API");
             return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty");
         }
         else if (path.startsWith("/nodes/v2/state/active/")) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 1f8f3d32043..cf1ccf9a052 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -142,11 +142,11 @@ public class MockNodeRepository extends NodeRepository {
         nodes = addNodes(nodes, Agent.system);
         nodes.remove(node7);
         nodes.remove(node55);
-        nodes = setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = deallocate(nodes, Agent.system, getClass().getSimpleName());
         setReady(nodes, Agent.system, getClass().getSimpleName());
 
         fail(node5.hostname(), Agent.system, getClass().getSimpleName());
-        dirtyRecursively(node55.hostname(), Agent.system, getClass().getSimpleName());
+        deallocateRecursively(node55.hostname(), Agent.system, getClass().getSimpleName());
 
         fail("dockerhost6.yahoo.com", Agent.operator, getClass().getSimpleName());
         removeRecursively("dockerhost6.yahoo.com");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
index bcf53a07490..5f357b9e4b0 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
@@ -228,12 +228,12 @@ public class NodeRepositoryTest {
         assertEquals(6, tester.nodeRepository().getNodes().size());
 
         // Should be OK to dirty host2 as it is in provisioned and its only child is in failed
-        tester.nodeRepository().dirtyRecursively("host2", Agent.system, NodeRepositoryTest.class.getSimpleName());
+        tester.nodeRepository().deallocateRecursively("host2", Agent.system, NodeRepositoryTest.class.getSimpleName());
         assertEquals(asSet("host2", "node20"), filterNodes(tester, node -> node.state() == Node.State.dirty));
 
         // Cant dirty host1, node11 is ready and node12 is active
         try {
-            tester.nodeRepository().dirtyRecursively("host1", Agent.system, NodeRepositoryTest.class.getSimpleName());
+            tester.nodeRepository().deallocateRecursively("host1", Agent.system, NodeRepositoryTest.class.getSimpleName());
             fail("Should not be able to dirty host1");
         } catch (IllegalArgumentException ignored) { } // Expected;
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 940404fa605..9f2f7541c91 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -312,7 +312,7 @@ public class FailedExpirerTest {
             List<Node> nodes = Stream.of(hostname)
                                      .map(this::get)
                                      .collect(Collectors.toList());
-            nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+            nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
             nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
             return this;
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
index 8bcf1552204..60ca625d07e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -60,12 +60,6 @@ public class LoadBalancerExpirerTest {
 
         // Expirer defers removal while nodes are still allocated to application
         expirer.maintain();
-        assertEquals(3, tester.loadBalancerService().instances().size());
-        assertEquals(2, tester.loadBalancerService().instances().get(lb1).reals().size());
-        dirtyNodesOf(app1, cluster1);
-
-        // Expirer prunes reals before expiration time of load balancer itself
-        expirer.maintain();
         assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals());
         assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals());
 
@@ -138,11 +132,11 @@ public class LoadBalancerExpirerTest {
     }
 
     private void dirtyNodesOf(ApplicationId application, ClusterSpec.Id cluster) {
-        tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application).stream()
-                                               .filter(node -> node.allocation().isPresent())
-                                               .filter(node -> node.allocation().get().membership().cluster().id().equals(cluster))
-                                               .collect(Collectors.toList()),
-                                         Agent.system, this.getClass().getSimpleName());
+        tester.nodeRepository().deallocate(tester.nodeRepository().getNodes(application).stream()
+                                                 .filter(node -> node.allocation().isPresent())
+                                                 .filter(node -> node.allocation().get().membership().cluster().id().equals(cluster))
+                                                 .collect(Collectors.toList()),
+                                           Agent.system, this.getClass().getSimpleName());
     }
 
     private void deployApplication(ApplicationId application, ClusterSpec.Id... clusters) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index c03e489def2..9adba744101 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -163,7 +163,7 @@ public class MetricsReporterTest {
         Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost",
                                       nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build();
         nodeRepository.addNodes(List.of(dockerHost), Agent.system);
-        nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName());
+        nodeRepository.deallocateRecursively("dockerHost", Agent.system, getClass().getSimpleName());
         nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName());
 
         Node container1 = Node.createDockerNode(Set.of("::2"), "container1",
@@ -225,7 +225,7 @@ public class MetricsReporterTest {
         ApplicationId application = ApplicationId.from("t1", "a1", "default");
         Map<String, String> dimensions = Map.of("applicationId", application.toFullString());
         NodeResources resources = new NodeResources(2, 8, 100, 1);
-        List<Node> activeNodes = tester.deploy(application, Capacity.from(new ClusterResources(4, 1, resources)));
+        List<Node> activeNodes = tester.deploy(application, ProvisioningTester.contentClusterSpec(), Capacity.from(new ClusterResources(4, 1, resources)));
         metricsReporter.maintain();
         assertEquals(0D, getMetric("nodes.nonActiveFraction", metric, dimensions));
         assertEquals(4, getMetric("nodes.active", metric, dimensions));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 12b682ae26c..d0473d08ea2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -259,7 +259,7 @@ public class NodeFailTester {
         }
 
         nodes = nodeRepository.addNodes(nodes, Agent.system);
-        nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
 
@@ -267,7 +267,7 @@ public class NodeFailTester {
         List<Node> nodes = tester.makeProvisionedNodes(count, (index) -> "parent" + index,
                                                        hostFlavors.getFlavorOrThrow("default"),
                                                        Optional.empty(), NodeType.host, 10, false);
-        nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
         tester.activateTenantHosts();
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index cd188bc017f..ae1e36d73dd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -96,10 +96,10 @@ public class PeriodicApplicationMaintainerTest {
         // Cause maintenance deployment which will update the applications with the re-activated nodes
         clock.advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited
         fixture.runApplicationMaintainer();
-        assertEquals("Superflous content nodes are retired",
+        assertEquals("Superfluous content nodes are retired",
                      reactivatedInApp2, fixture.getNodes(Node.State.active).retired().size());
-        assertEquals("Superflous container nodes are deactivated (this makes little point for container nodes)",
-                     reactivatedInApp1, fixture.getNodes(Node.State.inactive).size());
+        assertEquals("Superfluous container nodes are deallocated",
+                     reactivatedInApp1, fixture.getNodes(Node.State.dirty).size());
     }
 
     @Test(timeout = 60_000)
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
index 0246ec524e3..ef0e6d4ddc4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
@@ -14,11 +15,14 @@ import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.OutOfCapacityException;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.Agent;
 import org.junit.Test;
 
 import java.util.HashSet;
@@ -356,11 +360,15 @@ public class DockerProvisioningTest {
         tester.makeReadyHosts(5, r).activateTenantHosts();
 
         ApplicationId app1 = ProvisioningTester.applicationId("app1");
-        ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build();
+        ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build();
 
         tester.activate(app1, cluster1, Capacity.from(new ClusterResources(5, 1, r)));
         tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r)));
 
+        var tx = new ApplicationTransaction(new ProvisionLock(app1, tester.nodeRepository().lock(app1)), new NestedTransaction());
+        tester.nodeRepository().deactivate(tester.nodeRepository().list(app1, Node.State.active).retired().asList(), tx);
+        tx.nested().commit();
+
         assertEquals(2, tester.getNodes(app1, Node.State.active).size());
         assertEquals(3, tester.getNodes(app1, Node.State.inactive).size());
 
@@ -372,16 +380,16 @@ public class DockerProvisioningTest {
     }
 
     @Test
-    public void inactive_container_nodes_are_reused() {
-        assertInactiveReuse(ClusterSpec.Type.container);
+    public void inactive_container_nodes_are_not_reused() {
+        assertInactiveReuse(ClusterSpec.Type.container, false);
     }
 
     @Test
     public void inactive_content_nodes_are_reused() {
-        assertInactiveReuse(ClusterSpec.Type.content);
+        assertInactiveReuse(ClusterSpec.Type.content, true);
     }
 
-    private void assertInactiveReuse(ClusterSpec.Type clusterType) {
+    private void assertInactiveReuse(ClusterSpec.Type clusterType, boolean expectedReuse) {
         NodeResources r = new NodeResources(20, 40, 100, 4);
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east")))
                                                                     .flavors(List.of(new Flavor(r)))
@@ -398,9 +406,18 @@ public class DockerProvisioningTest {
         tester.nodeRepository().setRemovable(app1, tester.getNodes(app1).retired().asList());
         tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r)));
 
-        assertEquals(2, tester.getNodes(app1, Node.State.inactive).size());
-        tester.activate(app1, cluster1, Capacity.from(new ClusterResources(4, 1, r)));
-        assertEquals(0, tester.getNodes(app1, Node.State.inactive).size());
+        if (expectedReuse) {
+            assertEquals(2, tester.getNodes(app1, Node.State.inactive).size());
+            tester.activate(app1, cluster1, Capacity.from(new ClusterResources(4, 1, r)));
+            assertEquals(0, tester.getNodes(app1, Node.State.inactive).size());
+        }
+        else {
+            assertEquals(0, tester.getNodes(app1, Node.State.inactive).size());
+            assertEquals(2, tester.nodeRepository().getNodes(Node.State.dirty).size());
+            tester.nodeRepository().setReady(tester.nodeRepository().getNodes(Node.State.dirty), Agent.system, "test");
+            tester.activate(app1, cluster1, Capacity.from(new ClusterResources(4, 1, r)));
+        }
+
     }
 
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index eef342b527b..105f2122e0c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -263,7 +263,7 @@ public class LoadBalancerProvisionerTest {
     }
 
     private void dirtyNodesOf(ApplicationId application) {
-        tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName());
+        tester.nodeRepository().deallocate(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName());
     }
 
     private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
index acf7bda3cbf..bf2a6ba3627 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
@@ -153,7 +153,7 @@ public class NodeTypeProvisioningTest {
             assertEquals(10, nodes.size());
 
             // Verify that the node is now inactive
-            assertEquals(Node.State.inactive, tester.nodeRepository().getNode(nodeToRetire.hostname())
+            assertEquals(Node.State.dirty, tester.nodeRepository().getNode(nodeToRetire.hostname())
                     .orElseThrow(RuntimeException::new).state());
         }
     }
@@ -232,7 +232,7 @@ public class NodeTypeProvisioningTest {
             assertEquals(10, nodes.size());
 
             // Verify the node we previously set to retire has finished retiring
-            assertEquals(Node.State.inactive, tester.nodeRepository().getNode(currentyRetiringHostname)
+            assertEquals(Node.State.dirty, tester.nodeRepository().getNode(currentyRetiringHostname)
                     .orElseThrow(RuntimeException::new).state());
 
             // Verify that a node is currently retiring
@@ -263,9 +263,9 @@ public class NodeTypeProvisioningTest {
                 .count();
         assertEquals(11 - numNodesToRetire, numRetiredActiveProxyNodes);
 
-        // All the nodes that were marked with wantToRetire earlier are now inactive
+        // All the nodes that were marked with wantToRetire earlier are now dirty
         assertEquals(nodesToRetire.stream().map(Node::hostname).collect(Collectors.toSet()),
-                tester.nodeRepository().getNodes(Node.State.inactive).stream().map(Node::hostname).collect(Collectors.toSet()));
+                tester.nodeRepository().getNodes(Node.State.dirty).stream().map(Node::hostname).collect(Collectors.toSet()));
     }
 
     private List<HostSpec> deployProxies(ApplicationId application, ProvisioningTester tester) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 611c3839f56..7778077110a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -32,6 +32,7 @@ import com.yahoo.vespa.service.duper.InfraApplication;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -100,14 +101,15 @@ public class ProvisioningTest {
         SystemState state6 = prepare(application1, 0, 2, 0, 3, defaultResources, tester);
         tester.activate(application1, state6.allHosts);
         assertEquals(5, tester.getNodes(application1, Node.State.active).size());
-        assertEquals(5, tester.getNodes(application1, Node.State.inactive).size());
+        assertEquals(3, tester.getNodes(application1, Node.State.inactive).size());
 
         // delete app
         NodeList previouslyActive = tester.getNodes(application1, Node.State.active);
         NodeList previouslyInactive = tester.getNodes(application1, Node.State.inactive);
         tester.remove(application1);
-        assertEquals(tester.toHostNames(previouslyActive.asList()), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive)));
-        assertEquals(tester.toHostNames(previouslyInactive.asList()), tester.toHostNames(tester.nodeRepository().getNodes(Node.State.dirty)));
+        assertEquals(tester.toHostNames(previouslyActive.not().container().asList()),
+                     tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive)));
+        assertTrue(tester.nodeRepository().getNodes(Node.State.dirty).containsAll(previouslyActive.container().asList()));
         assertEquals(0, tester.getNodes(application1, Node.State.active).size());
         assertTrue(tester.nodeRepository().applications().get(application1).isEmpty());
 
@@ -203,7 +205,7 @@ public class ProvisioningTest {
 
         ApplicationId application1 = ProvisioningTester.applicationId();
 
-        tester.makeReadyHosts(24, defaultResources);
+        tester.makeReadyHosts(30, defaultResources);
         tester.activateTenantHosts();
 
         // deploy
@@ -219,8 +221,8 @@ public class ProvisioningTest {
         // decrease again
         SystemState state3 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state3.allHosts);
-        assertEquals("Superfluous container nodes are deactivated",
-                     3-2 + 4-2, tester.getNodes(application1, Node.State.inactive).size());
+        assertEquals("Superfluous container nodes are dirtyed",
+                     3-2 + 4-2, tester.nodeRepository().getNodes(Node.State.dirty).size());
         assertEquals("Superfluous content nodes are retired",
                      4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
 
@@ -229,7 +231,6 @@ public class ProvisioningTest {
         assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size());
         assertEquals("Earlier retired nodes are not unretired before activate",
                      4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
-        state4.assertExtends(state2);
         assertEquals("New and inactive nodes are reserved", 4 + 3, tester.getNodes(application1, Node.State.reserved).size());
         // Remove a retired host from one of the content clusters (which one is random depending on host names)
         HostSpec removed = state4.removeHost(tester.getNodes(application1, Node.State.active).retired().asList().get(0).hostname());
@@ -243,8 +244,8 @@ public class ProvisioningTest {
         // decrease again
         SystemState state5 = prepare(application1, 2, 2, 3, 3, defaultResources, tester);
         tester.activate(application1, state5.allHosts);
-        assertEquals("Superfluous container nodes are also deactivated",
-                     4-2 + 5-2 + 1, tester.getNodes(application1, Node.State.inactive).size()); //
+        assertEquals("Superfluous container nodes are also dirtyed",
+                     4-2 + 5-2 + 1 + 4-2, tester.nodeRepository().getNodes(Node.State.dirty).size());
         assertEquals("Superfluous content nodes are retired",
                      5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size());
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index ebd856e96a0..7c43a8d5859 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -456,7 +456,7 @@ public class ProvisioningTester {
         }
 
         nodes = nodeRepository.addNodes(nodes, Agent.system);
-        nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
 
         ConfigServerApplication application = new ConfigServerApplication();
@@ -482,7 +482,7 @@ public class ProvisioningTester {
     }
     public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = makeProvisionedNodes(n, flavor, reservedTo, type, ipAddressPoolSize, dualStack);
-        nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
 
@@ -523,7 +523,7 @@ public class ProvisioningTester {
             nodes.add(builder.build());
         }
         nodes = nodeRepository.addNodes(nodes, Agent.system);
-        nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
+        nodes = nodeRepository.deallocate(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
         return nodes;
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json
index 3107311b792..0a2bf95c5bd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json
@@ -1,14 +1,14 @@
 {
   "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com",
   "id": "host55.yahoo.com",
-  "state": "dirty",
+  "state": "parked",
   "type": "tenant",
   "hostname": "host55.yahoo.com",
   "openStackId": "node55",
   "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]",
   "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"},
   "environment": "DOCKER_CONTAINER",
-  "rebootGeneration": 1,
+  "rebootGeneration": 0,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "wantToRetire": true,
@@ -20,7 +20,7 @@
       "agent": "system"
     },
     {
-      "event": "deallocated",
+      "event": "parked",
       "at": 123,
       "agent": "system"
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
index 2b650bad39b..03df6c8e1dc 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
@@ -17,8 +17,8 @@
     @include(node2.json),
     @include(docker-node1.json),
     @include(node1.json),
-    @include(node55.json),
     @include(node5.json),
+    @include(node55.json),
     @include(dockerhost6.json)
   ]
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
index 55e216f454a..8835945dc92 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
@@ -17,7 +17,7 @@
     @include(node2.json),
     @include(docker-node1.json),
     @include(node1.json),
-    @include(node55.json),
-    @include(node5.json)
+    @include(node5.json),
+    @include(node55.json)
   ]
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
index 54ff2bc232f..db4d0bb1682 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
@@ -52,10 +52,10 @@
       "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com"
     }
   ]
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
index 27767be6315..68ff9fedc00 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
@@ -45,7 +45,6 @@
     "dirty": {
       "url": "http://localhost:8080/nodes/v2/state/dirty",
       "nodes": [
-        @include(node55.json)
       ]
     },
     "failed": {
@@ -57,6 +56,7 @@
     "parked": {
       "url": "http://localhost:8080/nodes/v2/state/parked",
       "nodes": [
+        @include(node55.json)
       ]
     },
     "deprovisioned": {