Merge branch 'master' into bratseth/add-filter-test

33 files changed, 130 insertions, 65 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 8b309112214..c707cfc08ae 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -55,7 +55,7 @@ import java.util.stream.Collectors;
  * @author bratseth
  */
 // Node state transitions:
-// 1) (new) - > provisioned -> ready -> reserved -> active -> inactive -> dirty -> ready
+// 1) (new) - > provisioned -> dirty -> ready -> reserved -> active -> inactive -> dirty -> ready
 // 2) inactive -> reserved
 // 3) reserved -> dirty
 // 3) * -> failed | parked -> dirty | active | (removed)
@@ -151,10 +151,12 @@ public class NodeRepository extends AbstractComponent {
     private List<Node> getTrustedNodes(Node node) {
         final List<Node> trustedNodes = new ArrayList<>();
 
+        // For all cases below: Trust nodes in same application (if node is part of an application)
+        node.allocation().ifPresent(allocation -> trustedNodes.addAll(getNodes(allocation.owner())));
+
         switch (node.type()) {
             case tenant:
                 // Tenant nodes trust nodes in same application and all infrastructure nodes
-                node.allocation().ifPresent(allocation -> trustedNodes.addAll(getNodes(allocation.owner())));
                 trustedNodes.addAll(getNodes(NodeType.proxy));
                 trustedNodes.addAll(getConfigNodes());
                 break;
@@ -166,16 +168,18 @@ public class NodeRepository extends AbstractComponent {
                 break;
 
             case proxy:
-                // Proxy nodes only trust config servers and other proxy nodes. They also trust any traffic to ports
+                // Proxy nodes trust nodes in same application and config servers. They also trust any traffic to ports
                 // 4080/4443, but these static rules are configured by the node itself
                 trustedNodes.addAll(getConfigNodes());
-                trustedNodes.addAll(getNodes(NodeType.proxy));
+                if ( ! node.allocation().isPresent()) // TODO: Remove when proxy nodes are in zone app everywhere
+                    trustedNodes.addAll(getNodes(NodeType.proxy));
                 break;
 
             case host:
-                // Docker hosts trust all config servers and all Docker hosts
+                // Docker hosts trust nodes in same application and config servers
                 trustedNodes.addAll(getConfigNodes());
-                trustedNodes.addAll(getNodes(NodeType.host));
+                if ( ! node.allocation().isPresent()) // TODO: Remove when Docker hosts are in zone app everywhere
+                    trustedNodes.addAll(getNodes(NodeType.host));
                 break;
 
             default:
@@ -248,8 +252,8 @@ public class NodeRepository extends AbstractComponent {
     /** Sets a list of nodes ready and returns the nodes in the ready state */
     public List<Node> setReady(List<Node> nodes) {
         for (Node node : nodes)
-            if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty)
-                throw new IllegalArgumentException("Can not set " + node + " ready. It is not provisioned or dirty.");
+            if (node.state() != Node.State.dirty)
+                throw new IllegalArgumentException("Can not set " + node + " ready. It is not dirty.");
         try (Mutex lock = lockUnallocated()) {
             return zkClient.writeTo(Node.State.ready, nodes);
         }
@@ -295,24 +299,24 @@ public class NodeRepository extends AbstractComponent {
         return zkClient.writeTo(Node.State.inactive, nodes, transaction);
     }
 
-    /** Deallocates these nodes, causing them to move to the dirty state */
-    public List<Node> deallocate(List<Node> nodes) {
+    /** Move nodes to the dirty state */
+    public List<Node> setDirty(List<Node> nodes) {
         return performOn(NodeListFilter.from(nodes), node -> zkClient.writeTo(Node.State.dirty, node));
     }
 
     /** 
-     * Deallocate a node which is in the failed or parked state. 
-     * Use this to recycle failed nodes which have been repaired or put on hold.
+     * Set a node dirty, which is in the provisioned, failed or parked state.
+     * Use this to clean newly provisioned nodes or to recycle failed nodes which have been repaired or put on hold.
      *
      * @throws IllegalArgumentException if the node has hardware failure
      */
-    public Node deallocate(String hostname) {
-        Optional<Node> nodeToDeallocate = getNode(hostname, Node.State.failed, Node.State.parked);
-        if ( ! nodeToDeallocate.isPresent())
-            throw new IllegalArgumentException("Could not deallocate " + hostname + ": No such node in the failed or parked state");
-        if (nodeToDeallocate.get().status().hardwareFailure().isPresent())
+    public Node setDirty(String hostname) {
+        Optional<Node> nodeToDirty = getNode(hostname, Node.State.provisioned, Node.State.failed, Node.State.parked);
+        if ( ! nodeToDirty.isPresent())
+            throw new IllegalArgumentException("Could not deallocate " + hostname + ": No such node in the provisioned, failed or parked state");
+        if (nodeToDirty.get().status().hardwareFailure().isPresent())
             throw new IllegalArgumentException("Could not deallocate " + hostname + ": It has a hardware failure");
-        return deallocate(Collections.singletonList(nodeToDeallocate.get())).get(0);
+        return setDirty(Collections.singletonList(nodeToDirty.get())).get(0);
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index 8db67a4af67..2a7121b3e7a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -52,7 +52,7 @@ public class FailedExpirer extends Expirer {
             if (failCountIndicatesHwFail(zone) && recycleCandidate.status().failCount() >= 5) continue;
             nodesToRecycle.add(recycleCandidate);
         }
-        nodeRepository.deallocate(nodesToRecycle);
+        nodeRepository.setDirty(nodesToRecycle);
     }
 
     private boolean failCountIndicatesHwFail(Zone zone) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
index 652a3783d4b..1e37020f846 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
@@ -33,7 +33,7 @@ public class InactiveExpirer extends Expirer {
 
     @Override
     protected void expire(List<Node> expired) {
-        nodeRepository.deallocate(expired);
+        nodeRepository.setDirty(expired);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
index f45f8ebd086..6745c24e237 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
@@ -28,6 +28,6 @@ public class ReservationExpirer extends Expirer {
     }
 
     @Override
-    protected void expire(List<Node> expired) { nodeRepository.deallocate(expired); }
+    protected void expire(List<Node> expired) { nodeRepository.setDirty(expired); }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java
index b728b862686..f17fb3ff8ca 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.Collections;
@@ -47,7 +48,7 @@ public class ApplicationFilter extends NodeFilter {
     }
 
     public static ApplicationFilter from(String applicationIds, NodeFilter next) {
-        return new ApplicationFilter(HostFilter.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next);
+        return new ApplicationFilter(StringUtilities.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next);
     }
 
     public static ApplicationId toApplicationId(String applicationIdString) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java
index 65fd9dd9d4a..de091aebb68 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.node.filter;
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.Collections;
@@ -36,7 +37,7 @@ public class NodeTypeFilter extends NodeFilter {
 
     /** Returns a node filter which matches a comma or space-separated list of types */
     public static NodeTypeFilter from(String types, NodeFilter next) {
-        return new NodeTypeFilter(HostFilter.split(types).stream().map(NodeType::valueOf).collect(Collectors.toSet()), next);
+        return new NodeTypeFilter(StringUtilities.split(types).stream().map(NodeType::valueOf).collect(Collectors.toSet()), next);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java
index edf0463835a..2eaf4c4ad39 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.node.filter;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.HostFilter;
+import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.Objects;
@@ -35,7 +36,7 @@ public class ParentHostFilter extends NodeFilter {
 
     /** Returns a copy of the given filter which only matches for the given parent */
     public static ParentHostFilter from(String parentNames, NodeFilter filter) {
-        return new ParentHostFilter(HostFilter.split(parentNames).stream().collect(Collectors.toSet()), filter);
+        return new ParentHostFilter(StringUtilities.split(parentNames).stream().collect(Collectors.toSet()), filter);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java
index fa5972e59e1..a3468cf3dfd 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.Collections;
@@ -45,7 +46,7 @@ public class StateFilter extends NodeFilter {
 
     /** Returns a node filter which matches a comma or space-separated list of states */
     public static StateFilter from(String states, NodeFilter next) {
-        return new StateFilter(HostFilter.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next);
+        return new StateFilter(StringUtilities.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index f9d878dabec..969a9d8475c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -107,7 +107,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
             return new MessageResponse("Moved " + lastElement(path) + " to parked");
         }
         else if (path.startsWith("/nodes/v2/state/dirty/")) {
-            nodeRepository.deallocate(lastElement(path));
+            nodeRepository.setDirty(lastElement(path));
             return new MessageResponse("Moved " + lastElement(path) + " to dirty");
         }
         else if (path.startsWith("/nodes/v2/state/active/")) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 4d949d88d68..176ea9c923f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -85,6 +85,7 @@ public class MockNodeRepository extends NodeRepository {
 
         nodes = addNodes(nodes);
         nodes.remove(6);
+        nodes = setDirty(nodes);
         setReady(nodes);
         fail("host5.yahoo.com");
         move("host55.yahoo.com", Node.State.dirty);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
index a722071370f..e2bf9ede545 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
@@ -101,6 +101,7 @@ public class ApplicationMaintainerTest {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
@@ -109,6 +110,7 @@ public class ApplicationMaintainerTest {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 67126f30623..02833d3e229 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -110,7 +110,8 @@ public class FailedExpirerTest {
         nodeRepository.write(node2);
 
         // Allocate the nodes
-        nodeRepository.setReady(nodeRepository.getNodes(NodeType.tenant, Node.State.provisioned));
+        List<Node> provisioned = nodeRepository.getNodes(NodeType.tenant, Node.State.provisioned);
+        nodeRepository.setReady(nodeRepository.setDirty(provisioned));
         ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
         provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(3), 1, null);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index bf954cf5feb..a3f636a3156 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -83,9 +83,10 @@ public class InactiveAndFailedExpirerTest {
         List<Node> inactiveNodes = tester.getNodes(applicationId, Node.State.inactive).asList();
         assertEquals(1, inactiveNodes.size());
 
-        // Check reboot generation before node is moved
+        // Check reboot generation before node is moved. New nodes transistion from provisioned to dirty, so their
+        // wanted reboot generation will always be 1.
         long wantedRebootGeneration = inactiveNodes.get(0).status().reboot().wanted();
-        assertEquals(0, wantedRebootGeneration);
+        assertEquals(1, wantedRebootGeneration);
 
         // Inactive times out and node is moved to dirty
         tester.advanceTime(Duration.ofMinutes(14));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index 8d1e0340ba4..9223412be9a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -13,11 +13,11 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 
-import java.time.Clock;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * Generic maintenance tester
@@ -38,6 +38,8 @@ public class MaintenanceTester {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
+        nodes = simulateInitialReboot(nodes);
         nodeRepository.setReady(nodes);
     }
 
@@ -46,7 +48,16 @@ public class MaintenanceTester {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
+        nodes = simulateInitialReboot(nodes);
         nodeRepository.setReady(nodes);
     }
 
+    /** Simulate the initial reboot the node performs when it's in dirty */
+    private List<Node> simulateInitialReboot(List<Node> nodes) {
+        return nodes.stream()
+                .map(n -> n.withCurrentRebootGeneration(n.status().reboot().wanted(), Instant.now(clock)))
+                .collect(Collectors.toList());
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 81d65bfefdf..23dd41daae6 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -179,6 +179,7 @@ public class NodeFailTester {
         for (int i = startIndex; i < startIndex + count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), flavor, nodeType));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
@@ -187,6 +188,7 @@ public class NodeFailTester {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
index 20e8e7fbe6e..8d359a892ea 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
@@ -17,42 +17,37 @@ public class NodeRebooterTest {
 
     @Test
     public void testRebootScheduling() throws InterruptedException {
+        Duration rebootInterval = Duration.ofMinutes(250);
         MaintenanceTester tester = new MaintenanceTester();
         tester.createReadyTenantNodes(15);
         tester.createReadyHostNodes(15);
+        // New nodes are rebooted when transitioning from dirty to ready. Advance the time so that additional reboots
+        // will be performed.
+        tester.clock.advance(rebootInterval);
         
-        NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, Duration.ofMinutes(250));
-        // No nodes have a reboot event - reboots should be scheduled for most nodes during 10 invocations
-        // (the rebooter run interval is 25 minutes).
-        maintenanceIterations(rebooter, tester, 5);
-        assertEquals("About half of the nodes have reboot scheduled",
-                     6,
-                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+        NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, rebootInterval);
 
-        maintenanceIterations(rebooter, tester, 5);
-        assertEquals("Most nodes have reboot scheduled", 
-                     13, 
-                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+        maintenanceIterations(rebooter, tester, 1);
+        assertEquals("All tenant nodes have reboot scheduled",
+                     15,
+                     withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
         assertEquals("No nodes have 2 reboots scheduled",
                      0,
-                     withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+                     withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
         assertEquals("Host nodes are not rebooted",
-                     0,
+                     15,
                      withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.host, Node.State.ready)).size());
 
         maintenanceIterations(rebooter, tester, 11);
         assertEquals("Reboot interval is 10x iteration interval, so the same number of nodes are now rebooted twice",
-                     13,
-                     withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
-        assertEquals("The last 2 nodes have had their first reboot",
-                     2,
-                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+                     15,
+                     withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
     }
     
     private void maintenanceIterations(NodeRebooter rebooter, MaintenanceTester tester, int iterations) {
         for (int i = 0; i < iterations; i++) {
-            rebooter.maintain();
             tester.clock.advance(Duration.ofMinutes(25));
+            rebooter.maintain();
             simulateReboot(tester);
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index 98640728ae7..8bf15bed0b7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -46,9 +46,10 @@ public class ReservationExpirerTest {
         nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
         nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.host));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
 
         // Reserve 2 nodes
-        assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.provisioned).size());
+        assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.dirty).size());
         nodeRepository.setReady(nodes);
         ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build();
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index a3f31ca5bcf..a71097cae06 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -127,6 +127,7 @@ public class RetiredExpirerTest {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
@@ -135,6 +136,7 @@ public class RetiredExpirerTest {
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 934c09495b2..1ed2bcd47ff 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -130,6 +130,31 @@ public class AclProvisioningTest {
         assertAcls(Arrays.asList(dockerHostNodes, configServers), acls.get(0));
     }
 
+
+    @Test
+    public void trusted_nodes_for_docker_hosts_and_proxy_nodes_in_zone_application() {
+        ApplicationId applicationId = tester.makeApplicationId(); // use same id for both allocate calls below
+        List<Node> configServers = setConfigServers("cfg1:1234,cfg2:1234,cfg3:1234");
+
+        // Populate repo
+        tester.makeReadyNodes(3, "default", NodeType.proxy);
+        tester.makeReadyNodes(2, "default", NodeType.host);
+
+        // Allocate 3 proxy nodes
+        List<Node> activeProxyNodes = allocateNodes(NodeType.proxy, applicationId);
+        assertEquals(3, activeProxyNodes.size());
+        // Allocate 2 Docker hosts, a total of 5 hosts
+        List<Node> activeDockerHostsAndProxyNodes = allocateNodes(NodeType.host, applicationId);
+        assertEquals(5, activeDockerHostsAndProxyNodes.size());
+
+        // Check trusted nodes for all nodes
+        activeDockerHostsAndProxyNodes.forEach(node -> {
+            System.out.println("Checking node " + node);
+            List<NodeAcl> nodeAcls = tester.nodeRepository().getNodeAcls(node, false);
+            assertAcls(Arrays.asList(activeDockerHostsAndProxyNodes, configServers), nodeAcls);
+        });
+    }
+
     @Test
     public void trusted_nodes_for_child_nodes_of_docker_host() {
         List<Node> configServers = setConfigServers("cfg1:1234,cfg2:1234,cfg3:1234");
@@ -173,10 +198,17 @@ public class AclProvisioningTest {
     }
 
     private List<Node> allocateNodes(int nodeCount) {
-        ApplicationId applicationId = tester.makeApplicationId();
+        return allocateNodes(Capacity.fromNodeCount(nodeCount), tester.makeApplicationId());
+    }
+
+    private List<Node> allocateNodes(NodeType nodeType, ApplicationId applicationId) {
+        return allocateNodes(Capacity.fromRequiredNodeType(nodeType), applicationId);
+    }
+
+    private List<Node> allocateNodes(Capacity capacity, ApplicationId applicationId) {
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"),
-                Optional.empty());
-        List<HostSpec> prepared = tester.prepare(applicationId, cluster, Capacity.fromNodeCount(nodeCount), 1);
+                                                  Optional.empty());
+        List<HostSpec> prepared = tester.prepare(applicationId, cluster, capacity, 1);
         tester.activate(applicationId, new HashSet<>(prepared));
         return tester.getNodes(applicationId, Node.State.active).asList();
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index da0fd9cd23b..cb46a511611 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -216,6 +216,7 @@ public class ProvisioningTester implements AutoCloseable {
                                                 nodeFlavors.getFlavorOrThrow(flavor),
                                                 type));
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
         return nodes;
     }
@@ -234,6 +235,7 @@ public class ProvisioningTester implements AutoCloseable {
                                                 nodeFlavors.getFlavorOrThrow(flavor), NodeType.tenant));
         }
         nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.setDirty(nodes);
         nodeRepository.setReady(nodes);
         return nodes;
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java
index 84ef91faaf7..48696714d69 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java
@@ -97,6 +97,7 @@ public class RestApiTest {
             nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
             nodes = addNodes(nodes);
             nodes.remove(5);
+            nodes = setDirty(nodes);
             setReady(nodes);
             fail("host5.yahoo.com");
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 20440c2bc9e..e1ab2914cff 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -66,7 +66,7 @@ public class RestApiTest {
         assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot",
                         new byte[0], Request.Method.POST));
         assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"),
-                               "\"rebootGeneration\":3");
+                               "\"rebootGeneration\":4");
 
         // POST new nodes
         assertResponse(new Request("http://localhost:8080/nodes/v2/node",
@@ -83,6 +83,9 @@ public class RestApiTest {
         assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json");
 
         // PUT nodes ready
+        assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com",
+                        new byte[0], Request.Method.PUT),
+                "{\"message\":\"Moved host8.yahoo.com to dirty\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com",
                                    new byte[0], Request.Method.PUT),
                        "{\"message\":\"Moved host8.yahoo.com to ready\"}");
@@ -248,6 +251,9 @@ public class RestApiTest {
                                 getBytes(StandardCharsets.UTF_8),
                         Request.Method.POST),
                 "{\"message\":\"Added 1 nodes to the provisioned state\"}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/" + hostname,
+                        new byte[0], Request.Method.PUT),
+                "{\"message\":\"Moved foo.yahoo.com to dirty\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/" + hostname,
                                    new byte[0], Request.Method.PUT),
                        "{\"message\":\"Moved foo.yahoo.com to ready\"}");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
index 58cfdb9213e..df80e8ff311 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
@@ -26,7 +26,7 @@
   },
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
index 99b890ab565..2702263b35c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
@@ -28,7 +28,7 @@
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
   "wantedDockerImage":"image-123",
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "vespaVersion": "5.104.142",
   "hostedVersion": "2.1.2408",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
index ee8d339da19..0e8f36e652d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
@@ -26,7 +26,7 @@
   },
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
index dce281cf6e5..83b82a0a0da 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
@@ -24,7 +24,7 @@
   },
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index 9c8d52be982..a9922340449 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -28,7 +28,7 @@
   "restartGeneration": 0,
   "currentRestartGeneration": 1,
   "wantedDockerImage": "image-123",
-  "rebootGeneration": 1,
+  "rebootGeneration": 2,
   "currentRebootGeneration": 1,
   "vespaVersion": "5.104.142",
   "hostedVersion": "2.1.2408",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
index f2e9f5c7e03..4df8115df68 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
@@ -28,7 +28,7 @@
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
   "wantedDockerImage":"image-123",
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "currentDockerImage":"image-12",
   "failCount": 0,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
index 9b820f7dbb8..27246d9cb86 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
@@ -13,7 +13,7 @@
   "description":"Flavor-name-is-default",
   "minCpuCores":2.0,
   "environment":"BARE_METAL",
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 1,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
index ff4d2e94eb5..b8e9696242c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
@@ -13,7 +13,7 @@
   "description":"Flavor-name-is-default",
   "minCpuCores":2.0,
   "environment":"BARE_METAL",
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "vespaVersion": "1.2.3",
   "currentDockerImage":"image-123",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
index 80218f5fd33..b63263282ae 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
@@ -12,7 +12,7 @@
   "description":"Flavor-name-is-default",
   "minCpuCores":2.0,
   "environment":"BARE_METAL",
-  "rebootGeneration": 1,
+  "rebootGeneration": 2,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
index 39469080e1a..b146ccac8d1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
@@ -26,7 +26,7 @@
   },
   "restartGeneration": 0,
   "currentRestartGeneration": 0,
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure" : false,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
index f22f852a09d..d864e207897 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
@@ -12,7 +12,7 @@
   "description": "Flavor-name-is-default",
   "minCpuCores": 2.0,
   "environment": "BARE_METAL",
-  "rebootGeneration": 0,
+  "rebootGeneration": 1,
   "currentRebootGeneration": 0,
   "failCount": 0,
   "hardwareFailure": false,