Merge pull request #1574 from yahoo/bratseth/reboot-periodicly

Bratseth/reboot periodicly

22 files changed, 273 insertions, 38 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
index ca8214f35d6..59baeb143d4 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
@@ -29,9 +29,9 @@ import static com.yahoo.prelude.querytransform.CJKSearcher.TERM_ORDER_RELAXATION
 /**
  * Replaces query terms with their stems
  *
- * @author  <a href="mailto:mathiasm@yahoo-inc.com">Mathias Lidal</a>
- * @author  bratseth
- * @author  <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Mathias Lidal
+ * @author bratseth
+ * @author Steinar Knutsen
  */
 @After({PhaseNames.UNBLENDED_RESULT, TERM_ORDER_RELAXATION})
 @Provides(StemmingSearcher.STEMMING)
@@ -135,7 +135,7 @@ public class StemmingSearcher extends Searcher {
         if (b instanceof PrefixItem || !b.isWords()) return (Item) b;
 
         if (b.isFromQuery() && !b.isStemmed()) {
-            final Index index = indexFacts.getIndex(b.getIndexName());
+            Index index = indexFacts.getIndex(b.getIndexName());
             StemMode stemMode = index.getStemMode();
             if (stemMode != StemMode.NONE) return stem(b, isCJK, language, reverseConnectivity, index);
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index fe648108209..6106d6e6ba5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -201,6 +201,15 @@ public final class Node {
                         allocation, history, type);
     }
 
+    /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */
+    public Node withCurrentRebootGeneration(long generation, Instant instant) {
+        Status newStatus = status().withReboot(status().reboot().withCurrent(generation));
+        History newHistory = history();
+        if (generation > status().reboot().current())
+            newHistory = history.with(new History.Event(History.Event.Type.rebooted, instant));
+        return this.with(newStatus).with(newHistory);
+    }
+    
     /** Returns a copy of this node with the given history. */
     public Node with(History history) {
         return new Node(openStackId, ipAddresses, hostname, parentHostname, flavor, status, state, allocation, history, type);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 1922295814d..8b309112214 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -13,6 +13,7 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.NodeAcl;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
 import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
@@ -64,6 +65,7 @@ public class NodeRepository extends AbstractComponent {
 
     private final CuratorDatabaseClient zkClient;
     private final Curator curator;
+    private final Clock clock;
     private final NodeFlavors flavors;
     private final NameResolver nameResolver;
 
@@ -83,6 +85,7 @@ public class NodeRepository extends AbstractComponent {
     public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver) {
         this.zkClient = new CuratorDatabaseClient(flavors, curator, clock, zone, nameResolver);
         this.curator = curator;
+        this.clock = clock;
         this.flavors = flavors;
         this.nameResolver = nameResolver;
 
@@ -453,6 +456,9 @@ public class NodeRepository extends AbstractComponent {
                 .collect(Collectors.toList());
     }
 
+    /** Returns the time keeper of this system */
+    public Clock clock() { return clock; }
+    
     /** Create a lock which provides exclusive rights to making changes to the given application */
     public Mutex lock(ApplicationId application) { return zkClient.lock(application); }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
index 5b6cc3b11a3..0136eb4ab2c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -41,10 +41,6 @@ public abstract class Expirer extends Maintainer {
         this.expiryTime = expiryTime;
     }
 
-    private static Duration min(Duration a, Duration b) {
-        return a.toMillis() < b.toMillis() ? a : b;
-    }
-
     @Override
     protected void maintain() {
         List<Node> expired = new ArrayList<>();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index 59cd8f5f85c..e988780ff3c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -37,6 +37,10 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     /** Returns the node repository */
     protected NodeRepository nodeRepository() { return nodeRepository; }
 
+    protected static Duration min(Duration a, Duration b) {
+        return a.toMillis() < b.toMillis() ? a : b;
+    }
+
     /** Returns the rate at which this job is set to run */
     protected Duration rate() { return rate; }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index af327801683..61381073334 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -68,10 +68,6 @@ public class NodeFailer extends Maintainer {
         constructionTime = clock.instant();
     }
 
-    private static Duration min(Duration d1, Duration d2) {
-        return d1.toMillis() < d2.toMillis() ? d1 : d2;
-    }
-
     @Override
     protected void maintain() {
         // Ready nodes
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
new file mode 100644
index 00000000000..c54c2c543a6
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
@@ -0,0 +1,61 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+
+/**
+ * This schedules periodic reboot of all nodes.
+ * We reboot nodes periodically to surface problems at reboot with a smooth frequency rather than
+ * potentially in burst when many nodes need to be rebooted for external reasons.
+ * 
+ * @author bratseth
+ */
+public class NodeRebooter extends Maintainer {
+    
+    private final Duration rebootInterval;
+    private final Clock clock;
+    private final Random random;
+
+    public NodeRebooter(NodeRepository nodeRepository, Clock clock, Duration rebootInterval) {
+        super(nodeRepository, min(Duration.ofMinutes(25), rebootInterval));
+        this.rebootInterval = rebootInterval;
+        this.clock = clock;
+        this.random = new Random(clock.millis()); // seed with clock for test determinism   
+    }
+
+    @Override
+    protected void maintain() {
+        // Reboot candidates: Nodes in long-term states, which we know an safely orchestrate a reboot
+        List<Node> rebootCandidates = nodeRepository().getNodes(NodeType.tenant, Node.State.active, Node.State.ready);
+        rebootCandidates.addAll(nodeRepository().getNodes(NodeType.proxy, Node.State.active, Node.State.ready));
+
+        for (Node node : rebootCandidates) {
+            if (shouldReboot(node))
+                nodeRepository().reboot(NodeListFilter.from(node));
+        }
+    }
+    
+    private boolean shouldReboot(Node node) {
+        Optional<History.Event> lastReboot = node.history().event(History.Event.Type.rebooted);
+        if (lastReboot.isPresent())
+            return lastReboot.get().at().plus(rebootInterval).isBefore(clock.instant());
+        else // schedule with a probability such that reboots of nodes are spread roughly over the reboot interval
+            return random.nextDouble() < (double)rate().getSeconds() / (double)rebootInterval.getSeconds();
+    }
+
+    @Override
+    public String toString() {
+        return "Node rebooter";
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index acd5b9d2184..6e3a03d4e5e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -31,6 +31,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final RetiredExpirer retiredExpirer;
     private final FailedExpirer failedExpirer;
     private final DirtyExpirer dirtyExpirer;
+    private final NodeRebooter nodeRebooter;
 
     @Inject
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Curator curator,
@@ -51,6 +52,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         inactiveExpirer = new InactiveExpirer(nodeRepository, clock, fromEnv("inactive_expiry").orElse(defaults.inactiveExpiry));
         failedExpirer = new FailedExpirer(nodeRepository, zone, clock, fromEnv("failed_expiry").orElse(defaults.failedExpiry));
         dirtyExpirer = new DirtyExpirer(nodeRepository, clock, fromEnv("dirty_expiry").orElse(defaults.dirtyExpiry));
+        nodeRebooter = new NodeRebooter(nodeRepository, clock, fromEnv("reboot_interval").orElse(defaults.rebootInterval));
     }
 
     private Optional<Duration> fromEnv(String envVariable) {
@@ -68,6 +70,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         retiredExpirer.deconstruct();
         failedExpirer.deconstruct();
         dirtyExpirer.deconstruct();
+        nodeRebooter.deconstruct();
     }
 
     private static class DefaultTimes {
@@ -85,6 +88,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration retiredExpiry;
         private final Duration failedExpiry;
         private final Duration dirtyExpiry;
+        private final Duration rebootInterval;
 
         DefaultTimes(Environment environment) {
             if (environment.equals(Environment.prod)) {
@@ -98,6 +102,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                 retiredExpiry = Duration.ofDays(4); // enough time to migrate data
                 failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night
                 dirtyExpiry = Duration.ofHours(2); // enough time to clean the node
+                rebootInterval = Duration.ofDays(30); 
             } else {
                 // These values ensure tests and development is not delayed due to nodes staying around
                 // Use non-null values as these also determine the maintenance interval
@@ -109,6 +114,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                 retiredExpiry = Duration.ofMinutes(1);
                 failedExpiry = Duration.ofMinutes(10);
                 dirtyExpiry = Duration.ofMinutes(30);
+                rebootInterval = Duration.ofDays(30);
             }
         }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
index e6507c335fd..9bbc6fcde2d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
@@ -8,6 +8,7 @@ import java.time.Instant;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * An immutable record of the last event of each type happening to this node.
@@ -47,7 +48,7 @@ public class History {
         return new History(builder.build());
     }
 
-    /** Returns a copy of this history with the given event type removed (or an identical if it was not present) */
+    /** Returns a copy of this history with the given event type removed (or an identical history if it was not present) */
     public History without(Event.Type type) {
         return new History(builderWithout(type).build());
     }
@@ -64,7 +65,7 @@ public class History {
     public History recordStateTransition(Node.State from, Node.State to, Instant at) {
         if (from == to) return this;
         switch (to) {
-            case ready:    return this.with(new Event(Event.Type.readied, at));
+            case ready:    return this.withoutApplicationEvents().with(new Event(Event.Type.readied, at));
             case active:   return this.with(new Event(Event.Type.activated, at));
             case inactive: return this.with(new Event(Event.Type.deactivated, at));
             case reserved: return this.with(new Event(Event.Type.reserved, at));
@@ -73,6 +74,14 @@ public class History {
             default:       return this;
         }
     }
+    
+    /** 
+     * Events can be application or node level. 
+     * This returns a copy of this history with all application level events removed. 
+     */
+    private History withoutApplicationEvents() {
+        return new History(events().stream().filter(e -> ! e.type().isApplicationLevel()).collect(Collectors.toList()));
+    }
 
     /** Returns the empty history */
     public static History empty() { return new History(Collections.emptyList()); }
@@ -107,12 +116,28 @@ public class History {
         public enum Type { 
             // State move events
             readied, reserved, activated, deactivated, failed, deallocated, 
-            // An active node was retired
+            // The active node was retired
             retired,
-            // An active node went down according to the service monitor
+            // The active node went down according to the service monitor
             down, 
-            // A node made a config request, indicating it is live
-            requested 
+            // The node made a config request, indicating it is live
+            requested,
+            // The node was rebooted
+            rebooted(false);
+            
+            private final boolean applicationLevel;
+            
+            /** Creates an application level event */
+            Type() {
+                this.applicationLevel = true;
+            }
+
+            Type(boolean applicationLevel) {
+                this.applicationLevel = applicationLevel;
+            }
+            
+            /** Returns true if this is an application level event and false it it is a node level event */
+            public boolean isApplicationLevel() { return applicationLevel; }
         }
 
         @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeAcl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
index 8e94e0475bb..25000cd6574 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeAcl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
@@ -1,7 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision;
+package com.yahoo.vespa.hosted.provision.node;
 
 import com.google.common.collect.ImmutableList;
+import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.List;
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java
index 63e0493d53d..3e04415c552 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.node.filter;
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.vespa.hosted.provision.Node;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -27,6 +28,10 @@ public class NodeListFilter extends NodeFilter {
         return nodes.contains(node);
     }
 
+    public static NodeListFilter from(Node nodes) {
+        return new NodeListFilter(Collections.singletonList(nodes), null);
+    }
+
     public static NodeListFilter from(List<Node> nodes) {
         return new NodeListFilter(nodes, null);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 4e985bee630..5218f92c5ae 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -150,7 +150,7 @@ public class CuratorDatabaseClient {
                                     newNodeStatus(node, toState),
                                     toState,
                                     toState.isAllocated() ? node.allocation() : Optional.empty(),
-                                    newNodeHistory(node, toState),
+                                    node.history().recordStateTransition(node.state(), toState, clock.instant()),
                                     node.type());
             curatorTransaction.add(CuratorOperations.delete(toPath(node).getAbsolute()))
                               .add(CuratorOperations.create(toPath(toState, newNode.hostname()).getAbsolute(), nodeSerializer.toJson(newNode)));
@@ -182,16 +182,6 @@ public class CuratorDatabaseClient {
         return zone.environment() == Environment.staging || zone.environment() == Environment.test;
     }
 
-    private History newNodeHistory(Node node, Node.State toState) {
-        History history = node.history();
-
-        // wipe history when a node *becomes* ready to avoid expiring based on events under the previous allocation
-        if (node.state() != Node.State.ready && toState == Node.State.ready)
-            history = History.empty();
-
-        return history.recordStateTransition(node.state(), toState, clock.instant());
-    }
-
     /**
      * Returns all nodes which are in one of the given states.
      * If no states are given this returns all nodes.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
index 1e0ed85faec..01a18fd0ee4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
@@ -61,4 +61,5 @@ public class DnsNameResolver implements NameResolver {
             this.value = value;
         }
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 441f76df0b3..a9a98b6f1c6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -259,6 +259,7 @@ public class NodeSerializer {
             case "deallocated" : return History.Event.Type.deallocated;
             case "down" : return History.Event.Type.down;
             case "requested" : return History.Event.Type.requested;
+            case "rebooted" : return History.Event.Type.rebooted;
         }
         throw new IllegalArgumentException("Unknown node event type '" + eventTypeString + "'");
     }
@@ -273,6 +274,7 @@ public class NodeSerializer {
             case deallocated : return "deallocated";
             case down : return "down";
             case requested: return "requested";
+            case rebooted: return "rebooted";
         }
         throw new IllegalArgumentException("Serialized form of '" + nodeEventType + "' not defined");
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
index 8330b9daa42..54b45e94bac 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
@@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeAcl;
+import com.yahoo.vespa.hosted.provision.node.NodeAcl;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.io.File;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 4fcbeb576b5..3248ba5b058 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.provision.node.Status;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Clock;
 import java.util.Optional;
 
 /**
@@ -24,15 +25,17 @@ import java.util.Optional;
 public class NodePatcher {
 
     private final NodeFlavors nodeFlavors;
-
     private final Inspector inspector;
+    private final Clock clock;
+
     private Node node;
 
-    public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node) {
+    public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node, Clock clock) {
         try {
+            this.nodeFlavors = nodeFlavors;
             inspector = SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)).get();
             this.node = node;
-            this.nodeFlavors = nodeFlavors;
+            this.clock = clock;
         }
         catch (IOException e) {
             throw new RuntimeException("Error reading request body", e);
@@ -59,7 +62,7 @@ public class NodePatcher {
             case "convergedStateVersion" :
                 return node.with(node.status().withStateVersion(asString(value)));
             case "currentRebootGeneration" :
-                return node.with(node.status().withReboot(node.status().reboot().withCurrent(asLong(value))));
+                return node.withCurrentRebootGeneration(asLong(value), clock.instant());
             case "currentRestartGeneration" :
                 return patchCurrentRestartGeneration(asLong(value));
             case "currentDockerImage" :
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index 86de87c9620..7e329e1599b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -122,7 +122,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         String path = request.getUri().getPath();
         if ( ! path.startsWith("/nodes/v2/node/")) return ErrorResponse.notFoundError("Nothing at '" + path + "'");
         Node node = nodeFromRequest(request);
-        nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node).apply());
+        nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository.clock()).apply());
         return new MessageResponse("Updated " + node.hostname());
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
index 293fa8b9361..a722071370f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
@@ -48,7 +48,7 @@ public class ApplicationMaintainerTest {
         Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
         NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
         NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
-                new MockNameResolver().mockAnyLookup());
+                                                           new MockNameResolver().mockAnyLookup());
 
         createReadyNodes(15, nodeRepository, nodeFlavors);
         createHostNodes(2, nodeRepository, nodeFlavors);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
new file mode 100644
index 00000000000..8d1e0340ba4
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -0,0 +1,52 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Generic maintenance tester
+ * 
+ * @author bratseth
+ */
+public class MaintenanceTester {
+
+    private final Curator curator = new MockCurator();
+    public final ManualClock clock = new ManualClock(Instant.ofEpochMilli(0L)); // determinism
+    private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
+    private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+    public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
+                                                                     new MockNameResolver().mockAnyLookup());
+
+    public void createReadyTenantNodes(int count) {
+        List<Node> nodes = new ArrayList<>(count);
+        for (int i = 0; i < count; i++)
+            nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant));
+        nodes = nodeRepository.addNodes(nodes);
+        nodeRepository.setReady(nodes);
+    }
+
+    public void createReadyHostNodes(int count) {
+        List<Node> nodes = new ArrayList<>(count);
+        for (int i = 0; i < count; i++)
+            nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
+        nodes = nodeRepository.addNodes(nodes);
+        nodeRepository.setReady(nodes);
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
new file mode 100644
index 00000000000..20e8e7fbe6e
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java
@@ -0,0 +1,74 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class NodeRebooterTest {
+
+    @Test
+    public void testRebootScheduling() throws InterruptedException {
+        MaintenanceTester tester = new MaintenanceTester();
+        tester.createReadyTenantNodes(15);
+        tester.createReadyHostNodes(15);
+        
+        NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, Duration.ofMinutes(250));
+        // No nodes have a reboot event - reboots should be scheduled for most nodes during 10 invocations
+        // (the rebooter run interval is 25 minutes).
+        maintenanceIterations(rebooter, tester, 5);
+        assertEquals("About half of the nodes have reboot scheduled",
+                     6,
+                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+
+        maintenanceIterations(rebooter, tester, 5);
+        assertEquals("Most nodes have reboot scheduled", 
+                     13, 
+                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+        assertEquals("No nodes have 2 reboots scheduled",
+                     0,
+                     withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+        assertEquals("Host nodes are not rebooted",
+                     0,
+                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.host, Node.State.ready)).size());
+
+        maintenanceIterations(rebooter, tester, 11);
+        assertEquals("Reboot interval is 10x iteration interval, so the same number of nodes are now rebooted twice",
+                     13,
+                     withCurrentRebootGeneration(2L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+        assertEquals("The last 2 nodes have had their first reboot",
+                     2,
+                     withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size());
+    }
+    
+    private void maintenanceIterations(NodeRebooter rebooter, MaintenanceTester tester, int iterations) {
+        for (int i = 0; i < iterations; i++) {
+            rebooter.maintain();
+            tester.clock.advance(Duration.ofMinutes(25));
+            simulateReboot(tester);
+        }
+    }
+    
+    /** Set current reboot generation to the wanted reboot generation whenever it is larger (i.e record a reboot) */
+    private void simulateReboot(MaintenanceTester tester) {
+        for (Node node : tester.nodeRepository.getNodes(Node.State.ready, Node.State.active)) {
+            if (node.status().reboot().wanted() > node.status().reboot().current())
+                tester.nodeRepository.write(node.withCurrentRebootGeneration(node.status().reboot().wanted(), 
+                                                                             tester.clock.instant()));
+        }
+    }
+    
+    /** Returns the subset of the give nodes which have the given current reboot generation */
+    private List<Node> withCurrentRebootGeneration(long generation, List<Node> nodes) {
+        return nodes.stream().filter(n -> n.status().reboot().current() == generation).collect(Collectors.toList());
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 5a280cf3548..934c09495b2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -9,7 +9,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeAcl;
+import com.yahoo.vespa.hosted.provision.node.NodeAcl;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Before;
 import org.junit.Test;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index 3511ae5f7ac..9c8d52be982 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -45,6 +45,10 @@
     {
       "event": "reserved",
       "at": 123
+    },
+    {
+      "event": "rebooted",
+      "at": 123
     }
   ]
 }